1518 lines
78 KiB
Bash
1518 lines
78 KiB
Bash
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ================================= MASTER CONFIGURATION =======================================
|
||
# ==============================================================================================
|
||
# All user-facing variables for the unRAID script ecosystem.
|
||
# Scripts source this file — edit here, changes apply everywhere on next git pull.
|
||
#
|
||
# ── HOW THIS FILE WORKS ───────────────────────────────────────────────────────────────────────
|
||
# Every script sources Master.conf and common.sh at startup.
|
||
# Change a value here and it affects all scripts that use it — no hunting through files.
|
||
# To disable something: comment it out with # rather than deleting it.
|
||
# To add a new rsync profile: add a key to each PROFILE_* array.
|
||
# To add or remove orchestrator jobs: edit the arrays in the ORCHESTRATORS section.
|
||
#
|
||
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||
#
|
||
# Section Description
|
||
# ───────────────────────────────────────────────────────────────────────────────────────────
|
||
# HOST CONFIGURATION Server hostnames, SSH keys, Emby connection details, DATA_DIR
|
||
# LOGGING Enable or disable verbose logging
|
||
# NOTIFICATIONS unRAID native and Discord webhook settings
|
||
# GIT / REPO Gitea repository and SSH settings
|
||
#
|
||
# ── ORCHESTRATORS ──────────────────────────────────────────────────────────────────────────
|
||
# ARRAY START Scripts launched at array start (array_start.sh)
|
||
# DAILY SYNC MAINTENANCE Job list + media shares (daily_sync_maintenance.sh)
|
||
# WEEKLY SYNC MAINTENANCE Job list + sync jobs + sync settings (weekly_sync_maintenance.sh)
|
||
# MEDIA MANAGEMENT Job list for media_management.sh
|
||
#
|
||
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||
# RSYNC DEFAULTS Global fallback rsync options and limits
|
||
# REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort
|
||
# RSYNC PROFILE SYSTEM Per-profile overrides for appdata syncs
|
||
#
|
||
# ── FAILOVER ───────────────────────────────────────────────────────────────────────────────
|
||
# FAILOVER Mutual container failover between two servers
|
||
# FAILOVER TEST Simulated outage settings for failover_test.sh
|
||
# DDNS Script-controlled DDNS — absolute rules
|
||
# INTERNET LOSS Containers to stop when internet is lost
|
||
# TIERED CONTAINER LISTS What each server runs for the other per tier
|
||
# TIER DELAY SETTINGS How long before each tier activates (minutes)
|
||
# RSYNC WRITEBACK JOBS Appdata synced back to primary on handback
|
||
#
|
||
# ── DOCKER ESSENTIALS ──────────────────────────────────────────────────────────────────────
|
||
# DOCKER DAILY RESTART Containers restarted daily
|
||
# DOCKER WEEKLY RESTART Containers restarted weekly
|
||
# DOCKER WATCHDOG Continuous two-tier self-healing container monitoring
|
||
# DOCKER NETWORK CONNECT Connect containers to extra networks on array start
|
||
#
|
||
# ── UNRAID ESSENTIALS ──────────────────────────────────────────────────────────────────────
|
||
# REBOOT User warning delay before scheduled reboot
|
||
# MOVER Mover stop timeout
|
||
# SYSLOG FILTER Docker veth noise filter file path
|
||
# PHP-FPM PHP-FPM max children config
|
||
# CLEAR LOGS System log file paths
|
||
# WEBGUI WATCHDOG WebGUI nginx + emhttp monitoring and restart
|
||
#
|
||
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||
# MEDIA PERMISSIONS Share list, mode and owner for permissions script
|
||
# MEDIA CLEANER Anime and media folder lists and file patterns
|
||
# ARR CLEANUP Lidarr, Sonarr, Radarr orphan file cleanup
|
||
# ARR FAILED/STALLED RECOVERY Auto blocklist + re-search failed imports and stalled downloads
|
||
#
|
||
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||
# TRANSCODE MANAGER Ramdisk and SSD fallback transcode management
|
||
# TRANSCODE SERVER ARRAY Multi-server session monitoring (Emby, Jellyfin, Plex)
|
||
#
|
||
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
|
||
# CERTIFICATE MONITOR SSL certificate expiry monitoring
|
||
# BACKUP VERIFY Random sample checksum verification against remote
|
||
# SMART HEALTH Drive SMART attribute monitoring
|
||
# ZFS MEMORY SNAPSHOT Weekly ZFS health and memory diagnostic report
|
||
# BANDWIDTH MONITOR Daily rsync transfer logging and weekly summary
|
||
# HEALTH DIGEST Aggregated system health digest — always/smart/weekly
|
||
# EMBY SESSION REPORT Weekly Emby usage statistics via API
|
||
#
|
||
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||
# SYSTEM WATCHDOG Continuous system health monitoring — last line of defense
|
||
#
|
||
# ==============================================================================================
|
||
|
||
# ==============================================================================================
|
||
# ── HOST CONFIGURATION ────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# ━━━ Hosts ━━━
|
||
# Hostnames must match exact Docker/unRAID hostnames — case sensitive.
|
||
# Used by detect_hosts() in common.sh to determine which server is local and which is remote.
|
||
# Both servers run identical scripts — host detection makes them bidirectional.
|
||
HOST1="your-host1-hostname"
|
||
HOST2="your-host2-hostname"
|
||
|
||
# Data directory — persistent script state and statistics files.
|
||
# Array share — survives reboots, no flash drive wear.
|
||
# Created automatically if it doesn't exist.
|
||
# Only truly critical files (failover state, watchdog reboot log) stay on /boot/config.
|
||
DATA_DIR="/mnt/user/appdata/unraid_scripts/data"
|
||
|
||
# SSH keys for server-to-server rsync and failover container operations.
|
||
# Both keys must be in /root/.ssh/ and authorised in the remote server's authorized_keys.
|
||
HOST1_SSH_KEY="/root/.ssh/your_host1_key"
|
||
HOST2_SSH_KEY="/root/.ssh/your_host2_key"
|
||
|
||
# ━━━ Emby ━━━
|
||
# Defined once here — referenced by transcode_manager.sh, emby_session_report.sh,
|
||
# emby_database_repair.sh, weekly_sync_maintenance.sh, and TRANSCODE_SERVERS array.
|
||
# API key: Emby Dashboard → API Keys → + New Key
|
||
HOST1_EMBY_CONTAINER="Emby"
|
||
HOST1_EMBY_URL="http://localhost:8096"
|
||
HOST1_EMBY_API_KEY="your-emby-api-key"
|
||
|
||
HOST2_EMBY_CONTAINER="Emby-Secondary"
|
||
HOST2_EMBY_URL="http://localhost:8096" # same port — different server, different key
|
||
HOST2_EMBY_API_KEY="your-emby-api-key"
|
||
|
||
# ==============================================================================================
|
||
# ── PARTNERSHIP ────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Manages the relationship lifecycle between two unRAID servers.
|
||
# HOST1 is always the owner (source of truth) — HOST2 is always the mirror.
|
||
# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a --transfer operation.
|
||
# All other vars derive from HOST1/HOST2 — no duplicate IP or key configuration needed.
|
||
#
|
||
# State files live on /boot/config — survives reboots, available before array starts:
|
||
# /boot/config/partnership_HOST1.db ← HOST1 writes only
|
||
# /boot/config/partnership_HOST2.db ← HOST2 writes only
|
||
# Each server SSHes to write/read the other's file directly — no rsync needed
|
||
#
|
||
# critical_sync_maintenance.sh runs --check every 15min:
|
||
# Reads both state files via SSH
|
||
# Detects offboard requests → finalises from owner side
|
||
# Increments offline counter → auto-offboards after threshold
|
||
# Silent when healthy ✅
|
||
|
||
PARTNERSHIP_ENABLED=false
|
||
PARTNERSHIP_OWNER_HOST="HOST1" # "HOST1" or "HOST2" — flips on --transfer
|
||
|
||
# Auth containers whose WebUI URL gets reconfigured on onboard/offboard
|
||
# Format: "ContainerName|Port"
|
||
# These are the containers the mirror operator clicks in his Docker UI
|
||
# On onboard → pointed at owner's Tailscale IP
|
||
# On offboard → pointed back at localhost
|
||
PARTNERSHIP_AUTH_WEBUIS=(
|
||
"NginxProxyManager|81"
|
||
"Lldap|17170"
|
||
"Authelia|9091"
|
||
"Authelia-Secondary|9092"
|
||
)
|
||
|
||
# Paths on HOST1 that HOST2 operator should collect before backup retention expires
|
||
# Notified on offboard — no auto-deletion, manual collection
|
||
PARTNERSHIP_MIRROR_BACKUPS=(
|
||
# "/mnt/user/appdata-Failover/Host2-Emby"
|
||
# add any paths HOST2 wants to collect after separation
|
||
)
|
||
|
||
# Offboard settings
|
||
PARTNERSHIP_GRACE_HOURS=3 # hours after offboard before Tailscale removal
|
||
# backup access also expires at this time
|
||
# collect anything needed before this window closes
|
||
PARTNERSHIP_REMOVE_TAILSCALE=true # remove mirror from Tailscale on offboard
|
||
# false = skip removal (manual or testing)
|
||
|
||
# Tailscale API — used to remove HOST2 from tailnet on offboard
|
||
# API key: https://login.tailscale.com/admin/settings/keys
|
||
# Tailnet: your tailnet name (e.g. "yourname.github" or "youremail.com")
|
||
TAILSCALE_API_KEY="" # tskey-api-...
|
||
TAILSCALE_TAILNET="" # your tailnet name
|
||
|
||
# Transfer safety
|
||
PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership"
|
||
PARTNERSHIP_TRANSFER_STRIKES=3 # consecutive health checks required
|
||
PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20 # max attempts before giving up
|
||
|
||
# Onboard settings
|
||
PARTNERSHIP_ONBOARD_VERIFY=true # verify WebUI reachable after reconfiguration
|
||
PARTNERSHIP_ONBOARD_NOTIFY=true # notify both servers on completion
|
||
PARTNERSHIP_SYNC_INTERVAL=15 # minutes — Critical-Data sync frequency
|
||
|
||
# Auto-offboard threshold
|
||
PARTNERSHIP_OFFLINE_THRESHOLD=30 # days either server unreachable before auto-offboard
|
||
# works both directions — mirror offline 30d → owner offboards
|
||
# owner offline 30d → mirror offboards itself
|
||
|
||
# ==============================================================================================
|
||
# ── LOGGING ───────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# Controls verbose [LOG] output across all scripts.
|
||
# true = show detailed [LOG] lines — useful for debugging or first-time setup
|
||
# false = show only user-facing output — cleaner for scheduled runs
|
||
ENABLE_LOGGING=true
|
||
|
||
# ==============================================================================================
|
||
# ── NOTIFICATIONS ─────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# unRAID native notification system — integrates with the bell icon in the WebGUI.
|
||
# normal = job completed successfully / warning = something failed or needs attention
|
||
NOTIFY_UNRAID=true
|
||
|
||
# Discord webhook URL — leave blank to disable
|
||
DISCORD_WEBHOOK=""
|
||
|
||
# ==============================================================================================
|
||
# ── GIT / REPO ────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# Gitea self-hosted repository — used by git_pull_execute.sh.
|
||
# Detects Gitea container location at runtime — works through failover automatically.
|
||
# Falls back to GITEA_DOMAIN if local and Tailscale both fail.
|
||
GITEA_CONTAINER="Gitea" # exact Docker container name
|
||
GITEA_REPO_PATH="youruser/Unraid_Scripts.git" # repo path on Gitea server
|
||
GITEA_DOMAIN="" # e.g. git.yourdomain.com — requires NPM + DNS setup
|
||
TARGET_DIR="/mnt/user/appdata/unraid_scripts" # where scripts are cloned to
|
||
GITEA_SSH_KEY="/root/.ssh/your_gitea_key" # SSH key for authenticating to Gitea
|
||
SSH_PORT=221 # Gitea SSH port (default 22, Gitea often uses 222/221)
|
||
|
||
# ==============================================================================================
|
||
# ── ORCHESTRATORS ──────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# All orchestrator job lists live here — edit arrays to add/remove scripts.
|
||
# No changes to orchestrator scripts needed when adding or removing jobs.
|
||
|
||
# ━━━ Array Start ━━━
|
||
# Scripts launched by array_start.sh when the array comes online.
|
||
# Launched in order — each as a background process.
|
||
# One-shot scripts (ramdisk, syslog, fpm, network) run and exit naturally.
|
||
# Continuous scripts (watchdogs, failover) run until array stops.
|
||
|
||
ARRAY_START_SCRIPTS=(
|
||
"Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts
|
||
"unRAID_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill
|
||
"unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning
|
||
"unRAID_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted
|
||
"Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers
|
||
"unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop
|
||
"Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop
|
||
"Failover/failover.sh" # mutual failover — continuous loop
|
||
)
|
||
|
||
# ━━━ Daily Sync Maintenance ━━━
|
||
# daily_sync_maintenance.sh runs the media share sync built into the script first,
|
||
# then iterates DAILY_MAINTENANCE_SCRIPTS for additional jobs.
|
||
# Schedule: 0 1 * * * (1am daily)
|
||
|
||
DAILY_MAINTENANCE_SCRIPTS=(
|
||
"git_pull_execute.sh" # pull latest scripts — always runs first
|
||
"Docker_Essentials/docker_daily_restart.sh" # daily container restarts
|
||
)
|
||
|
||
# Media shares synced daily by daily_sync_maintenance.sh.
|
||
# Each server syncs only the shares it owns (source of truth) — direction is automatic.
|
||
# HOST1 pushes its truth shares to HOST2. HOST2 pushes its truth shares to HOST1.
|
||
# Never both pushing the same share — one server is always the truth holder.
|
||
# These shares use DEFAULT_RSYNC_OPTS — no profile entry needed.
|
||
# For shares needing custom options or container stops — create a profile in the RSYNC section.
|
||
|
||
HOST1_DAILY_SYNC_SHARES=(
|
||
/mnt/user/Books
|
||
/mnt/user/Intros
|
||
/mnt/user/Kids_Movies
|
||
/mnt/user/Kids_Tv_Shows
|
||
/mnt/user/Movies
|
||
/mnt/user/Music
|
||
/mnt/user/Music_Videos
|
||
/mnt/user/Nextcloud
|
||
/mnt/user/stand-up_comedy
|
||
/mnt/user/Sports
|
||
/mnt/user/Tv_Shows
|
||
/mnt/user/Anime_Shows-Old
|
||
/mnt/user/Anime_Movies-Old
|
||
)
|
||
|
||
HOST2_DAILY_SYNC_SHARES=(
|
||
/mnt/user/Anime_Movies
|
||
/mnt/user/Anime_Shows
|
||
)
|
||
|
||
# Personal encrypted shares — synced for offsite backup, independent of media shares.
|
||
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
|
||
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
|
||
HOST1_PERSONAL_SHARES=(
|
||
# /mnt/user/Host1-Personal # uncomment after creating encrypted dataset
|
||
)
|
||
|
||
HOST2_PERSONAL_SHARES=(
|
||
# /mnt/user/Host2-Personal # uncomment after creating encrypted dataset
|
||
)
|
||
|
||
# ━━━ Weekly Sync Maintenance ━━━
|
||
# weekly_sync_maintenance.sh handles the critical sync built into the script first:
|
||
# stop containers both sides → pull updates → sync Emby + Critical-Data → restart
|
||
# Then iterates WEEKLY_MAINTENANCE_SCRIPTS for additional jobs.
|
||
# Schedule: 30 2 * * 0 (Sunday 2:30am)
|
||
|
||
WEEKLY_MAINTENANCE_SCRIPTS=(
|
||
"Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync
|
||
)
|
||
|
||
# Shares synced by weekly_sync_maintenance.sh during the maintenance window.
|
||
# Containers are stopped both sides before these sync — full clean state guaranteed.
|
||
# Profiles drive container stops, excludes, and options — configure in RSYNC section.
|
||
# Order matters — Emby first, then auth stack.
|
||
WEEKLY_SYNC_JOBS=(
|
||
"/mnt/user/Media_Server/Emby" # emby profile — full clean mirror
|
||
"/mnt/user/appdata-Failover/Critical-Data" # critical-data profile — auth stack
|
||
)
|
||
|
||
# ━━━ Critical Sync Maintenance ━━━
|
||
# Shares synced every 15 minutes by critical_sync_maintenance.sh
|
||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||
# Order matters — Critical-Data first (auth stack), then emby-failover (dirty sync)
|
||
CRITICAL_SYNC_SHARES=(
|
||
"/mnt/user/appdata-Failover/Critical-Data" # auth stack — full sync
|
||
"/mnt/user/Media_Server/Emby|emby-failover" # Emby dirty sync — stays running
|
||
)
|
||
|
||
# Container update toggles for the weekly sync window.
|
||
# Containers are already stopped for the sync — updates pull at no extra downtime.
|
||
# Both false → sync only, no updates.
|
||
# Toggle false temporarily to skip updates without changing the schedule.
|
||
CRITICAL_SYNC_UPDATES=true # pull container updates locally
|
||
CRITICAL_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH
|
||
|
||
# Orchestrators that use rsync — read by rsync_stop.sh for auto-detection
|
||
# Add any new orchestrator that calls rsync.sh here — no script changes needed
|
||
# rsync_stop.sh will detect and handle it automatically on stop
|
||
RSYNC_ORCHESTRATORS=(
|
||
"daily_sync_maintenance"
|
||
"weekly_sync_maintenance"
|
||
)
|
||
|
||
# ━━━ Media Management ━━━
|
||
# Job list run directly by daily_sync_maintenance.sh after the media share sync.
|
||
# Runs sequentially — permissions first, then cleaners, then arr cleanup.
|
||
# Comment out any job to disable without removing it.
|
||
# Each individual script can still be run manually for one-off maintenance.
|
||
|
||
MEDIA_MANAGEMENT_JOBS=(
|
||
"Media/media_shares_permissions.sh" # apply permissions — runs first
|
||
"Media/media_cleaner.sh anime" # remove junk from anime shares
|
||
"Media/media_cleaner.sh media" # remove junk from media shares
|
||
"Media/lidarr_cleanup.sh" # remove orphaned music files
|
||
"Media/sonarr_cleanup.sh" # remove orphaned TV files
|
||
"Media/radarr_cleanup.sh" # remove orphaned movie files
|
||
"Docker_Essentials/downloaders_reset.sh" # clear stuck states + purge old history
|
||
)
|
||
|
||
# ==============================================================================================
|
||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# ━━━ Rsync Enable/Disable ━━━
|
||
# Two-tier toggle system — Tier 1 overrides Tier 2.
|
||
#
|
||
# Tier 1 — Global gate:
|
||
# RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions
|
||
# Use when: remote server completely offline, major maintenance, disaster recovery
|
||
#
|
||
# Tier 2 — Per-orchestrator (only applies when Tier 1 is true):
|
||
# Fine grained control — disable specific orchestrators while keeping others
|
||
# Use when: rebuilding secondary, testing, per-window bandwidth management
|
||
#
|
||
# Real world example (HOST2 data rebuild — your current situation):
|
||
# RSYNC_ENABLED=true ← rsync works, individual scripts run fine
|
||
# DAILY_RSYNC_ENABLED=false ← skip daily HDD syncs during rebuild
|
||
# WEEKLY_RSYNC_ENABLED=true ← Emby + Critical-Data still sync (NVMe, separate BW)
|
||
# FAILOVER_RSYNC_ENABLED=true ← handback writeback still works when needed
|
||
# → Run individual: bash Rsync/rsync.sh /mnt/user/Movies (test each share manually)
|
||
# → When ready: DAILY_RSYNC_ENABLED=true
|
||
|
||
RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below
|
||
DAILY_RSYNC_ENABLED=false # Tier 2 — daily_sync_maintenance.sh rsync section
|
||
WEEKLY_RSYNC_ENABLED=true # Tier 2 — weekly_sync_maintenance.sh rsync section
|
||
FAILOVER_RSYNC_ENABLED=true # Tier 2 — failover.sh writeback jobs on handback
|
||
CRITICAL_RSYNC_ENABLED=true # Tier 2 — critical_sync_maintenance.sh (every 15min)
|
||
|
||
# ━━━ Rsync Defaults ━━━
|
||
# Global fallback values used when no profile match is found.
|
||
# Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed.
|
||
# Appdata shares match profiles by directory basename (lowercased).
|
||
# If a profile key exists it overrides the global. If missing the global is used.
|
||
|
||
BW_LIMIT=12500 # KB/s — 12500 ≈ 100Mbit — network transfer speed cap
|
||
RETRY_COUNT=3 # retry attempts if rsync fails before giving up
|
||
SLEEP=300 # seconds between retry attempts
|
||
CRITICAL_CONTAINER_NAMES=() # containers to stop on REMOTE before rsync — profiles override
|
||
DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override
|
||
CONTAINER_DELAY=5 # seconds to wait before starting delayed containers
|
||
EXCLUDE_DIRS=() # directories to exclude from transfer — profiles override
|
||
|
||
# --delete removes files on remote that no longer exist on source (mirror behaviour)
|
||
# --inplace writes directly to destination — better for large files, avoids temp copies
|
||
# --no-whole-file forces delta transfer even on fast connections — sends only changed blocks
|
||
DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file)
|
||
|
||
# ━━━ Remote Health Checks ━━━
|
||
# Pre-flight check — aborts if remote rootfs (/) usage is at or above this percentage.
|
||
# When remote array is down, rsync writes land on rootfs — fills fast and crashes the server.
|
||
ROOTFS_WARN=75
|
||
|
||
# ━━━ Rsync Profile System ━━━
|
||
# Profiles allow per-share rsync behaviour without touching script logic.
|
||
# Profile key matched by basename of directory passed to rsync.sh (lowercased).
|
||
# Override with --profile=name flag.
|
||
#
|
||
# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit DEFAULT_RSYNC_OPTS.
|
||
# List ALL desired options explicitly when defining a profile.
|
||
#
|
||
# Current profiles:
|
||
# arrs_stack — arr databases — lower bandwidth, containers stopped for consistency
|
||
# critical-data — auth stack — containers stopped both sides, Authelia delayed start
|
||
# host1 — server-specific appdata — no container stops needed
|
||
# important-data — NextCloud + Postgres — NextCloud delayed start after Postgres
|
||
# emby — weekly clean sync — both Emby stopped, full mirror, minimal excludes
|
||
# called by weekly_sync_maintenance.sh only — do NOT schedule separately
|
||
# emby-failover — frequent dirty sync — Emby stays running, WAL excluded, critical data only
|
||
# also used for failover writeback on handback
|
||
|
||
declare -A PROFILE_RSYNC_OPTS=(
|
||
[arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace"
|
||
[critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete"
|
||
[host1]="-av --info=progress2 --bwlimit=$BW_LIMIT"
|
||
[important-data]="-av --human-readable --bwlimit=$BW_LIMIT"
|
||
[emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
|
||
[emby-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
|
||
)
|
||
|
||
# Per-profile bandwidth limits in KB/s — overrides global BW_LIMIT for that profile only
|
||
# Lower for shares running alongside other jobs, higher for time-sensitive critical data
|
||
declare -A PROFILE_BW_LIMIT=(
|
||
[arrs_stack]=5000 # lower — runs alongside other syncs, avoids saturating link
|
||
[critical-data]=9500 # high — small dataset, get it synced fast and clean
|
||
[host1]=8000
|
||
[important-data]=9500 # high — database sync needs to be fast
|
||
[emby]=8000 # medium — large full mirror, steady transfer
|
||
[emby-failover]=9500 # high — small critical dataset, sync as fast as possible
|
||
)
|
||
|
||
# Retry attempts per profile — how many times to retry before giving up on a failed sync
|
||
declare -A PROFILE_RETRY_COUNT=(
|
||
[arrs_stack]=3
|
||
[critical-data]=3
|
||
[host1]=3
|
||
[important-data]=3
|
||
[emby]=3
|
||
[emby-failover]=3
|
||
)
|
||
|
||
# Seconds to wait between retry attempts
|
||
# emby-failover shorter — frequent sync, faster retry on transient failures
|
||
declare -A PROFILE_SLEEP=(
|
||
[arrs_stack]=300
|
||
[critical-data]=300
|
||
[host1]=300
|
||
[important-data]=300
|
||
[emby]=300
|
||
[emby-failover]=120 # shorter — frequent dirty sync, retry faster
|
||
)
|
||
|
||
# Containers stopped on BOTH LOCAL and REMOTE servers before rsync.
|
||
# Local stops first — flushes databases cleanly before pushing data out.
|
||
# Remote stops next — prevents writes to destination while receiving.
|
||
# Only containers that were running get restarted — stopped containers stay stopped.
|
||
# Same container names on both servers — consistent naming is required by this ecosystem.
|
||
# If a container is not found on a server it is skipped gracefully, not errored.
|
||
# SPACE-SEPARATED STRINGS — converted to array at runtime
|
||
declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
|
||
[arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
|
||
[critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap NginxProxyManager Authelia Authelia-Secondary"
|
||
[host1]="Organizrv2 UptimeKuma VaultWarden"
|
||
[important-data]="Postgres-NextCloud NextCloud"
|
||
[emby]="Emby" # weekly clean sync — both Emby instances stopped, WAL checkpointed
|
||
[emby-failover]="" # dirty sync — Emby stays running both sides, WAL excluded from sync
|
||
)
|
||
|
||
# Containers that need a delay before starting after rsync completes.
|
||
# Database containers must be accepting connections before dependent apps start.
|
||
# Authelia waits for Mariadb + Redis. NextCloud waits for Postgres.
|
||
# SPACE-SEPARATED STRINGS — converted to array at runtime
|
||
declare -A PROFILE_DELAYED_CONTAINERS=(
|
||
[arrs_stack]=""
|
||
[critical-data]="Authelia Authelia-Secondary" # wait for Mariadb + Redis to be ready
|
||
[host1]=""
|
||
[important-data]="NextCloud" # wait for Postgres to accept connections
|
||
[emby]=""
|
||
[emby-failover]=""
|
||
)
|
||
|
||
# Seconds to wait before starting delayed containers
|
||
# 15s gives Mariadb, Redis, and LLDAP time to accept connections before Authelia starts
|
||
declare -A PROFILE_CONTAINER_DELAY=(
|
||
[arrs_stack]=5
|
||
[critical-data]=15 # Mariadb + Redis need time to accept connections
|
||
[host1]=5
|
||
[important-data]=10 # Postgres needs time before NextCloud
|
||
[emby]=5
|
||
[emby-failover]=5
|
||
)
|
||
|
||
# Directories excluded from rsync transfer per profile
|
||
# emby-failover excludes WAL files — safe to sync while Emby is running
|
||
# emby clean sync only excludes logs, transcodes, cache — full metadata mirror
|
||
# SPACE-SEPARATED STRINGS — converted to array at runtime
|
||
declare -A PROFILE_EXCLUDE_DIRS=(
|
||
[arrs_stack]="logs *.tmp"
|
||
[host1]="logs *.tmp"
|
||
[important-data]="logs *.tmp"
|
||
[critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt"
|
||
[emby]="logs transcodes cache crash*"
|
||
# emby-failover: Emby running, WAL excluded — only safe critical data synced
|
||
# users.db, library.db, authentication.db, config/ — everything else excluded
|
||
[emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root"
|
||
)
|
||
|
||
# Skip per-disk space check is no longer needed — check_remote_disks() auto-detects
|
||
# XFS and ZFS filesystem types from disks.ini, no manual configuration required
|
||
|
||
# ==============================================================================================
|
||
# ── FAILOVER ──────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Mutual container failover between two unRAID servers.
|
||
# Each server runs Failover/failover.sh independently via array_start.sh.
|
||
# All decisions based on two pings: remote reachable + internet reachable.
|
||
#
|
||
# States: NORMAL | FAILOVER | NO_INTERNET | DARK
|
||
#
|
||
# DDNS rules — absolute:
|
||
# Internet loss → stop own DDNS immediately
|
||
# Failover → start remote DDNS first (Tier 1)
|
||
# Handback → stop remote DDNS → rsync → start containers → start local DDNS last
|
||
#
|
||
# Tiers:
|
||
# Tier 1 — Immediate — vital services + Live TV
|
||
# Tier 2 — configurable delay — productivity services
|
||
# Tier 3 — configurable delay — secondary services
|
||
# Tier 4 — configurable delay — arrs + downloaders
|
||
|
||
EXTERNAL_IP="8.8.8.8"
|
||
FAILOVER_CHECK_INTERVAL=120
|
||
FAILOVER_HANDBACK_STRIKES=2
|
||
FAILOVER_STATE_FILE="/boot/config/failover_state.db"
|
||
FAILOVER_ENABLED=false # set true when HOST2 is back online and tested
|
||
# false = suppresses "not running" warnings in status scripts
|
||
|
||
# ━━━ Failover Test ━━━
|
||
FAILOVER_TEST_BLOCK_WAIT=150
|
||
FAILOVER_TEST_HANDBACK_WAIT=360
|
||
|
||
# ━━━ DDNS ━━━
|
||
HOST1_DDNS_CONTAINERS=(
|
||
"yourdomain.com"
|
||
)
|
||
|
||
HOST2_DDNS_CONTAINERS=(
|
||
"yourdomain.us"
|
||
)
|
||
|
||
# ━━━ Internet Loss ━━━
|
||
FAILOVER_HOST1_STOP_ON_NO_NET=(
|
||
"yourdomain.com"
|
||
)
|
||
|
||
FAILOVER_HOST2_STOP_ON_NO_NET=(
|
||
"yourdomain.us"
|
||
)
|
||
|
||
# ━━━ Tiered Container Lists ━━━
|
||
|
||
# HOST1 runs for HOST2
|
||
FAILOVER_HOST1_RUNS_FOR_HOST2_IMMEDIATE=(
|
||
"yourdomain.us"
|
||
"VaultWarden-Secondary"
|
||
# "container-placeholder"
|
||
)
|
||
|
||
FAILOVER_HOST1_RUNS_FOR_HOST2_2HR=(
|
||
# "container-placeholder"
|
||
)
|
||
|
||
FAILOVER_HOST1_RUNS_FOR_HOST2_6HR=(
|
||
# "container-placeholder"
|
||
)
|
||
|
||
FAILOVER_HOST1_RUNS_FOR_HOST2_18HR=(
|
||
# "container-placeholder"
|
||
)
|
||
|
||
# HOST2 runs for HOST1
|
||
FAILOVER_HOST2_RUNS_FOR_HOST1_IMMEDIATE=(
|
||
"yourdomain.com"
|
||
"Emby"
|
||
"VaultWarden"
|
||
"Dispatcharr"
|
||
"Dispatcharr-Basic"
|
||
"Dispatcharr-Iptv-Users"
|
||
"ErsatzTV-Emby"
|
||
)
|
||
|
||
FAILOVER_HOST2_RUNS_FOR_HOST1_2HR=(
|
||
"Postgres-NextCloud"
|
||
"NextCloud"
|
||
"PostgreSQL_Immich"
|
||
"Immich"
|
||
# "container-placeholder"
|
||
)
|
||
|
||
FAILOVER_HOST2_RUNS_FOR_HOST1_6HR=(
|
||
"Gitea"
|
||
# "container-placeholder"
|
||
)
|
||
|
||
FAILOVER_HOST2_RUNS_FOR_HOST1_18HR=(
|
||
"Sonarr"
|
||
"Radarr"
|
||
"Lidarr"
|
||
"Readarr"
|
||
"Prowlarr"
|
||
"Bazarr"
|
||
"SABnzbd"
|
||
"Qbittorrent"
|
||
"LidaTube"
|
||
"Pinchflat"
|
||
"ChannelTube"
|
||
# "container-placeholder"
|
||
)
|
||
|
||
# ━━━ Tier Delay Settings ━━━
|
||
# How long the primary server must be down before each tier activates — in minutes.
|
||
# Tier 1 is always immediate — Live TV and media can't wait.
|
||
# Set independently per host — adjust based on hardware and what's worth starting.
|
||
# Longer delays = less resource usage on covering server but slower recovery.
|
||
#
|
||
# HOST1's containers running on HOST2 (HOST1 is down):
|
||
HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich — can wait
|
||
HOST1_TIER3_DELAY=720 # 12 hours — secondary services — Gitea etc.
|
||
HOST1_TIER4_DELAY=1440 # 24 hours — full workflow — arrs and downloaders
|
||
|
||
# HOST2's containers running on HOST1 (HOST2 is down):
|
||
HOST2_TIER2_DELAY=240
|
||
HOST2_TIER3_DELAY=720
|
||
HOST2_TIER4_DELAY=1440
|
||
|
||
# ━━━ Rsync Writeback Jobs ━━━
|
||
# Syncs critical appdata BACK to primary server during handback after failover.
|
||
# Containers are stopped before writeback runs — clean source, no competing writes.
|
||
# Purpose: primary comes back online with the state that built up during its outage
|
||
# (watch states, auth changes, library updates that happened on HOST2)
|
||
#
|
||
# HOST*_TIER1_WRITEBACK_DELAY:
|
||
# Short outages skip Tier 1 writeback — primary state is more reliable than dirty sync data
|
||
# Only writeback if outage lasted longer than this many minutes
|
||
# 60 minutes = if HOST1 was down less than 1hr, don't bother writing back Emby
|
||
#
|
||
# Tier 4 writeback automatically syncs HOST*_DAILY_SYNC_SHARES back — no need to list those here
|
||
# Only add paths that are NOT in DAILY_SYNC_SHARES and need writeback after extended outage
|
||
|
||
HOST1_TIER1_WRITEBACK_DELAY=60 # minutes — skip Emby writeback if outage under 1hr
|
||
HOST2_TIER1_WRITEBACK_DELAY=60
|
||
|
||
# HOST1 writeback — run by HOST2 during HOST1 handback
|
||
FAILOVER_HOST1_WRITEBACK_TIER1=(
|
||
"/mnt/user/Media_Server/Emby" # watch states, playstates built up during outage
|
||
)
|
||
|
||
FAILOVER_HOST1_WRITEBACK_TIER2=(
|
||
"/mnt/user/appdata-Failover/Important-Data" # NextCloud + Postgres — files added during outage
|
||
)
|
||
|
||
FAILOVER_HOST1_WRITEBACK_TIER3=(
|
||
# "location-placeholder"
|
||
)
|
||
|
||
FAILOVER_HOST1_WRITEBACK_TIER4=(
|
||
# Edge cases outside HOST1_DAILY_SYNC_SHARES
|
||
"/mnt/user/appdata-Failover/Arrs_Stack" # arr databases — downloads queued during outage
|
||
)
|
||
|
||
# HOST2 writeback — run by HOST1 during HOST2 handback
|
||
FAILOVER_HOST2_WRITEBACK_TIER1=(
|
||
# "/mnt/user/appdata-Failover/Host2-Emby"
|
||
)
|
||
|
||
FAILOVER_HOST2_WRITEBACK_TIER2=(
|
||
# "/mnt/user/appdata-Failover/Host2-Important"
|
||
)
|
||
|
||
FAILOVER_HOST2_WRITEBACK_TIER3=(
|
||
# "location-placeholder"
|
||
)
|
||
|
||
FAILOVER_HOST2_WRITEBACK_TIER4=(
|
||
# Edge cases outside HOST2_DAILY_SYNC_SHARES
|
||
"/mnt/user/appdata-Failover/Arrs_Stack"
|
||
)
|
||
|
||
# ==============================================================================================
|
||
# ── DOCKER ESSENTIALS ─────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# ━━━ Downloaders Reset ━━━
|
||
# Daily maintenance reset for all download clients.
|
||
# Called by daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS before container restarts.
|
||
# Clears stuck states, purges old history, prepares each downloader for a clean daily cycle.
|
||
#
|
||
# Retention period — applies to: slskd failed imports, SABnzbd completed and failed history
|
||
DOWNLOADER_RETENTION_DAYS=7
|
||
|
||
# ── slskd ──
|
||
# Clears stuck/errored searches, dead transfer records, purges expired failed imports
|
||
# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected
|
||
HOST1_SLSKD_URL="http://localhost:8980"
|
||
HOST1_SLSKD_API_KEY="your-slskd-api-key"
|
||
HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports"
|
||
|
||
# ── SABnzbd ──
|
||
# Clears completed history, failed history, and stalled paused queue items
|
||
HOST1_SABNZBD_URL="http://localhost:8180"
|
||
HOST1_SABNZBD_API_KEY="your-sabnzbd-api-key"
|
||
|
||
# ── qBittorrent ──
|
||
# Last chance failsafe — deletes torrents older than QBIT_FAILSAFE_MIN_DAYS
|
||
# qBittorrent's own rules handle normal cleanup (ratio >= 1.25 OR 45 days inactive)
|
||
# This catches anything missed after extended time
|
||
# deleteFiles=false — removes torrent from qBit but leaves files on disk
|
||
# Radarr/Sonarr manage actual files independently
|
||
# QBIT_FAILSAFE_MIN_RATIO=0 disables ratio gate — age is the only condition
|
||
HOST1_QBIT_URL="http://localhost:8080"
|
||
HOST1_QBIT_USERNAME="admin"
|
||
HOST1_QBIT_PASSWORD="your-qbit-password"
|
||
QBIT_FAILSAFE_MIN_DAYS=180
|
||
QBIT_FAILSAFE_MIN_RATIO=0 # 0 = age only, no ratio requirement
|
||
|
||
# ━━━ Docker Daily Restart ━━━
|
||
# Containers restarted every day by docker_daily_restart.sh via daily_sync_maintenance.sh.
|
||
# These containers run better with a daily restart — not just "keeping things fresh".
|
||
# Dispatcharr specifically degrades over time without restart — daily is intentional.
|
||
# Schedule is set in daily_sync_maintenance.sh — runs at 1am as part of daily window.
|
||
# Case-sensitive — must match exact Docker container names.
|
||
DAILY_RESTART_CONTAINERS=(
|
||
"NginxProxyManager"
|
||
"Authelia"
|
||
"Dispatcharr-Iptv-Users"
|
||
"Dispatcharr" # Live TV scheduler — degrades without daily restart
|
||
"Dispatcharr-Basic"
|
||
"ErsatzTV-Emby"
|
||
)
|
||
|
||
# ━━━ Docker Weekly Restart ━━━
|
||
# Less critical services restarted weekly by docker_weekly_restart.sh.
|
||
# Called by weekly_sync_maintenance.sh Sunday 2:30am — containers already stopped
|
||
# for the weekly sync window so restart adds zero extra downtime.
|
||
# Weekly restarts also catch any pending image updates not applied during weekly sync.
|
||
WEEKLY_RESTART_CONTAINERS=(
|
||
"NextCloud"
|
||
"Organizrv2"
|
||
"AdGuard-Home"
|
||
"Immich"
|
||
)
|
||
|
||
# ━━━ Docker Watchdog ━━━
|
||
# Continuous two-tier self-healing container monitoring.
|
||
# Started by array_start.sh — runs until array stops.
|
||
# Re-sources Master.conf each cycle — add/remove containers without restarting watchdog.
|
||
# Silent when all healthy — only logs when something needs attention.
|
||
# Heartbeat fires periodically as proof of life even when everything is healthy.
|
||
#
|
||
# Tier 1 — strict monitoring of explicitly configured containers:
|
||
# Memory hard limits — immediate restart if container exceeds limit
|
||
# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT sustained strikes
|
||
# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT failed checks
|
||
# Required containers — must always be running, strike + skip list with auto-clear
|
||
#
|
||
# Tier 2 — global health scan of ALL running containers:
|
||
# Unhealthy status — Docker HEALTHCHECK unhealthy → restart
|
||
# OOM killed — kernel killed container → restart + notify
|
||
# Crash loop detection — RestartCount climbing → notify, critical above limit
|
||
# Dead containers — remove and restart
|
||
# Unexpected exits — non-zero exit code → restart
|
||
|
||
# Memory hard limits in MB — immediate restart if exceeded
|
||
# Container restarted the moment it crosses this line — no strike system
|
||
# 20GB=20480 16GB=16384 12GB=12288 10GB=10240
|
||
# 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024
|
||
declare -A WATCHDOG_CONTAINERS=(
|
||
["Emby"]=16384
|
||
["LidaTube"]=6144
|
||
["Tdarr"]=6144
|
||
["Code-Server"]=1024
|
||
)
|
||
|
||
# HTTP health check URLs — checked every cycle, strike system before restart
|
||
# Container must respond with HTTP 200 within CURL_TIMEOUT seconds
|
||
# Per-host — HOST1 and HOST2 may run different containers on different ports
|
||
declare -A HOST1_WATCHDOG_CONTAINER_URLS=(
|
||
["Emby"]="http://localhost:8096"
|
||
)
|
||
|
||
declare -A HOST2_WATCHDOG_CONTAINER_URLS=(
|
||
["Emby"]="http://localhost:8096"
|
||
)
|
||
|
||
# Required containers — must always be running
|
||
# Strike system: SYS_WATCHDOG_STRIKE_LIMIT strikes before restart attempt
|
||
# Persistent skip list: added after WATCHDOG_CONTAINER_RESTART_LIMIT restarts in window
|
||
# Skip list auto-clears when container recovers — no manual intervention needed
|
||
# Per-host — each server has different critical containers
|
||
HOST1_WATCHDOG_REQUIRED_CONTAINERS=(
|
||
"NginxProxyManager"
|
||
"Lldap"
|
||
"Authelia"
|
||
"Mariadb-Authelia"
|
||
"Redis-Authelia"
|
||
"Authelia-Secondary"
|
||
"Redis-Authelia-Secondary"
|
||
)
|
||
|
||
HOST2_WATCHDOG_REQUIRED_CONTAINERS=(
|
||
"NginxProxyManager"
|
||
# add HOST2 required containers here
|
||
)
|
||
|
||
# Strike state file — /tmp resets on reboot which is correct
|
||
# Fresh start after reboot means no stale strikes carrying over
|
||
WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db"
|
||
|
||
# CPU thresholds — normalised against total core count automatically at runtime
|
||
# SOFT = warn only, HARD = strike toward restart
|
||
# CPU_FAIL_LIMIT = consecutive HARD strikes before restart
|
||
SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU
|
||
HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU
|
||
CPU_FAIL_LIMIT=2 # consecutive hard CPU strikes before container restart
|
||
|
||
# Memory soft threshold — warn when container reaches this % of its WATCHDOG_CONTAINERS hard limit
|
||
# Does not trigger restart — informational only
|
||
SOFT_MEM_THRESHOLD=80
|
||
|
||
# HTTP responsiveness — consecutive failed checks before restart
|
||
# CURL_TIMEOUT = seconds before curl gives up on a single check
|
||
RESP_FAIL_LIMIT=2 # consecutive failed checks before restart
|
||
CURL_TIMEOUT=5 # seconds per check before timeout
|
||
|
||
# How often the watchdog runs its checks
|
||
# 900 = 15 minutes — long enough to not be noisy, short enough to catch issues quickly
|
||
# Containers have this long to recover before next check
|
||
DOCKER_WATCHDOG_INTERVAL=900 # seconds between watchdog cycles
|
||
|
||
# Heartbeat — proof of life logged periodically even when everything is healthy
|
||
# Useful to confirm the watchdog is still running without flooding logs
|
||
DOCKER_WATCHDOG_HEARTBEAT=true # true = log heartbeat / false = completely silent
|
||
DOCKER_WATCHDOG_HEARTBEAT_HOURS=1 # how often to log heartbeat (hours)
|
||
|
||
# Tier 2 master toggle — set false to disable global container scanning entirely
|
||
# When false only WATCHDOG_CONTAINERS and required containers are monitored
|
||
WATCHDOG_SCAN_ALL=true
|
||
|
||
# Containers to skip in Tier 2 scan entirely
|
||
# Useful for containers that legitimately exit/restart frequently
|
||
WATCHDOG_SCAN_IGNORE=(
|
||
"DashGate"
|
||
"PIA-WG-Config-Generator"
|
||
"Aperture"
|
||
"Aperture-Kids"
|
||
"pgvector-18-Apeture-Kids"
|
||
"Pgvector18-Aperture"
|
||
)
|
||
|
||
# Individual Tier 2 check toggles — disable specific checks without disabling Tier 2
|
||
WATCHDOG_RESTART_UNHEALTHY=true # restart containers with Docker HEALTHCHECK = unhealthy
|
||
WATCHDOG_RESTART_DEAD=true # restart containers in dead state
|
||
WATCHDOG_RESTART_CRASHED=true # restart containers that exited with non-zero code
|
||
WATCHDOG_NOTIFY_OOM=true # notify + restart OOM killed containers
|
||
WATCHDOG_NOTIFY_CRASHLOOP=true # notify when Docker RestartCount keeps climbing
|
||
|
||
# Crash loop threshold — notify critical if Docker has restarted this many times total
|
||
# Above this number the notification escalates to critical — manual intervention needed
|
||
WATCHDOG_CRASH_LIMIT=5
|
||
|
||
# Startup grace period — skip restarts while system is still booting after array start
|
||
# Prevents watchdog from restarting containers that are legitimately still initializing
|
||
WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures
|
||
|
||
# Restart loop protection — stops hammering a broken container
|
||
# If watchdog restarts a container more than LIMIT times in WINDOW hours → skip list
|
||
# Skip list auto-clears when container recovers healthy
|
||
WATCHDOG_CONTAINER_RESTART_LIMIT=3 # max watchdog restarts allowed in window
|
||
WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours
|
||
WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db"
|
||
# rolling restart history for loop detection
|
||
|
||
# Notification batching — one clean summary per cycle instead of one ping per event
|
||
# true = batch all events into one notification at end of cycle
|
||
# false = send one notification per event (noisy on busy systems)
|
||
WATCHDOG_BATCH_NOTIFY=true
|
||
|
||
# Dependency ordering — skip restarting a container if its dependency is also down
|
||
# Prevents restarting Authelia before its database is ready
|
||
# Space-separated list of dependencies per container
|
||
declare -A WATCHDOG_DEPENDENCIES=(
|
||
["Authelia"]="Mariadb-Authelia Redis-Authelia"
|
||
["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
|
||
["NextCloud"]="Postgres-NextCloud"
|
||
)
|
||
|
||
# ━━━ Docker Network Connect ━━━
|
||
# Connects containers to extra Docker networks on array start via array_start.sh.
|
||
# Useful for containers that need their own custom network but also need to be
|
||
# reachable from your main custom bridge network.
|
||
# Every container in NETWORK_CONNECT_CONTAINERS is connected to every network in
|
||
# NETWORK_CONNECT_NETWORKS — containers not found are skipped gracefully.
|
||
NETWORK_CONNECT_CONTAINERS=(
|
||
"memcached"
|
||
"Npm-CrowdSec"
|
||
)
|
||
|
||
NETWORK_CONNECT_NETWORKS=(
|
||
"high-availability" # must exist before array start — create in Docker settings
|
||
)
|
||
|
||
# ==============================================================================================
|
||
# ── UNRAID ESSENTIALS ─────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# ━━━ inotify Tuning ━━━
|
||
# Linux inotify limits — applied at every array start by inotify_tuning.sh
|
||
# Default unRAID values are very low — with many Docker containers watching files
|
||
# (Sonarr, Radarr, Lidarr, NextCloud etc.) you can silently exhaust the limit.
|
||
# Symptoms: containers miss file events, downloads not detected, library not updated.
|
||
# These settings are lost on reboot — reapplied automatically at array start.
|
||
INOTIFY_MAX_INSTANCES=1024 # default: 128 — max inotify instances per user
|
||
INOTIFY_MAX_WATCHES=524288 # default: 8192 — max files watched per instance
|
||
INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — max events queued before dropping
|
||
|
||
# ━━━ System Tuning Monitor ━━━
|
||
# Tracks inotify and php-fpm usage over time — read by sunday_morning_coffee_report.sh
|
||
# Snapshot written every 6 hours by system_tuning_monitor.sh
|
||
# Log bounded to TUNING_LOG_RETENTION days — auto-purges on each write
|
||
INOTIFY_WARN_PCT=80 # warn if inotify instances exceed this % of limit
|
||
PHP_FPM_WARN_PCT=80 # warn if php-fpm workers exceed this % of max_children
|
||
TUNING_MONITOR_LOG="$DATA_DIR/system_tuning_history.db"
|
||
TUNING_LOG_RETENTION=30 # days — enough for monthly trend visibility
|
||
|
||
# ━━━ Reboot ━━━
|
||
# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots.
|
||
# Gives users time to save work — 300s = 5 minutes
|
||
REBOOT_SLEEP=300
|
||
|
||
# ━━━ Mover ━━━
|
||
# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process.
|
||
# Gives mover time to finish current file transfer before being interrupted.
|
||
MOVER_STOP_TIMEOUT=300
|
||
|
||
# ━━━ Syslog Filter ━━━
|
||
# Path for the rsyslog filter file that suppresses Docker veth interface noise.
|
||
# Docker creates a new veth interface for each container — generates hundreds of
|
||
# log lines per hour that have no diagnostic value. Filter removes them at source.
|
||
FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"
|
||
|
||
# ━━━ PHP-FPM ━━━
|
||
# Higher max_children allows more concurrent PHP requests to the unRAID WebGUI.
|
||
# Default is very low — increasing it prevents WebGUI slowdowns under load.
|
||
# 250 is safe for servers with 32GB+ RAM.
|
||
PHP_CONF="/etc/php-fpm.d/www.conf"
|
||
PHP_MAX_CHILDREN=250
|
||
|
||
# ━━━ Clear Logs ━━━
|
||
# System log files cleared weekly to prevent rootfs fill over time.
|
||
# These grow continuously — without clearing they eventually consume all rootfs space.
|
||
LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg)
|
||
|
||
# ━━━ WebGUI Watchdog ━━━
|
||
# Monitors unRAID WebGUI responsiveness — escalates through nginx restart → emhttp restart.
|
||
# Separate from docker_watchdog — this monitors the unRAID UI itself, not containers.
|
||
# WEBGUI_NGINX_WAIT = seconds after nginx restart before rechecking
|
||
# WEBGUI_EMHTTP_WAIT = seconds after emhttp restart before rechecking
|
||
WEBGUI_URL="http://localhost"
|
||
WEBGUI_TIMEOUT=5 # seconds before curl gives up on WebGUI check
|
||
WEBGUI_NGINX_WAIT=15 # seconds after nginx restart before rechecking
|
||
WEBGUI_EMHTTP_WAIT=30 # seconds after emhttp restart before rechecking
|
||
|
||
# ==============================================================================================
|
||
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# ━━━ Media Permissions ━━━
|
||
# Applied recursively to all shares in MEDIA_PERMISSION_SHARES by media_shares_permissions.sh.
|
||
# Runs first in MEDIA_MANAGEMENT_JOBS — arr cleanup scripts depend on correct ownership.
|
||
# 777 mode = read/write/execute for all users — standard for unRAID media shares
|
||
# nobody:users = standard unRAID media share ownership
|
||
PERMISSIONS_MODE="777"
|
||
PERMISSIONS_OWNER="nobody:users"
|
||
|
||
MEDIA_PERMISSION_SHARES=(
|
||
/mnt/user/Anime_Movies
|
||
/mnt/user/Anime_Movies-Old
|
||
/mnt/user/Anime_Shows
|
||
/mnt/user/Anime_Shows-Old
|
||
/mnt/user/appcache
|
||
/mnt/user/Books
|
||
/mnt/user/Downloads
|
||
/mnt/user/Games
|
||
/mnt/user/Intros
|
||
/mnt/user/Kids_Movies
|
||
/mnt/user/Kids_Tv_Shows
|
||
/mnt/user/Movie_Recordings
|
||
/mnt/user/Movies
|
||
/mnt/user/Music
|
||
/mnt/user/Music_Videos
|
||
/mnt/user/Photo
|
||
/mnt/user/Sports
|
||
/mnt/user/stand-up_comedy
|
||
/mnt/user/Temp_Storage
|
||
/mnt/user/Tv_Recordings
|
||
/mnt/user/Tv_Shows
|
||
/mnt/user/YouTube
|
||
)
|
||
|
||
# ━━━ Media Cleaner ━━━
|
||
# Removes junk files from media shares — two profiles: anime and media.
|
||
# Called via MEDIA_MANAGEMENT_JOBS. Run manually: Media/media_cleaner.sh anime|media
|
||
|
||
ANIME_CLEAN_FOLDERS=(
|
||
/mnt/user/Anime_Movies
|
||
/mnt/user/Anime_Movies-Old
|
||
/mnt/user/Anime_Shows
|
||
/mnt/user/Anime_Shows-Old
|
||
)
|
||
|
||
MEDIA_CLEAN_FOLDERS=(
|
||
/mnt/user/Kids_Movies
|
||
/mnt/user/Kids_Tv_Shows
|
||
/mnt/user/Movies
|
||
/mnt/user/Music
|
||
/mnt/user/Sports
|
||
/mnt/user/stand-up_comedy
|
||
/mnt/user/Tv_Shows
|
||
)
|
||
|
||
ANIME_FILE_PATTERNS=(
|
||
'*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk'
|
||
'*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*'
|
||
'*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp'
|
||
'*.log' '*.json'
|
||
)
|
||
|
||
MEDIA_FILE_PATTERNS=(
|
||
'*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk'
|
||
'*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*'
|
||
'*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp'
|
||
'*.log' '*.json' '*.iso' '*.lrc'
|
||
)
|
||
|
||
# ━━━ Arr Cleanup ━━━
|
||
# Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs.
|
||
# Compares tracked file paths from API against disk — deletes untracked files older than ORPHAN_AGE.
|
||
# detect_hosts() selects correct URL, API key, and root path at runtime.
|
||
#
|
||
# Protected patterns are NEVER deleted — cover art, metadata, subtitles generated by the arr
|
||
# are not included in the tracked file API response but must not be deleted.
|
||
#
|
||
# API versions and endpoint patterns:
|
||
# Sonarr v4 → /api/v3/series (get IDs) → /api/v3/episodefile?seriesId=X per series
|
||
# Radarr v5 → /api/v3/movie (get IDs) → /api/v3/moviefile?movieId=X per movie
|
||
# Lidarr v3 → /api/v1/artist (get IDs) → /api/v1/trackFile?artistId=X per artist
|
||
# All require per-ID loops — bulk endpoints removed in newer versions
|
||
#
|
||
# Version checking — scripts verify the arr major version matches before running
|
||
# If the arr updates and breaks the API the script exits safely before touching files
|
||
# Update the MAJOR version here when the script is updated to support a new version
|
||
# MINOR = 0 means any minor version within that major is accepted
|
||
|
||
SONARR_VERSION_MAJOR=4 # tested major version — script exits if major differs
|
||
RADARR_VERSION_MAJOR=6 # tested major version — script exits if major differs
|
||
LIDARR_VERSION_MAJOR=3 # tested major version — script exits if major differs
|
||
|
||
# ── Lidarr ────────────────────────────────────────────────────────────────────────────────────
|
||
HOST1_LIDARR_URL="http://localhost:8686"
|
||
HOST1_LIDARR_API_KEY="your-lidarr-api-key"
|
||
HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New"
|
||
LIDARR_LOCK_WARN_AGE=3600 # 1hr — large libraries take time, not stuck
|
||
|
||
# Container path → host path translation
|
||
# Lidarr stores file paths using container paths — script scans host paths
|
||
# Add one entry per root folder configured in Lidarr Settings → Media Management → Root Folders
|
||
declare -A HOST1_LIDARR_PATH_MAP=(
|
||
["/ext-music"]="/mnt/user/Music-New"
|
||
)
|
||
declare -A HOST2_LIDARR_PATH_MAP=(
|
||
# HOST2 does not run Lidarr — fill in if that changes
|
||
# ["/ext-music"]="/mnt/user/Music-New"
|
||
)
|
||
|
||
LIDARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion
|
||
# protects files that may still be mid-import or recently downloaded
|
||
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
|
||
LIDARR_PROTECTED_PATTERNS=(
|
||
# Metadata
|
||
"*.nfo" "*.tbn"
|
||
# Images — album art, artist images, Emby artwork
|
||
"*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
|
||
"poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
|
||
"banner.*" "thumb.*" "landscape.*"
|
||
"folder.*" "cover.*" "album.*" "artist.*" "disc.*"
|
||
# Lyrics
|
||
"*.lrc"
|
||
)
|
||
# NEVER deleted — cover art, metadata, lyrics
|
||
# Lidarr generates these but doesn't include them in trackFile API
|
||
# Without this protection cleanup would delete all your artwork
|
||
LIDARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this
|
||
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||
# protects against API returning partial data on a bad day
|
||
LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
|
||
# persists last known tracked count for percentage comparison
|
||
|
||
# ── Sonarr ────────────────────────────────────────────────────────────────────────────────────
|
||
HOST1_SONARR_URL="http://localhost:8989"
|
||
HOST1_SONARR_API_KEY="your-sonarr-api-key"
|
||
HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows"
|
||
|
||
# Container path → host path translation
|
||
# Add one entry per root folder configured in Sonarr Settings → Media Management → Root Folders
|
||
# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder
|
||
declare -A HOST1_SONARR_PATH_MAP=(
|
||
["/tv"]="/mnt/user/Tv_Shows"
|
||
["/ext-standup-comedy"]="/mnt/user/stand-up_comedy"
|
||
["/kids tv"]="/mnt/user/Kids_Tv_Shows"
|
||
["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old"
|
||
)
|
||
|
||
HOST2_SONARR_URL="http://localhost:8989"
|
||
HOST2_SONARR_API_KEY="your-sonarr-api-key"
|
||
HOST2_SONARR_TV_ROOT="/mnt/user/Anime_Shows"
|
||
|
||
declare -A HOST2_SONARR_PATH_MAP=(
|
||
# Fill in when HOST2 is back online
|
||
# ["/tv"]="/mnt/user/Anime_Shows"
|
||
)
|
||
|
||
SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion
|
||
SONARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this
|
||
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
|
||
SONARR_PROTECTED_PATTERNS=(
|
||
# Subtitles
|
||
"*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt"
|
||
# Metadata
|
||
"*.nfo" "*.tbn"
|
||
# Images — cover art, posters, fanart, Emby artwork
|
||
"*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
|
||
"poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
|
||
"banner.*" "thumb.*" "landscape.*"
|
||
# Kodi/Emby extras — not tracked by Sonarr API
|
||
"*-trailer.*" "*-featurette.*" "*-behindthescenes.*"
|
||
"*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*"
|
||
"*-clip.*" "*-other.*"
|
||
# Theme songs — stored in show folder, not tracked
|
||
"theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka"
|
||
)
|
||
|
||
# ── Radarr ────────────────────────────────────────────────────────────────────────────────────
|
||
HOST1_RADARR_URL="http://localhost:7878"
|
||
HOST1_RADARR_API_KEY="your-radarr-api-key"
|
||
HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies"
|
||
|
||
# Container path → host path translation
|
||
# Add one entry per root folder configured in Radarr Settings → Media Management → Root Folders
|
||
# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder
|
||
declare -A HOST1_RADARR_PATH_MAP=(
|
||
["/movies"]="/mnt/user/Movies"
|
||
["/kids movies"]="/mnt/user/Kids_Movies"
|
||
["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy"
|
||
["/anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||
)
|
||
|
||
HOST2_RADARR_URL="http://localhost:7878"
|
||
HOST2_RADARR_API_KEY="your-radarr-api-key"
|
||
HOST2_RADARR_MOVIES_ROOT="/mnt/user/Anime_Movies"
|
||
|
||
declare -A HOST2_RADARR_PATH_MAP=(
|
||
# Fill in when HOST2 is back online
|
||
# ["/anime-movies"]="/mnt/user/Anime_Movies"
|
||
)
|
||
|
||
RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion
|
||
RADARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this
|
||
RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov")
|
||
RADARR_PROTECTED_PATTERNS=(
|
||
# Subtitles
|
||
"*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt"
|
||
# Metadata
|
||
"*.nfo" "*.tbn"
|
||
# Images — cover art, posters, fanart, Emby artwork
|
||
"*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
|
||
"poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
|
||
"banner.*" "thumb.*" "landscape.*"
|
||
# Kodi/Emby extras — not tracked by Radarr API
|
||
"*-trailer.*" "*-featurette.*" "*-behindthescenes.*"
|
||
"*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*"
|
||
"*-clip.*" "*-other.*"
|
||
# Theme songs — stored in movie folder, not tracked
|
||
"theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka"
|
||
)
|
||
|
||
# ━━━ Arr Failed/Stalled Recovery ━━━
|
||
# Auto blocklist + re-search failed imports and stalled downloads.
|
||
# Runs every 6 hours — schedule: 0 */6 * * *
|
||
#
|
||
# Targets four problem types:
|
||
# importFailed — downloaded but arr couldn't import
|
||
# importPending — downloaded, stuck waiting to import (won't self-resolve)
|
||
# error status — serious failure not covered above
|
||
# stalled — download stuck with no connections or progress
|
||
#
|
||
# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first.
|
||
# API versions: Sonarr /api/v3/ — Radarr /api/v3/ — Lidarr /api/v1/
|
||
# Lidarr runs on HOST1 only — exits cleanly on HOST2.
|
||
|
||
ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this
|
||
# gives the arr time to retry on its own before we intervene
|
||
# matches cron interval — items are eligible after one missed cycle
|
||
|
||
# Per-arr enable/disable toggles — set false to temporarily disable without removing from cron
|
||
# Useful if an arr is having issues and you want to skip it for a few runs
|
||
HOST1_SONARR_RECOVERY=true # Tv_Shows import recovery
|
||
HOST1_RADARR_RECOVERY=true # Movies import recovery
|
||
HOST1_LIDARR_RECOVERY=true # Music import recovery — HOST1 only, exits cleanly on HOST2
|
||
HOST2_SONARR_RECOVERY=true # Anime_Shows import recovery
|
||
HOST2_RADARR_RECOVERY=true # Anime_Movies import recovery
|
||
|
||
# ==============================================================================================
|
||
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Session-based storage allocator using filesystem symlink indirection.
|
||
# ffmpeg resolves the symlink ONCE at session start — existing sessions are never affected.
|
||
#
|
||
# How it works:
|
||
# ramdisk_setup.sh — creates tmpfs and symlink at array start via array_start.sh
|
||
# transcode_management.sh — every 3min, runs cleanup then manager in correct order
|
||
# transcode_cleanup.sh — removes old inactive files
|
||
# transcode_manager.sh — manages symlink direction based on usage thresholds
|
||
#
|
||
# ⚠️ Docker mount — must use shared propagation:
|
||
# --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared
|
||
# Standard rprivate bind mounts lock the inode — sessions drift to SSD permanently.
|
||
|
||
# ━━━ Transcode Manager ━━━
|
||
# tmpfs mount point — created at array start by ramdisk_setup.sh
|
||
# Must exist before Emby starts so the symlink resolves correctly
|
||
RAMDISK_PATH="/mnt/ramdisk_transcodes"
|
||
|
||
# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront
|
||
# Set this to a comfortable limit based on your typical concurrent stream count
|
||
# Real-world: 9 streams peaked at ~5.5GB — 8G gives comfortable headroom
|
||
RAMDISK_SIZE="8G"
|
||
|
||
# Symlink that Emby points at — this path NEVER changes regardless of ramdisk/SSD state
|
||
# Emby resolves the symlink once per session at start — symlink flips are transparent
|
||
# Must match the container path configured in Emby's Extra Parameters
|
||
TRANSCODE_LINK="/mnt/ram-transcode"
|
||
|
||
# SSD fallback location — where transcodes land when ramdisk is too full
|
||
# Must have enough free space to handle peak session load
|
||
TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
|
||
|
||
# Usage thresholds in GB — hysteresis gap between WARN and LOW prevents flip-flop
|
||
# RAMDISK_WARN_GB: flip symlink to SSD when ramdisk usage reaches this
|
||
# RAMDISK_LOW_GB: flip symlink back to ramdisk when usage drops to this
|
||
# Gap (6.8 - 5.5 = 1.3GB) means ramdisk must drop 1.3GB before flipping back
|
||
# Without hysteresis a session right at the threshold causes rapid flipping
|
||
RAMDISK_WARN_GB=6.8
|
||
RAMDISK_LOW_GB=5.5
|
||
|
||
# Minimum free GB on SSD before allowing a flip to SSD
|
||
# Prevents flipping to SSD when it's almost full — that would be worse than a full ramdisk
|
||
RAMDISK_SSD_MIN_GB=20
|
||
|
||
# File age thresholds in minutes before cleanup eligibility
|
||
# TRANSCODE_MAX_AGE: HLS segment files older than this with no active session = clean up
|
||
# TRANSCODE_ORPHAN_AGE: files with no matching session at all = clean up
|
||
TRANSCODE_MAX_AGE=20
|
||
TRANSCODE_ORPHAN_AGE=30
|
||
|
||
# Notify if symlink flips this many times in one hour
|
||
# Frequent flips indicate the ramdisk is too small or thresholds need adjustment
|
||
TRANSCODE_FLIP_WARN=3
|
||
|
||
# Permissions applied to ramdisk and SSD transcode directories
|
||
TRANSCODE_OWNER="nobody:users"
|
||
TRANSCODE_CHMOD="755"
|
||
|
||
# Operating mode — controls symlink direction behaviour
|
||
# smart — auto-flips between ramdisk and SSD based on RAMDISK_WARN_GB / RAMDISK_LOW_GB
|
||
# hysteresis gap prevents flip-flop — default for production
|
||
# ramdisk — always uses ramdisk, never flips to SSD
|
||
# warns if RAMDISK_WARN_GB exceeded but holds position
|
||
# use during SSD maintenance or when SSD space is low
|
||
# ssd — always uses SSD, never flips to ramdisk
|
||
# use during ramdisk maintenance or after a ramdisk issue
|
||
TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd
|
||
|
||
# Daily statistics log — read by weekly_health_digest.sh for transcode summary
|
||
# Tracks peak usage, flip count, session ratio, files cleaned per day
|
||
# Bounded to TRANSCODE_LOG_RETENTION days — auto-purges old entries on each write
|
||
TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db"
|
||
TRANSCODE_LOG_RETENTION=90 # days before old entries are purged
|
||
|
||
# ━━━ Transcode Server Array ━━━
|
||
# All media servers sharing the ramdisk transcode space.
|
||
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
|
||
# Entries with placeholder API keys are skipped automatically.
|
||
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD.
|
||
|
||
TRANSCODE_SERVERS=(
|
||
"${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby"
|
||
# "${HOST2_EMBY_CONTAINER}|${HOST2_EMBY_URL}|${HOST2_EMBY_API_KEY}|emby"
|
||
# "Jellyfin|http://localhost:8097|jellyfin-api-key|jellyfin"
|
||
# "Plex|http://localhost:32400|plex-token|plex"
|
||
)
|
||
|
||
TRANSCODE_CHECK_EMBY=true
|
||
|
||
# ==============================================================================================
|
||
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# ━━━ Certificate Monitor ━━━
|
||
# Checks SSL certificate expiry via direct openssl connection — no NPM dependency.
|
||
# Checks the actual certificate served by each domain, not what NPM thinks it has.
|
||
# CERT_WARN_DAYS = notify this many days before expiry
|
||
# CERT_CRIT_DAYS = escalate to critical this many days before expiry
|
||
# CERT_TIMEOUT = seconds before giving up on the openssl connection
|
||
CERT_MONITOR_DOMAINS=(
|
||
"yourdomain.com"
|
||
"yourdomain.us"
|
||
)
|
||
CERT_WARN_DAYS=30 # warn when cert expires within this many days
|
||
CERT_CRIT_DAYS=7 # critical alert within this many days
|
||
CERT_TIMEOUT=10 # seconds per domain check
|
||
|
||
# ━━━ Backup Verify ━━━
|
||
# Verifies rsync mirror health by comparing random file checksums between servers.
|
||
# Catches silent corruption or incomplete syncs that rsync itself wouldn't detect.
|
||
# Leave BACKUP_VERIFY_SHARES empty to use HOST*_DAILY_SYNC_SHARES automatically.
|
||
# BACKUP_VERIFY_SAMPLE = number of random files to checksum per share
|
||
# BACKUP_VERIFY_MIN_SIZE = skip files smaller than this (small files are rarely corrupted)
|
||
BACKUP_VERIFY_SHARES=(
|
||
# leave empty to use HOST*_DAILY_SYNC_SHARES automatically
|
||
)
|
||
BACKUP_VERIFY_SAMPLE=10 # random files to check per share
|
||
BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample
|
||
|
||
# ━━━ SMART Health ━━━
|
||
# Monitors drive SMART attributes — discovers all drives automatically via /dev/sd* and /dev/nvme*.
|
||
# Reads live SMART data — no persistent writes.
|
||
# SMART_IGNORE_DRIVES = drives to skip (boot USB, drives without meaningful SMART data)
|
||
SMART_TEMP_WARN=45 # Celsius — warn above this temperature
|
||
SMART_TEMP_CRIT=55 # Celsius — critical above this temperature
|
||
SMART_IGNORE_DRIVES=(
|
||
"sda" # boot USB — SMART not meaningful on flash drives
|
||
)
|
||
|
||
# ━━━ ZFS Memory Snapshot ━━━
|
||
# Weekly ZFS pool health and memory diagnostic report — informational only, no action taken.
|
||
# ZFS_REPORT_ARC_WARN_PCT = warn if ARC is using more than this % of its max
|
||
# ZFS_REPORT_FREE_WARN_GB = warn if less than this GB free RAM
|
||
# ZFS_REPORT_AVAIL_WARN_GB = warn if less than this GB available on ZFS pool
|
||
# ZFS_REPORT_DOCKER_TOP = how many top Docker containers to show by memory usage
|
||
# ZFS_REPORT_IGNORE_POOLS = individual disk pools to skip (unRAID array disks as ZFS)
|
||
ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log"
|
||
ZFS_REPORT_ARC_WARN_PCT=90
|
||
ZFS_REPORT_FREE_WARN_GB=10
|
||
ZFS_REPORT_AVAIL_WARN_GB=20
|
||
ZFS_REPORT_DOCKER_TOP=10
|
||
ZFS_REPORT_IGNORE_POOLS=(
|
||
"disk10"
|
||
"disk9"
|
||
"disk8"
|
||
"disk6"
|
||
"disk5"
|
||
)
|
||
|
||
# ━━━ Bandwidth Monitor ━━━
|
||
# Called automatically by rsync.sh after each sync — one bounded write per run.
|
||
# Tracks transfer size, duration and profile per sync for weekly summary reporting.
|
||
# BANDWIDTH_LOG_RETENTION = days to keep entries before auto-purging old records
|
||
# BANDWIDTH_WARN_GB = flag in weekly summary if a single sync exceeded this size
|
||
BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db"
|
||
BANDWIDTH_LOG_RETENTION=90 # days before old entries are purged
|
||
BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report
|
||
|
||
# Stats files — written by cleanup and recovery scripts, read by coffee report
|
||
# All in DATA_DIR — array always running when these are written
|
||
ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats
|
||
ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats
|
||
|
||
# ━━━ Health Digest ━━━
|
||
# Aggregated system health summary — reads existing state files, no new writes.
|
||
# Three profiles control when the digest email is sent:
|
||
# always — sends every run regardless of findings
|
||
# smart — sends only when DIGEST_SMART_ON_* conditions are found
|
||
# weekly — sends once per week on DIGEST_DAY only
|
||
# Smart profile triggers — set true to send digest when finding is detected:
|
||
DIGEST_PROFILE="weekly" # always | smart | weekly
|
||
DIGEST_DAY="Sunday" # day of week for weekly profile
|
||
DIGEST_SMART_ON_WATCHDOG=true # send if any watchdog strikes are active
|
||
DIGEST_SMART_ON_FAILOVER=true # send if failover state is not NORMAL
|
||
DIGEST_SMART_ON_CERT_WARN=true # send if any cert is under CERT_WARN_DAYS
|
||
DIGEST_SMART_ON_BANDWIDTH=true # send if any transfer exceeded BANDWIDTH_WARN_GB
|
||
|
||
# ━━━ Emby Session Report ━━━
|
||
# Weekly Emby usage statistics via API — no persistent writes, queries fresh each run.
|
||
# Shows top content, most active users, session counts over the report period.
|
||
# URL and API key pulled from HOST1/HOST2_EMBY_URL and HOST1/HOST2_EMBY_API_KEY
|
||
# defined in Host Configuration at the top of this file — no duplication needed.
|
||
EMBY_REPORT_DAYS=7 # days to include in the report period
|
||
EMBY_REPORT_TOP_N=10 # number of top content items to show
|
||
|
||
# ==============================================================================================
|
||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Continuous system health monitoring — last line of defense before a crash.
|
||
# Started by array_start.sh — runs until array stops.
|
||
# Re-sources Master.conf each cycle — config changes take effect on next cycle.
|
||
# Strike system: sustained threshold hits trigger reboot — single spikes ignored.
|
||
# Reboot loop protection: shuts down instead if reboot limit hit in rolling window.
|
||
# Silent when healthy — logs only when a threshold is triggered.
|
||
|
||
# ━━━ State Files ━━━
|
||
SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" # /tmp resets on reboot ✅
|
||
SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" # survives reboots
|
||
SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db" # reboot loop detection
|
||
|
||
# ━━━ Strike and Reboot Loop Settings ━━━
|
||
# Strike system: a check must fail this many consecutive cycles before action is taken
|
||
# Single spikes (one bad reading) are ignored — sustained problems trigger reboot
|
||
SYS_WATCHDOG_STRIKE_LIMIT=2 # consecutive failures before reboot trigger
|
||
|
||
# How often checks run — 300s = 5 minutes
|
||
# At STRIKE_LIMIT=2 and INTERVAL=300: problem must persist 10min before reboot
|
||
SYSTEM_WATCHDOG_INTERVAL=300 # seconds between watchdog cycles
|
||
|
||
# Reboot loop protection — if system keeps rebooting something is seriously wrong
|
||
# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS hours → shutdown instead of reboot
|
||
# Prevents infinite reboot loops when the underlying problem can't be fixed by rebooting
|
||
SYS_WATCHDOG_REBOOT_LIMIT=3 # max reboots before shutdown instead
|
||
SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 # rolling window in hours
|
||
|
||
# Heartbeat — proof of life logged periodically even when everything is healthy
|
||
SYSTEM_WATCHDOG_HEARTBEAT=true # true = log heartbeat / false = completely silent
|
||
SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1 # how often to log heartbeat (hours)
|
||
|
||
# ━━━ Thresholds ━━━
|
||
# Set at "about to become unstable" levels — not "things are a bit high"
|
||
# These should be high enough that normal operation never triggers them
|
||
|
||
# rootfs (/) usage percentage — when array is down rsync writes land on rootfs
|
||
# fills rapidly and can crash the server — 95% is almost too late, act fast
|
||
SYS_WATCHDOG_ROOTFS_PCT=95
|
||
|
||
# /var/log usage percentage — log spam can fill rootfs, indicates something broken
|
||
SYS_WATCHDOG_LOG_PCT=95
|
||
|
||
# Free RAM in GB — below this is critically low, OOM or swap imminent
|
||
# Your server has 128GB — 4GB free means something is consuming everything
|
||
SYS_WATCHDOG_MEM_GB=4
|
||
|
||
# ZFS ARC pinned percentage — ARC not releasing after reclaim = memory stuck
|
||
# SYS_WATCHDOG_ARC_RELEASE_PCT = after reclaim attempt, if still above this → trigger
|
||
SYS_WATCHDOG_ARC_PINNED_PCT=98
|
||
SYS_WATCHDOG_ARC_RELEASE_PCT=95
|
||
|
||
# Load average multiplier — threshold = MULTIPLIER × CPU core count
|
||
# MULTIPLIER=3 on 16-core = load average of 48 before triggering
|
||
# Set high — transcoding causes legitimate high load spikes
|
||
SYS_WATCHDOG_LOAD_MULTIPLIER=3
|
||
|
||
# Zombie process count — large numbers indicate serious process management failure
|
||
# A few zombies are normal — 50 means something is very wrong
|
||
SYS_WATCHDOG_ZOMBIE_LIMIT=50
|
||
|
||
# CPU temperature in Celsius — sustained high temp causes throttling or kernel panic
|
||
# 95°C is close to tjmax on most CPUs — triggers before thermal shutdown
|
||
SYS_WATCHDOG_CPU_TEMP_MAX=95
|
||
|
||
# ━━━ Check Toggles ━━━
|
||
# Disable individual checks without disabling the whole watchdog
|
||
# All enabled by default except load — transcoding causes legitimate load spikes
|
||
SYS_WATCHDOG_CHECK_ROOTFS=true
|
||
SYS_WATCHDOG_CHECK_LOG=true
|
||
SYS_WATCHDOG_CHECK_RAM=true
|
||
SYS_WATCHDOG_CHECK_ARC=true
|
||
SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
||
SYS_WATCHDOG_CHECK_LOAD=false # disabled — load spikes during transcoding are normal
|
||
SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||
SYS_WATCHDOG_CHECK_CONTAINERS=true # checks docker_watchdog persistent skip list
|
||
SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true # checks if Docker daemon is responding
|
||
|
||
# ━━━ Abort Toggles ━━━
|
||
# Conditions that prevent reboot even when a threshold is hit
|
||
# true = abort reboot if this condition is active (conservative — avoid data loss)
|
||
# false = reboot anyway (aggressive — a clean reboot beats a hard crash)
|
||
# Philosophy: aborting is safer for data, rebooting is safer for stability
|
||
SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # rebooting with bad pool risks data loss
|
||
SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity is better than crashing mid-check
|
||
SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting move is better than crashing mid-move
|
||
|
||
# ==============================================================================================
|
||
# ──────────────────────── End Of User Variables ───────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
important_sync_maintenance.sh should be a new orch and follow the same structure in master as our other orchs. it should trigger the important rsync and trigger, arrs_failed_stalled. any thing else it should trigger |