1360 lines
68 KiB
Bash
1360 lines
68 KiB
Bash
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ================================= MASTER CONFIGURATION =======================================
|
||
# ==============================================================================================
|
||
# All user-facing variables for the unRAID script ecosystem.
|
||
# Scripts source this file — edit here, changes apply everywhere on next git pull.
|
||
#
|
||
# ── HOW THIS FILE WORKS ───────────────────────────────────────────────────────────────────────
|
||
# Every script sources Master.conf and common.sh at startup.
|
||
# Change a value here and it affects all scripts that use it — no hunting through files.
|
||
# To disable something: comment it out with # rather than deleting it.
|
||
# To add a new rsync profile: add a key to each PROFILE_* array in the RSYNC section.
|
||
# To add or remove orchestrator jobs: edit the arrays in the ORCHESTRATORS section.
|
||
#
|
||
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||
#
|
||
# Section Description
|
||
# ───────────────────────────────────────────────────────────────────────────────────────────
|
||
# HOST CONFIGURATION Server hostnames, SSH keys, Emby connection details, DATA_DIR
|
||
# PARTNERSHIP Mirror relationship lifecycle — onboard/offboard/transfer
|
||
# LOGGING Enable or disable verbose logging
|
||
# NOTIFICATIONS unRAID native and Discord webhook settings
|
||
# GIT / REPO Gitea repository and SSH settings
|
||
#
|
||
# ── ORCHESTRATORS ──────────────────────────────────────────────────────────────────────────
|
||
# ARRAY START Scripts launched at array start (array_start.sh)
|
||
# DAILY SYNC MAINTENANCE Job list + media shares (daily_sync_maintenance.sh)
|
||
# WEEKLY SYNC MAINTENANCE Job list + sync shares + update toggles (weekly_sync_maintenance.sh)
|
||
# CRITICAL SYNC MAINTENANCE 15-minute jobs + sync shares + partnership check (critical_sync_maintenance.sh)
|
||
#
|
||
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||
# RSYNC ENABLE/DISABLE Two-tier toggle system — global gate + per-orchestrator
|
||
# RSYNC DEFAULTS Global fallback rsync options and limits
|
||
# REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort
|
||
# RSYNC PROFILE SYSTEM Per-profile overrides for appdata syncs
|
||
#
|
||
# ── FAILOVER ───────────────────────────────────────────────────────────────────────────────
|
||
# FAILOVER Mutual container failover between two servers
|
||
# FAILOVER TEST Simulated outage settings for failover_test.sh
|
||
# DDNS Script-controlled DDNS — absolute rules
|
||
# INTERNET LOSS Containers to stop when internet is lost
|
||
# TIERED CONTAINER LISTS What each server runs for the other per tier
|
||
# TIER DELAY SETTINGS How long before each tier activates (minutes)
|
||
# RSYNC WRITEBACK JOBS Appdata synced back to primary on handback
|
||
#
|
||
# ── DOCKER ESSENTIALS ──────────────────────────────────────────────────────────────────────
|
||
# DOWNLOADERS RESET Daily reset for slskd, SABnzbd, qBittorrent
|
||
# DOCKER DAILY RESTART Containers restarted daily
|
||
# DOCKER WEEKLY RESTART Containers restarted weekly
|
||
# DOCKER WATCHDOG Continuous two-tier self-healing container monitoring
|
||
# DOCKER NETWORK CONNECT Ensure networks exist + connect containers on array start
|
||
#
|
||
# ── UNRAID ESSENTIALS ──────────────────────────────────────────────────────────────────────
|
||
# INOTIFY TUNING inotify limits — raised at array start by inotify_tuning.sh
|
||
# SYSTEM TUNING MONITOR Tracks inotify + php-fpm usage over time
|
||
# REBOOT User warning delay before scheduled reboot
|
||
# MOVER Mover stop timeout
|
||
# SYSLOG FILTER Docker veth noise filter file path
|
||
# PHP-FPM PHP-FPM max children config
|
||
# CLEAR LOGS System log file paths
|
||
# WEBGUI WATCHDOG WebGUI nginx + emhttp monitoring and restart
|
||
#
|
||
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||
# MEDIA PERMISSIONS Share list, mode and owner for permissions script
|
||
# MEDIA CLEANER Anime and media folder lists and file patterns
|
||
# ARR CLEANUP Lidarr, Sonarr, Radarr orphan file cleanup
|
||
# ARR FAILED/STALLED RECOVERY Auto blocklist + re-search failed imports and stalled downloads
|
||
#
|
||
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||
# TRANSCODE MANAGER Ramdisk and SSD fallback transcode management
|
||
# TRANSCODE SERVER ARRAY Multi-server session monitoring (Emby, Jellyfin, Plex)
|
||
#
|
||
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
|
||
# CERTIFICATE MONITOR SSL certificate expiry monitoring
|
||
# BACKUP VERIFY Random sample checksum verification against remote
|
||
# SMART HEALTH Drive SMART attribute monitoring
|
||
# ZFS MEMORY SNAPSHOT Weekly ZFS health and memory diagnostic report
|
||
# BANDWIDTH MONITOR Daily rsync transfer logging and weekly summary
|
||
# HEALTH DIGEST Aggregated system health digest — always/smart/weekly
|
||
# EMBY SESSION REPORT Weekly Emby usage statistics via API
|
||
#
|
||
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||
# SYSTEM WATCHDOG Continuous system health monitoring — last line of defense
|
||
#
|
||
# ==============================================================================================
|
||
|
||
# ==============================================================================================
|
||
# ── HOST CONFIGURATION ────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# ━━━ Hosts ━━━
|
||
# Hostnames must match exact Docker/unRAID hostnames AND Tailscale device names — case sensitive.
|
||
# Used by detect_hosts() in common.sh to determine which server is local and which is remote.
|
||
# Both servers run identical scripts — host detection makes them bidirectional.
|
||
HOST1="unRAID-Gmer4Lfe"
|
||
HOST2="unRAID-Jayred365"
|
||
|
||
# Data directory — persistent script state and statistics files.
|
||
# Array share — survives reboots, no flash drive wear.
|
||
# Created automatically if it doesn't exist.
|
||
# Only truly critical files (failover state, watchdog reboot log) stay on /boot/config.
|
||
DATA_DIR="/mnt/user/appdata/unraid_scripts/data"
|
||
|
||
# SSH keys for server-to-server rsync and failover container operations.
|
||
# Both keys must be in /root/.ssh/ and authorised in the remote server's authorized_keys.
|
||
HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key"
|
||
HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key"
|
||
|
||
# ━━━ Emby ━━━
|
||
# Defined once here — referenced by transcode_manager.sh, emby_session_report.sh,
|
||
# emby_database_repair.sh, weekly_sync_maintenance.sh, and TRANSCODE_SERVERS array.
|
||
# API key: Emby Dashboard → API Keys → + New Key
|
||
HOST1_EMBY_CONTAINER="Emby"
|
||
HOST1_EMBY_URL="http://localhost:8096"
|
||
HOST1_EMBY_API_KEY="your-host1-emby-api-key"
|
||
|
||
HOST2_EMBY_CONTAINER="Emby-Jayred365"
|
||
HOST2_EMBY_URL="http://localhost:8096"
|
||
HOST2_EMBY_API_KEY="your-host2-emby-api-key"
|
||
|
||
# ==============================================================================================
|
||
# ── PARTNERSHIP ───────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Manages the relationship lifecycle between two unRAID servers.
|
||
# HOST1 is always the owner (source of truth) — HOST2 is always the mirror.
|
||
# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a --transfer operation.
|
||
# All vars derive from HOST1/HOST2 — no duplicate IP or key configuration needed.
|
||
# Hostnames already match Tailscale device names — IP resolution is automatic.
|
||
#
|
||
# State files on /boot/config — survives reboots, available before array starts:
|
||
# /boot/config/partnership_HOST1.db ← HOST1 writes only
|
||
# /boot/config/partnership_HOST2.db ← HOST2 writes only
|
||
# Propagated via SSH — no rsync needed
|
||
#
|
||
# critical_sync_maintenance.sh runs --check every 15min:
|
||
# Reads both state files via SSH
|
||
# Detects offboard requests → finalises from owner side
|
||
# Increments offline counter → auto-offboards after threshold
|
||
# Silent when healthy ✅
|
||
#
|
||
# See README-Partnership.md for full lifecycle documentation.
|
||
|
||
PARTNERSHIP_ENABLED=false
|
||
PARTNERSHIP_OWNER_HOST="HOST1" # "HOST1" or "HOST2" — flips on --transfer
|
||
|
||
# Auth containers reconfigured on onboard/offboard
|
||
# Format: "ContainerName|WebUIPort"
|
||
# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's)
|
||
# On offboard → WebUI pointed back at localhost
|
||
PARTNERSHIP_AUTH_WEBUIS=(
|
||
"NginxProxyManager|81"
|
||
"Lldap-Gmer4Lfe|17170"
|
||
"Authelia|9091"
|
||
"Authelia-Secondary|9092"
|
||
)
|
||
|
||
# Paths on HOST1 that HOST2 should collect during the grace window after offboard
|
||
# Notified on offboard — no auto-deletion, manual collection
|
||
PARTNERSHIP_MIRROR_BACKUPS=(
|
||
# "/mnt/user/appdata-Failover/Jayred365-Emby"
|
||
)
|
||
|
||
# Timing — single var controls both Tailscale removal and backup access expiry
|
||
# Both expire at the same time — keeping backups accessible beyond Tailscale removal is pointless
|
||
PARTNERSHIP_GRACE_HOURS=6 # hours after offboard before Tailscale removal
|
||
# backup access expires at the same time
|
||
PARTNERSHIP_OFFLINE_THRESHOLD=30 # days either server unreachable before auto-offboard
|
||
# works both directions independently
|
||
|
||
# Tailscale removal on offboard
|
||
PARTNERSHIP_REMOVE_TAILSCALE=true # remove mirror from Tailscale tailnet on offboard
|
||
# false = skip removal (manual or testing)
|
||
|
||
# Tailscale API — required when PARTNERSHIP_REMOVE_TAILSCALE=true
|
||
# API key: https://login.tailscale.com/admin/settings/keys → Devices write scope
|
||
TAILSCALE_API_KEY="" # tskey-api-...
|
||
TAILSCALE_TAILNET="" # your tailnet name (e.g. yourname.github)
|
||
|
||
# Transfer safety
|
||
PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership"
|
||
PARTNERSHIP_TRANSFER_STRIKES=3 # consecutive health checks required
|
||
PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20 # max health check attempts before giving up
|
||
|
||
# Onboard settings
|
||
PARTNERSHIP_ONBOARD_VERIFY=true # verify WebUI reachable after reconfiguration
|
||
PARTNERSHIP_ONBOARD_NOTIFY=true # notify both servers on completion
|
||
PARTNERSHIP_SYNC_INTERVAL=15 # minutes — informational, actual schedule in cron
|
||
|
||
# ==============================================================================================
|
||
# ── LOGGING ───────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# Controls verbose [LOG] output across all scripts.
|
||
# true = show detailed [LOG] lines — useful for debugging or first-time setup
|
||
# false = show only user-facing output — cleaner for scheduled runs
|
||
ENABLE_LOGGING=true
|
||
|
||
# ==============================================================================================
|
||
# ── NOTIFICATIONS ─────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# unRAID native notification system — integrates with the bell icon in the WebGUI.
|
||
# normal = job completed successfully / warning = something failed or needs attention
|
||
NOTIFY_UNRAID=true
|
||
|
||
# Discord webhook URL — leave blank to disable
|
||
DISCORD_WEBHOOK=""
|
||
|
||
# ==============================================================================================
|
||
# ── GIT / REPO ────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# Gitea self-hosted repository — used by git_pull_execute.sh.
|
||
# Detects Gitea container location at runtime — works through failover automatically.
|
||
# Falls back to GITEA_DOMAIN if local and Tailscale both fail.
|
||
GITEA_CONTAINER="Gitea"
|
||
GITEA_REPO_PATH="FailedProxy/Unraid_Scripts.git"
|
||
GITEA_DOMAIN="" # e.g. git.yourdomain.com — requires NPM + DNS
|
||
TARGET_DIR="/mnt/user/appdata/unraid_scripts"
|
||
GITEA_SSH_KEY="/root/.ssh/unraid_gitea"
|
||
SSH_PORT=221 # Gitea SSH port (default 22, Gitea often uses 221/222)
|
||
|
||
# ==============================================================================================
|
||
# ── ORCHESTRATORS ─────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# All orchestrator job lists live here — edit arrays to add/remove scripts.
|
||
# No changes to orchestrator scripts needed when adding or removing jobs.
|
||
|
||
# ━━━ Array Start ━━━
|
||
# Scripts launched by array_start.sh when the array comes online.
|
||
# Launched in order — each as a background process.
|
||
# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally.
|
||
# Continuous scripts (watchdogs, failover) run until array stops.
|
||
ARRAY_START_SCRIPTS=(
|
||
"Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts
|
||
"unRAID_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill
|
||
"unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning
|
||
"unRAID_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted
|
||
"Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers
|
||
"unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop
|
||
"Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop
|
||
# "Failover/failover.sh" # mutual failover — enable when HOST2 ready
|
||
)
|
||
|
||
# ━━━ Daily Sync Maintenance ━━━
|
||
# daily_sync_maintenance.sh runs media share sync first, then iterates
|
||
# DAILY_MAINTENANCE_SCRIPTS for all pre and post sync jobs.
|
||
# Schedule: 0 1 * * * (1am daily)
|
||
DAILY_MAINTENANCE_SCRIPTS=(
|
||
"git_pull_execute.sh" # pull latest scripts — always runs first
|
||
"Media/media_shares_permissions.sh" # apply permissions
|
||
"Media/media_cleaner.sh anime" # remove junk from anime shares
|
||
"Media/media_cleaner.sh media" # remove junk from media shares
|
||
#"Media/lidarr_cleanup.sh" # remove orphaned music files — enable when ready
|
||
#"Media/sonarr_cleanup.sh" # remove orphaned TV files — enable when ready
|
||
#"Media/radarr_cleanup.sh" # remove orphaned movie files — enable when ready
|
||
"Docker_Essentials/docker_daily_restart.sh" # daily container restarts — runs last
|
||
)
|
||
|
||
# Media shares synced daily by daily_sync_maintenance.sh.
|
||
# Each server syncs only the shares it owns — direction is automatic.
|
||
# HOST1 pushes its shares to HOST2. HOST2 pushes its shares to HOST1.
|
||
# Never both pushing the same share — one server is always the truth holder.
|
||
# These shares use DEFAULT_RSYNC_OPTS — no profile entry needed.
|
||
# For shares needing custom options or container stops — create a profile in RSYNC section.
|
||
HOST1_DAILY_SYNC_SHARES=(
|
||
/mnt/user/Books
|
||
/mnt/user/Intros
|
||
/mnt/user/Kids_Movies
|
||
/mnt/user/Kids_Tv_Shows
|
||
/mnt/user/Movies
|
||
/mnt/user/Music
|
||
/mnt/user/Music_Videos
|
||
/mnt/user/Nextcloud
|
||
/mnt/user/stand-up_comedy
|
||
/mnt/user/Sports
|
||
/mnt/user/Tv_Shows
|
||
/mnt/user/Anime_Shows-Old
|
||
/mnt/user/Anime_Movies-Old
|
||
)
|
||
|
||
HOST2_DAILY_SYNC_SHARES=(
|
||
/mnt/user/Anime_Movies
|
||
/mnt/user/Anime_Shows
|
||
)
|
||
|
||
# Personal encrypted shares — synced for offsite backup, independent of media shares.
|
||
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
|
||
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
|
||
HOST1_PERSONAL_SHARES=(
|
||
# /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset
|
||
)
|
||
|
||
HOST2_PERSONAL_SHARES=(
|
||
# /mnt/user/HOST2-Personal # uncomment after creating encrypted dataset
|
||
)
|
||
|
||
# ━━━ Weekly Sync Maintenance ━━━
|
||
# weekly_sync_maintenance.sh stops containers both sides → pulls updates →
|
||
# syncs WEEKLY_SYNC_SHARES → restarts → then iterates WEEKLY_MAINTENANCE_SCRIPTS.
|
||
# Schedule: 30 2 * * 0 (Sunday 2:30am)
|
||
WEEKLY_MAINTENANCE_SCRIPTS=(
|
||
"Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync
|
||
)
|
||
|
||
# Shares synced during the weekly maintenance window.
|
||
# Containers stopped both sides before sync — full clean state guaranteed.
|
||
# Profiles drive container stops, excludes, and options — configure in RSYNC section.
|
||
# Order matters — Emby first (larger), then Critical-Data (auth stack).
|
||
WEEKLY_SYNC_SHARES=(
|
||
"/mnt/user/Media_Server/Emby" # emby profile — full clean mirror
|
||
"/mnt/user/appdata-Failover/Critical-Data" # critical-data profile — auth stack
|
||
)
|
||
|
||
# Container update toggles for the weekly sync window.
|
||
# Containers already stopped for sync — updates pull at no extra downtime.
|
||
# Both false → sync only, no updates.
|
||
WEEKLY_SYNC_UPDATES=true # pull container updates locally during weekly window
|
||
WEEKLY_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH
|
||
|
||
# ━━━ Critical Sync Maintenance ━━━
|
||
# critical_sync_maintenance.sh runs every 15 minutes.
|
||
# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check
|
||
# partnership --check always runs last regardless of rsync gate.
|
||
# Format for shares: "/path/to/share" or "/path/to/share|profile-name"
|
||
|
||
# Jobs run every 15 minutes before the rsync shares.
|
||
# Comment out to disable without removing.
|
||
CRITICAL_MAINTENANCE_SCRIPTS=(
|
||
"Docker_Essentials/downloaders_reset.sh" # clear stuck download states every 15min
|
||
)
|
||
|
||
# Shares synced during the critical sync window.
|
||
CRITICAL_SYNC_SHARES=(
|
||
"/mnt/user/appdata-Failover/Critical-Data" # auth stack — full sync
|
||
"/mnt/user/Media_Server/Emby|emby-failover" # Emby dirty sync — stays running
|
||
)
|
||
|
||
# ==============================================================================================
|
||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# ━━━ Rsync Enable/Disable ━━━
|
||
# Two-tier toggle system — Tier 1 overrides Tier 2.
|
||
#
|
||
# Tier 1 — Global gate:
|
||
# RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions
|
||
# Use when: remote completely offline, major maintenance, disaster recovery
|
||
#
|
||
# Tier 2 — Per-orchestrator (only applies when Tier 1 is true):
|
||
# Fine grained control — disable specific orchestrators while keeping others
|
||
# Use when: rebuilding secondary, testing, per-window bandwidth management
|
||
#
|
||
# Example — HOST2 data rebuild:
|
||
# RSYNC_ENABLED=true ← rsync works, individual scripts run fine
|
||
# DAILY_RSYNC_ENABLED=false ← skip daily HDD syncs during rebuild
|
||
# WEEKLY_RSYNC_ENABLED=true ← Emby + Critical-Data still sync (NVMe)
|
||
# CRITICAL_RSYNC_ENABLED=true ← 15min auth stack sync still runs
|
||
# FAILOVER_RSYNC_ENABLED=true ← handback writeback still works
|
||
# → Run individual: bash Rsync/rsync.sh /mnt/user/Movies
|
||
# → When ready: DAILY_RSYNC_ENABLED=true
|
||
RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below
|
||
DAILY_RSYNC_ENABLED=false # Tier 2 — HOST2 rebuild in progress, re-enable when ready
|
||
WEEKLY_RSYNC_ENABLED=true # Tier 2 — weekly_sync_maintenance.sh rsync section
|
||
CRITICAL_RSYNC_ENABLED=true # Tier 2 — critical_sync_maintenance.sh (every 15min)
|
||
FAILOVER_RSYNC_ENABLED=true # Tier 2 — failover.sh writeback jobs on handback
|
||
|
||
# ━━━ Rsync Defaults ━━━
|
||
# Global fallback values used when no profile match is found.
|
||
# Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed.
|
||
# Appdata shares match profiles by directory basename (lowercased).
|
||
BW_LIMIT=12500 # KB/s — 12500 ≈ 100Mbit
|
||
RETRY_COUNT=3 # retry attempts before giving up
|
||
SLEEP=300 # seconds between retry attempts
|
||
CRITICAL_CONTAINER_NAMES=() # containers stopped on REMOTE before rsync — profiles override
|
||
DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override
|
||
CONTAINER_DELAY=5 # seconds before starting delayed containers
|
||
EXCLUDE_DIRS=() # directories excluded from transfer — profiles override
|
||
|
||
# --delete removes files on remote not on source (mirror behaviour)
|
||
# --inplace writes directly to destination — better for large files
|
||
# --no-whole-file forces delta transfer — sends only changed blocks
|
||
DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file)
|
||
|
||
# ━━━ Remote Health Checks ━━━
|
||
# Pre-flight — aborts if remote rootfs (/) usage is at or above this percentage.
|
||
# When remote array is down, rsync writes land on rootfs and fill it rapidly.
|
||
ROOTFS_WARN=75
|
||
|
||
# ━━━ Rsync Profile System ━━━
|
||
# Profiles allow per-share rsync behaviour without touching script logic.
|
||
# Profile key matched by basename of directory passed to rsync.sh (lowercased).
|
||
# Override with --profile=name flag.
|
||
#
|
||
# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit DEFAULT_RSYNC_OPTS.
|
||
# List ALL desired options explicitly when defining a profile.
|
||
#
|
||
# Current profiles:
|
||
# arrs_stack — arr databases — lower bandwidth, containers stopped for consistency
|
||
# critical-data — auth stack — containers stopped both sides, Authelia delayed start
|
||
# gmer4lfe — server-specific appdata — no container stops needed
|
||
# important-data — NextCloud + Postgres — NextCloud delayed start after Postgres
|
||
# emby — weekly clean sync — both Emby stopped, full mirror
|
||
# called by weekly_sync_maintenance.sh only — do NOT schedule separately
|
||
# emby-failover — frequent dirty sync — Emby stays running, WAL excluded
|
||
# also used for failover writeback on handback
|
||
|
||
declare -A PROFILE_RSYNC_OPTS=(
|
||
[arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace"
|
||
[critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete"
|
||
[gmer4lfe]="-av --info=progress2 --bwlimit=$BW_LIMIT"
|
||
[important-data]="-av --human-readable --bwlimit=$BW_LIMIT"
|
||
[emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
|
||
[emby-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
|
||
)
|
||
|
||
# Per-profile bandwidth limits in KB/s
|
||
declare -A PROFILE_BW_LIMIT=(
|
||
[arrs_stack]=5000 # lower — runs alongside other syncs
|
||
[critical-data]=9500 # high — small dataset, sync fast
|
||
[gmer4lfe]=8000
|
||
[important-data]=9500 # high — database sync
|
||
[emby]=8000 # medium — large full mirror
|
||
[emby-failover]=9500 # high — small critical dataset
|
||
)
|
||
|
||
# Retry attempts per profile
|
||
declare -A PROFILE_RETRY_COUNT=(
|
||
[arrs_stack]=3
|
||
[critical-data]=3
|
||
[gmer4lfe]=3
|
||
[important-data]=3
|
||
[emby]=3
|
||
[emby-failover]=3
|
||
)
|
||
|
||
# Seconds between retry attempts
|
||
declare -A PROFILE_SLEEP=(
|
||
[arrs_stack]=300
|
||
[critical-data]=300
|
||
[gmer4lfe]=300
|
||
[important-data]=300
|
||
[emby]=300
|
||
[emby-failover]=120 # shorter — frequent dirty sync, retry faster
|
||
)
|
||
|
||
# Containers stopped on BOTH LOCAL and REMOTE before rsync.
|
||
# Local stops first — flushes databases cleanly. Remote stops next — prevents writes.
|
||
# Only running containers get restarted — stopped containers stay stopped.
|
||
# SPACE-SEPARATED STRINGS — converted to array at runtime
|
||
declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
|
||
[arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
|
||
[critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary"
|
||
[gmer4lfe]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
|
||
[important-data]="Postgres-NextCloud NextCloud"
|
||
[emby]="Emby"
|
||
[emby-failover]="" # dirty sync — Emby stays running both sides
|
||
)
|
||
|
||
# Containers needing a delay after rsync before starting.
|
||
# SPACE-SEPARATED STRINGS — converted to array at runtime
|
||
declare -A PROFILE_DELAYED_CONTAINERS=(
|
||
[arrs_stack]=""
|
||
[critical-data]="Authelia Authelia-Secondary" # wait for Mariadb + Redis
|
||
[gmer4lfe]=""
|
||
[important-data]="NextCloud" # wait for Postgres
|
||
[emby]=""
|
||
[emby-failover]=""
|
||
)
|
||
|
||
# Seconds before starting delayed containers
|
||
declare -A PROFILE_CONTAINER_DELAY=(
|
||
[arrs_stack]=5
|
||
[critical-data]=15 # Mariadb + Redis need time to accept connections
|
||
[gmer4lfe]=5
|
||
[important-data]=10 # Postgres needs time before NextCloud
|
||
[emby]=5
|
||
[emby-failover]=5
|
||
)
|
||
|
||
# Directories excluded from rsync per profile.
|
||
# SPACE-SEPARATED STRINGS — converted to array at runtime
|
||
declare -A PROFILE_EXCLUDE_DIRS=(
|
||
[arrs_stack]="logs *.tmp"
|
||
[gmer4lfe]="logs *.tmp"
|
||
[important-data]="logs *.tmp"
|
||
[critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt"
|
||
[emby]="logs transcodes cache crash*"
|
||
[emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root"
|
||
)
|
||
|
||
# Note: disk check is auto-detected from disks.ini — no PROFILE_SKIP_DISK_CHECK needed.
|
||
# check_remote_disks() reads fsType per disk and handles XFS and ZFS automatically.
|
||
|
||
# ==============================================================================================
|
||
# ── FAILOVER ──────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Mutual container failover between two unRAID servers.
|
||
# Each server runs Failover/failover.sh independently via array_start.sh.
|
||
# All decisions based on two pings: remote reachable + internet reachable.
|
||
#
|
||
# States: NORMAL | FAILOVER | NO_INTERNET | DARK
|
||
#
|
||
# DDNS rules — absolute:
|
||
# Internet loss → stop own DDNS immediately
|
||
# Failover → start remote DDNS first (Tier 1)
|
||
# Handback → stop remote DDNS → rsync → start containers → start local DDNS last
|
||
#
|
||
# Tiers:
|
||
# Tier 1 — Immediate — vital services + Live TV
|
||
# Tier 2 — configurable delay — productivity services
|
||
# Tier 3 — configurable delay — secondary services
|
||
# Tier 4 — configurable delay — arrs + downloaders
|
||
|
||
EXTERNAL_IP="8.8.8.8"
|
||
FAILOVER_CHECK_INTERVAL=120
|
||
FAILOVER_HANDBACK_STRIKES=2
|
||
FAILOVER_STATE_FILE="/boot/config/failover_state.db"
|
||
FAILOVER_ENABLED=false # HOST2 being rebuilt — set true when back online and tested
|
||
# false = suppresses "not running" warnings in status scripts
|
||
|
||
# ━━━ Failover Test ━━━
|
||
FAILOVER_TEST_BLOCK_WAIT=150
|
||
FAILOVER_TEST_HANDBACK_WAIT=360
|
||
|
||
# ━━━ DDNS ━━━
|
||
HOST1_DDNS_CONTAINERS=(
|
||
"Gmer4Lfe.com"
|
||
)
|
||
|
||
HOST2_DDNS_CONTAINERS=(
|
||
"Gmer4Lfe.us"
|
||
)
|
||
|
||
# ━━━ Internet Loss ━━━
|
||
FAILOVER_HOST1_STOP_ON_NO_NET=(
|
||
"Gmer4Lfe.com"
|
||
)
|
||
|
||
FAILOVER_HOST2_STOP_ON_NO_NET=(
|
||
"Gmer4Lfe.us"
|
||
)
|
||
|
||
# ━━━ Tiered Container Lists ━━━
|
||
|
||
# HOST1 runs for HOST2 (HOST2 is down)
|
||
FAILOVER_HOST1_RUNS_FOR_HOST2_IMMEDIATE=(
|
||
"Gmer4Lfe.us"
|
||
"VaultWarden-Jayred365"
|
||
)
|
||
|
||
FAILOVER_HOST1_RUNS_FOR_HOST2_2HR=(
|
||
# "container-placeholder"
|
||
)
|
||
|
||
FAILOVER_HOST1_RUNS_FOR_HOST2_6HR=(
|
||
# "container-placeholder"
|
||
)
|
||
|
||
FAILOVER_HOST1_RUNS_FOR_HOST2_18HR=(
|
||
# "container-placeholder"
|
||
)
|
||
|
||
# HOST2 runs for HOST1 (HOST1 is down)
|
||
FAILOVER_HOST2_RUNS_FOR_HOST1_IMMEDIATE=(
|
||
"Gmer4Lfe.com"
|
||
"Emby"
|
||
"VaultWarden-Gmer4Lfe"
|
||
"Dispatcharr"
|
||
"Dispatcharr-Basic"
|
||
"Dispatcharr-Iptv-Users"
|
||
"ErsatzTV-Emby"
|
||
)
|
||
|
||
FAILOVER_HOST2_RUNS_FOR_HOST1_2HR=(
|
||
"Postgres-NextCloud"
|
||
"NextCloud"
|
||
"PostgreSQL_Immich"
|
||
"Immich-Gmer4Lfe"
|
||
)
|
||
|
||
FAILOVER_HOST2_RUNS_FOR_HOST1_6HR=(
|
||
"Gitea"
|
||
)
|
||
|
||
FAILOVER_HOST2_RUNS_FOR_HOST1_18HR=(
|
||
"Sonarr"
|
||
"Radarr"
|
||
"Lidarr"
|
||
"Readarr"
|
||
"Prowlarr"
|
||
"Bazarr"
|
||
"SABnzbd-Gmer4Lfe"
|
||
"Qbittorrent-Gmer4Lfe"
|
||
"LidaTube"
|
||
"Pinchflat"
|
||
"ChannelTube"
|
||
)
|
||
|
||
# ━━━ Tier Delay Settings ━━━
|
||
# How long the primary must be down before each tier activates — in minutes.
|
||
# Tier 1 is always immediate. Set independently per host.
|
||
|
||
# HOST1's containers running on HOST2 (HOST1 is down)
|
||
HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich
|
||
HOST1_TIER3_DELAY=720 # 12 hours — secondary services
|
||
HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders
|
||
|
||
# HOST2's containers running on HOST1 (HOST2 is down)
|
||
HOST2_TIER2_DELAY=240
|
||
HOST2_TIER3_DELAY=720
|
||
HOST2_TIER4_DELAY=1440
|
||
|
||
# ━━━ Rsync Writeback Jobs ━━━
|
||
# Syncs critical appdata BACK to primary during handback after failover.
|
||
# Containers stopped before writeback runs — clean source, no competing writes.
|
||
#
|
||
# HOST*_TIER1_WRITEBACK_DELAY:
|
||
# Short outages skip Tier 1 writeback — primary state is more reliable than dirty sync data
|
||
# Only writeback if outage lasted longer than this many minutes
|
||
#
|
||
# Tier 4 writeback automatically syncs HOST*_DAILY_SYNC_SHARES — no need to list those here.
|
||
# Only add paths NOT in DAILY_SYNC_SHARES that need writeback after extended outage.
|
||
HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr
|
||
HOST2_TIER1_WRITEBACK_DELAY=60
|
||
|
||
# HOST1 writeback — run by HOST2 during HOST1 handback
|
||
FAILOVER_HOST1_WRITEBACK_TIER1=(
|
||
"/mnt/user/Media_Server/Emby" # watch states built up during outage
|
||
)
|
||
|
||
FAILOVER_HOST1_WRITEBACK_TIER2=(
|
||
"/mnt/user/appdata-Failover/Important-Data" # NextCloud + Postgres
|
||
)
|
||
|
||
FAILOVER_HOST1_WRITEBACK_TIER3=(
|
||
# "location-placeholder"
|
||
)
|
||
|
||
FAILOVER_HOST1_WRITEBACK_TIER4=(
|
||
"/mnt/user/appdata-Failover/Arrs_Stack" # arr databases — downloads queued during outage
|
||
)
|
||
|
||
# HOST2 writeback — run by HOST1 during HOST2 handback
|
||
FAILOVER_HOST2_WRITEBACK_TIER1=(
|
||
# "/mnt/user/appdata-Failover/Jayred365-Emby"
|
||
)
|
||
|
||
FAILOVER_HOST2_WRITEBACK_TIER2=(
|
||
# "/mnt/user/appdata-Failover/Jayred365-Important"
|
||
)
|
||
|
||
FAILOVER_HOST2_WRITEBACK_TIER3=(
|
||
# "location-placeholder"
|
||
)
|
||
|
||
FAILOVER_HOST2_WRITEBACK_TIER4=(
|
||
"/mnt/user/appdata-Failover/Arrs_Stack"
|
||
)
|
||
|
||
# ==============================================================================================
|
||
# ── DOCKER ESSENTIALS ─────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# ━━━ Downloaders Reset ━━━
|
||
# Runs every 15 minutes via CRITICAL_MAINTENANCE_SCRIPTS.
|
||
# Clears stuck states, purges old history, prepares each client for a clean cycle.
|
||
DOWNLOADER_RETENTION_DAYS=7
|
||
|
||
# ── slskd ──
|
||
# Clears stuck searches, dead transfers, purges expired failed imports.
|
||
# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected.
|
||
HOST1_SLSKD_URL="http://localhost:8980"
|
||
HOST1_SLSKD_API_KEY="4bF9kL2mNpQrT7vWxYz1A3dEgHjKoRsU"
|
||
HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports"
|
||
|
||
# ── SABnzbd ──
|
||
HOST1_SABNZBD_URL="http://localhost:8180"
|
||
HOST1_SABNZBD_API_KEY="8bfefe41d83b4d50883e32859b55ca9a"
|
||
|
||
# ── qBittorrent ──
|
||
# deleteFiles=false — removes torrent from qBit but leaves files on disk.
|
||
# Radarr/Sonarr manage actual files independently.
|
||
HOST1_QBIT_URL="http://localhost:8080"
|
||
HOST1_QBIT_USERNAME="root"
|
||
HOST1_QBIT_PASSWORD="Stay0utD!ck"
|
||
QBIT_FAILSAFE_MIN_DAYS=180
|
||
QBIT_FAILSAFE_MIN_RATIO=0 # 0 = age only, no ratio requirement
|
||
|
||
# ━━━ Docker Daily Restart ━━━
|
||
# Containers restarted every day via daily_sync_maintenance.sh.
|
||
# Dispatcharr degrades over time without restart — daily is intentional.
|
||
DAILY_RESTART_CONTAINERS=(
|
||
"NginxProxyManager"
|
||
"Lldap-Gmer4Lfe"
|
||
"Authelia"
|
||
"Authelia-Secondary"
|
||
"Dispatcharr-Iptv-Users"
|
||
"Dispatcharr" # Live TV scheduler — degrades without daily restart
|
||
"Dispatcharr-Basic"
|
||
"ErsatzTV-Emby"
|
||
)
|
||
|
||
# ━━━ Docker Weekly Restart ━━━
|
||
# Less critical services restarted weekly via weekly_sync_maintenance.sh (Sunday 2:30am).
|
||
# Containers already stopped for weekly sync — restart adds zero extra downtime.
|
||
WEEKLY_RESTART_CONTAINERS=(
|
||
"NextCloud"
|
||
"Organizrv2-Gmer4Lfe"
|
||
"AdGuard-Home"
|
||
"Immich-Gmer4Lfe"
|
||
)
|
||
|
||
# ━━━ Docker Watchdog ━━━
|
||
# Continuous two-tier self-healing container monitoring.
|
||
# Started by array_start.sh — runs until array stops.
|
||
# Re-sources Master.conf each cycle — add/remove containers without restarting watchdog.
|
||
#
|
||
# Tier 1 — strict monitoring of explicitly configured containers:
|
||
# Memory hard limits — immediate restart if exceeded
|
||
# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT sustained strikes
|
||
# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT failed checks
|
||
# Required containers — must always be running, strike + skip list with auto-clear
|
||
#
|
||
# Tier 2 — global health scan of ALL running containers:
|
||
# Unhealthy status — Docker HEALTHCHECK unhealthy → restart
|
||
# OOM killed — kernel killed → restart + notify
|
||
# Crash loop detection — RestartCount climbing → notify, critical above limit
|
||
# Dead containers — remove and restart
|
||
# Unexpected exits — non-zero exit code → restart
|
||
|
||
# Memory hard limits in MB — immediate restart if exceeded
|
||
# 20GB=20480 16GB=16384 12GB=12288 10GB=10240 8GB=8192 4GB=4096 2GB=2048 1GB=1024
|
||
declare -A WATCHDOG_CONTAINERS=(
|
||
["Emby"]=16384
|
||
["LidaTube"]=6144
|
||
["Tdarr"]=6144
|
||
["Code-Server"]=1024
|
||
)
|
||
|
||
# HTTP health check URLs — checked every cycle, strike system before restart
|
||
declare -A HOST1_WATCHDOG_CONTAINER_URLS=(
|
||
["Emby"]="http://localhost:8096"
|
||
)
|
||
|
||
declare -A HOST2_WATCHDOG_CONTAINER_URLS=(
|
||
["Emby"]="http://localhost:8096"
|
||
)
|
||
|
||
# Required containers — must always be running
|
||
# Strike system before restart, skip list after repeated failures, auto-clears on recovery
|
||
HOST1_WATCHDOG_REQUIRED_CONTAINERS=(
|
||
"NginxProxyManager"
|
||
"Lldap-Gmer4Lfe"
|
||
"Authelia"
|
||
"Authelia-Secondary"
|
||
"Mariadb-Authelia"
|
||
"Mariadb-Authelia-Secondary"
|
||
"Redis-Authelia"
|
||
"Redis-Authelia-Secondary"
|
||
)
|
||
|
||
HOST2_WATCHDOG_REQUIRED_CONTAINERS=(
|
||
"NginxProxyManager"
|
||
# add HOST2 required containers here
|
||
)
|
||
|
||
# Strike state file — /tmp resets on reboot (correct — no stale strikes after reboot)
|
||
WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db"
|
||
|
||
# CPU thresholds — normalised against total core count at runtime
|
||
SOFT_CPU_THRESHOLD=80
|
||
HARD_CPU_THRESHOLD=85
|
||
CPU_FAIL_LIMIT=2 # consecutive hard CPU strikes before container restart
|
||
|
||
# Memory soft threshold — warn when container reaches this % of its hard limit
|
||
SOFT_MEM_THRESHOLD=80
|
||
|
||
# HTTP responsiveness
|
||
RESP_FAIL_LIMIT=2 # consecutive failed checks before restart
|
||
CURL_TIMEOUT=5 # seconds per check
|
||
|
||
# Watchdog cycle interval
|
||
DOCKER_WATCHDOG_INTERVAL=900 # seconds between cycles (15 minutes)
|
||
|
||
# Heartbeat
|
||
DOCKER_WATCHDOG_HEARTBEAT=true
|
||
DOCKER_WATCHDOG_HEARTBEAT_HOURS=1
|
||
|
||
# Tier 2 master toggle
|
||
WATCHDOG_SCAN_ALL=true # false = only WATCHDOG_CONTAINERS + required containers
|
||
|
||
# Containers to skip in Tier 2 scan — legitimately stopped or frequently restarting
|
||
WATCHDOG_SCAN_IGNORE=(
|
||
"DashGate"
|
||
"PIA-WG-Config-Generator"
|
||
"Aperture"
|
||
"Aperture-Kids"
|
||
"pgvector-18-Apeture-Kids"
|
||
"Pgvector18-Aperture"
|
||
)
|
||
|
||
# Individual Tier 2 check toggles
|
||
WATCHDOG_RESTART_UNHEALTHY=true
|
||
WATCHDOG_RESTART_DEAD=true
|
||
WATCHDOG_RESTART_CRASHED=true
|
||
WATCHDOG_NOTIFY_OOM=true
|
||
WATCHDOG_NOTIFY_CRASHLOOP=true
|
||
|
||
# Crash loop threshold
|
||
WATCHDOG_CRASH_LIMIT=5
|
||
|
||
# Startup grace period — skip restarts while system is still booting
|
||
WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts
|
||
|
||
# Restart loop protection
|
||
WATCHDOG_CONTAINER_RESTART_LIMIT=3
|
||
WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours
|
||
WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db"
|
||
|
||
# Notification batching — one summary per cycle instead of one ping per event
|
||
WATCHDOG_BATCH_NOTIFY=true
|
||
|
||
# Dependency ordering — skip restarting a container if its dependency is also down
|
||
declare -A WATCHDOG_DEPENDENCIES=(
|
||
["Authelia"]="Mariadb-Authelia Redis-Authelia"
|
||
["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
|
||
["NextCloud"]="Postgres-NextCloud"
|
||
)
|
||
|
||
# ━━━ Docker Network Connect ━━━
|
||
# Ensures custom networks exist and connects containers at array start.
|
||
NETWORK_CONNECT_CONTAINERS=(
|
||
"memcached"
|
||
"Npm-CrowdSec"
|
||
)
|
||
|
||
NETWORK_CONNECT_NETWORKS=(
|
||
"high-availability"
|
||
)
|
||
|
||
# ==============================================================================================
|
||
# ── UNRAID ESSENTIALS ─────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# ━━━ inotify Tuning ━━━
|
||
# Linux inotify limits — applied at every array start by inotify_tuning.sh.
|
||
# Default unRAID values are very low — with many Docker containers watching files
|
||
# you can silently exhaust the limit causing containers to miss file events.
|
||
# These settings are lost on reboot — reapplied automatically at array start.
|
||
INOTIFY_MAX_INSTANCES=1024 # default: 128 — max inotify instances per user
|
||
INOTIFY_MAX_WATCHES=524288 # default: 8192 — max files watched per instance
|
||
INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — max events queued before dropping
|
||
|
||
# ━━━ System Tuning Monitor ━━━
|
||
# Tracks inotify and php-fpm usage over time.
|
||
# Snapshot written every 6 hours by system_tuning_monitor.sh.
|
||
# Read by sunday_morning_coffee_report.sh for weekly peak/avg/warning summary.
|
||
INOTIFY_WARN_PCT=80 # warn if inotify instances exceed this % of limit
|
||
PHP_FPM_WARN_PCT=80 # warn if php-fpm workers exceed this % of max_children
|
||
TUNING_MONITOR_LOG="$DATA_DIR/system_tuning_history.db"
|
||
TUNING_LOG_RETENTION=30 # days before old entries are purged
|
||
|
||
# ━━━ Reboot ━━━
|
||
# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots.
|
||
# Gives users time to save work — 300s = 5 minutes
|
||
REBOOT_SLEEP=300
|
||
|
||
# ━━━ Mover ━━━
|
||
# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process.
|
||
# Gives mover time to finish current file transfer before being interrupted.
|
||
MOVER_STOP_TIMEOUT=300
|
||
|
||
# ━━━ Syslog Filter ━━━
|
||
# Path for the rsyslog filter file that suppresses Docker veth interface noise.
|
||
# Docker creates a new veth interface for each container — generates hundreds of
|
||
# log lines per hour that have no diagnostic value. Filter removes them at source.
|
||
FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"
|
||
|
||
# ━━━ PHP-FPM ━━━
|
||
# Higher max_children allows more concurrent PHP requests to the unRAID WebGUI.
|
||
# Default is very low — increasing it prevents WebGUI slowdowns under load.
|
||
# 250 is safe for servers with 32GB+ RAM.
|
||
PHP_CONF="/etc/php-fpm.d/www.conf"
|
||
PHP_MAX_CHILDREN=250
|
||
|
||
# ━━━ Clear Logs ━━━
|
||
# System log files cleared weekly to prevent rootfs fill over time.
|
||
# These grow continuously — without clearing they eventually consume all rootfs space.
|
||
LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg)
|
||
|
||
# ━━━ WebGUI Watchdog ━━━
|
||
# Monitors unRAID WebGUI responsiveness — escalates through nginx restart → emhttp restart.
|
||
# Separate from docker_watchdog — this monitors the unRAID UI itself, not containers.
|
||
# WEBGUI_NGINX_WAIT = seconds after nginx restart before rechecking
|
||
# WEBGUI_EMHTTP_WAIT = seconds after emhttp restart before rechecking
|
||
WEBGUI_URL="http://localhost"
|
||
WEBGUI_TIMEOUT=5 # seconds before curl gives up on WebGUI check
|
||
WEBGUI_NGINX_WAIT=15 # seconds after nginx restart before rechecking
|
||
WEBGUI_EMHTTP_WAIT=30 # seconds after emhttp restart before rechecking
|
||
|
||
# ==============================================================================================
|
||
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# ━━━ Media Permissions ━━━
|
||
# Applied recursively by media_shares_permissions.sh.
|
||
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
|
||
PERMISSIONS_MODE="777"
|
||
PERMISSIONS_OWNER="nobody:users"
|
||
|
||
MEDIA_PERMISSION_SHARES=(
|
||
/mnt/user/Anime_Movies
|
||
/mnt/user/Anime_Movies-Old
|
||
/mnt/user/Anime_Shows
|
||
/mnt/user/Anime_Shows-Old
|
||
/mnt/user/appcache
|
||
/mnt/user/Books
|
||
/mnt/user/Downloads
|
||
/mnt/user/Games
|
||
/mnt/user/Intros
|
||
/mnt/user/Kids_Movies
|
||
/mnt/user/Kids_Tv_Shows
|
||
/mnt/user/Movie_Recordings
|
||
/mnt/user/Movies
|
||
/mnt/user/Music
|
||
/mnt/user/Music_Videos
|
||
/mnt/user/Photo
|
||
/mnt/user/Sports
|
||
/mnt/user/stand-up_comedy
|
||
/mnt/user/Temp_Storage
|
||
/mnt/user/Tv_Recordings
|
||
/mnt/user/Tv_Shows
|
||
/mnt/user/YouTube
|
||
)
|
||
|
||
# ━━━ Media Cleaner ━━━
|
||
# Removes junk files from media shares — two profiles: anime and media.
|
||
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
|
||
ANIME_CLEAN_FOLDERS=(
|
||
/mnt/user/Anime_Movies
|
||
/mnt/user/Anime_Movies-Old
|
||
/mnt/user/Anime_Shows
|
||
/mnt/user/Anime_Shows-Old
|
||
)
|
||
|
||
MEDIA_CLEAN_FOLDERS=(
|
||
/mnt/user/Kids_Movies
|
||
/mnt/user/Kids_Tv_Shows
|
||
/mnt/user/Movies
|
||
/mnt/user/Music
|
||
/mnt/user/Sports
|
||
/mnt/user/stand-up_comedy
|
||
/mnt/user/Tv_Shows
|
||
)
|
||
|
||
ANIME_FILE_PATTERNS=(
|
||
'*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk'
|
||
'*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*'
|
||
'*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp'
|
||
'*.log' '*.json'
|
||
)
|
||
|
||
MEDIA_FILE_PATTERNS=(
|
||
'*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk'
|
||
'*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*'
|
||
'*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp'
|
||
'*.log' '*.json' '*.iso' '*.lrc'
|
||
)
|
||
|
||
# ━━━ Arr Cleanup ━━━
|
||
# Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs.
|
||
# detect_hosts() selects correct URL, API key, and root path at runtime.
|
||
#
|
||
# API versions:
|
||
# Sonarr v4 → /api/v3/series → /api/v3/episodefile?seriesId=X
|
||
# Radarr v6 → /api/v3/movie → /api/v3/moviefile?movieId=X
|
||
# Lidarr v3 → /api/v1/artist → /api/v1/trackFile?artistId=X
|
||
#
|
||
# Version checking — scripts verify major version matches before running.
|
||
# Update the MAJOR version here when script is updated to support a new arr version.
|
||
SONARR_VERSION_MAJOR=4
|
||
RADARR_VERSION_MAJOR=6
|
||
LIDARR_VERSION_MAJOR=3
|
||
|
||
# ── Lidarr ────────────────────────────────────────────────────────────────────────────────────
|
||
HOST1_LIDARR_URL="http://localhost:8686"
|
||
HOST1_LIDARR_API_KEY="your-lidarr-api-key"
|
||
HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New"
|
||
LIDARR_LOCK_WARN_AGE=3600 # 1hr — large libraries take time, not stuck
|
||
|
||
# Container path → host path translation
|
||
# Lidarr stores file paths using container paths — script scans host paths
|
||
# Add one entry per root folder configured in Lidarr Settings → Media Management → Root Folders
|
||
declare -A HOST1_LIDARR_PATH_MAP=(
|
||
["/ext-music"]="/mnt/user/Music-New"
|
||
)
|
||
declare -A HOST2_LIDARR_PATH_MAP=(
|
||
# HOST2 does not run Lidarr — fill in if that changes
|
||
)
|
||
|
||
LIDARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion
|
||
LIDARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this
|
||
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||
# protects against API returning partial data on a bad day
|
||
LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
|
||
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
|
||
LIDARR_PROTECTED_PATTERNS=(
|
||
# Metadata
|
||
"*.nfo" "*.tbn"
|
||
# Images — album art, artist images, Emby artwork
|
||
"*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
|
||
"poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
|
||
"banner.*" "thumb.*" "landscape.*"
|
||
"folder.*" "cover.*" "album.*" "artist.*" "disc.*"
|
||
# Lyrics
|
||
"*.lrc"
|
||
)
|
||
|
||
# ── Sonarr ────────────────────────────────────────────────────────────────────────────────────
|
||
HOST1_SONARR_URL="http://localhost:8989"
|
||
HOST1_SONARR_API_KEY="your-sonarr-api-key"
|
||
HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows"
|
||
|
||
# Container path → host path translation
|
||
# Add one entry per root folder configured in Sonarr Settings → Media Management → Root Folders
|
||
# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder
|
||
declare -A HOST1_SONARR_PATH_MAP=(
|
||
["/tv"]="/mnt/user/Tv_Shows"
|
||
["/ext-standup-comedy"]="/mnt/user/stand-up_comedy"
|
||
["/kids tv"]="/mnt/user/Kids_Tv_Shows"
|
||
["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old"
|
||
)
|
||
|
||
HOST2_SONARR_URL="http://localhost:8989"
|
||
HOST2_SONARR_API_KEY="your-host2-sonarr-api-key"
|
||
HOST2_SONARR_TV_ROOT="/mnt/user/Anime_Shows"
|
||
|
||
declare -A HOST2_SONARR_PATH_MAP=(
|
||
# fill in when HOST2 is back online
|
||
# ["/tv"]="/mnt/user/Anime_Shows"
|
||
)
|
||
|
||
SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion
|
||
SONARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this
|
||
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
|
||
SONARR_PROTECTED_PATTERNS=(
|
||
# Subtitles
|
||
"*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt"
|
||
# Metadata
|
||
"*.nfo" "*.tbn"
|
||
# Images
|
||
"*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
|
||
"poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
|
||
"banner.*" "thumb.*" "landscape.*"
|
||
# Kodi/Emby extras — not tracked by Sonarr API
|
||
"*-trailer.*" "*-featurette.*" "*-behindthescenes.*"
|
||
"*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*"
|
||
"*-clip.*" "*-other.*"
|
||
# Theme songs — stored in show folder, not tracked
|
||
"theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka"
|
||
)
|
||
|
||
# ── Radarr ────────────────────────────────────────────────────────────────────────────────────
|
||
HOST1_RADARR_URL="http://localhost:7878"
|
||
HOST1_RADARR_API_KEY="your-radarr-api-key"
|
||
HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies"
|
||
|
||
# Container path → host path translation
|
||
# Add one entry per root folder configured in Radarr Settings → Media Management → Root Folders
|
||
# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder
|
||
declare -A HOST1_RADARR_PATH_MAP=(
|
||
["/movies"]="/mnt/user/Movies"
|
||
["/kids movies"]="/mnt/user/Kids_Movies"
|
||
["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy"
|
||
["/anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||
)
|
||
|
||
HOST2_RADARR_URL="http://localhost:7878"
|
||
HOST2_RADARR_API_KEY="your-host2-radarr-api-key"
|
||
HOST2_RADARR_MOVIES_ROOT="/mnt/user/Anime_Movies"
|
||
|
||
declare -A HOST2_RADARR_PATH_MAP=(
|
||
# fill in when HOST2 is back online
|
||
# ["/anime-movies"]="/mnt/user/Anime_Movies"
|
||
)
|
||
|
||
RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion
|
||
RADARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this
|
||
RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov")
|
||
RADARR_PROTECTED_PATTERNS=(
|
||
# Subtitles
|
||
"*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt"
|
||
# Metadata
|
||
"*.nfo" "*.tbn"
|
||
# Images
|
||
"*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
|
||
"poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
|
||
"banner.*" "thumb.*" "landscape.*"
|
||
# Kodi/Emby extras — not tracked by Radarr API
|
||
"*-trailer.*" "*-featurette.*" "*-behindthescenes.*"
|
||
"*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*"
|
||
"*-clip.*" "*-other.*"
|
||
# Theme songs — stored in movie folder, not tracked
|
||
"theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka"
|
||
)
|
||
|
||
# ━━━ Arr Failed/Stalled Recovery ━━━
|
||
# Auto blocklist + re-search failed imports and stalled downloads.
|
||
# Runs every 6 hours — schedule: 0 */6 * * *
|
||
#
|
||
# Targets four problem types:
|
||
# importFailed — downloaded but arr couldn't import
|
||
# importPending — downloaded, stuck waiting to import (won't self-resolve)
|
||
# error status — serious failure not covered above
|
||
# stalled — download stuck with no connections or progress
|
||
#
|
||
# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first.
|
||
# Lidarr runs on HOST1 only — exits cleanly on HOST2.
|
||
ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this
|
||
# matches cron interval — items eligible after one missed cycle
|
||
|
||
HOST1_SONARR_RECOVERY=true
|
||
HOST1_RADARR_RECOVERY=true
|
||
HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2
|
||
HOST2_SONARR_RECOVERY=true
|
||
HOST2_RADARR_RECOVERY=true
|
||
|
||
# ==============================================================================================
|
||
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Session-based storage allocator using filesystem symlink indirection.
|
||
# ffmpeg resolves the symlink ONCE at session start — existing sessions never affected.
|
||
#
|
||
# ⚠️ Docker mount — must use shared propagation:
|
||
# --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared
|
||
|
||
# ━━━ Transcode Manager ━━━
|
||
# tmpfs mount point — created at array start by ramdisk_setup.sh
|
||
# Must exist before Emby starts so the symlink resolves correctly
|
||
RAMDISK_PATH="/mnt/ramdisk_transcodes"
|
||
|
||
# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront
|
||
# Real-world: 9 streams peaked at ~5.5GB — 8G gives comfortable headroom
|
||
RAMDISK_SIZE="8G"
|
||
|
||
# Symlink that Emby points at — this path NEVER changes regardless of ramdisk/SSD state
|
||
# Emby resolves the symlink once per session at start — symlink flips are transparent
|
||
# Must match the container path configured in Emby's Extra Parameters
|
||
TRANSCODE_LINK="/mnt/ram-transcode"
|
||
|
||
# SSD fallback location — where transcodes land when ramdisk is too full
|
||
TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
|
||
|
||
# Usage thresholds in GB — hysteresis gap between WARN and LOW prevents flip-flop
|
||
# Gap (6.8 - 5.5 = 1.3GB) means ramdisk must drop 1.3GB before flipping back
|
||
RAMDISK_WARN_GB=6.8 # flip to SSD when ramdisk usage reaches this
|
||
RAMDISK_LOW_GB=5.5 # flip back to ramdisk when usage drops to this
|
||
RAMDISK_SSD_MIN_GB=20 # minimum free GB on SSD before allowing flip to SSD
|
||
|
||
TRANSCODE_MAX_AGE=20 # minutes — HLS segment age before cleanup eligibility
|
||
TRANSCODE_ORPHAN_AGE=30 # minutes — files with no matching session at all
|
||
TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in one hour
|
||
|
||
TRANSCODE_OWNER="nobody:users"
|
||
TRANSCODE_CHMOD="755"
|
||
|
||
# Operating mode — controls symlink direction behaviour
|
||
# smart — auto-flips between ramdisk and SSD based on thresholds (default)
|
||
# ramdisk — always uses ramdisk, warns if RAMDISK_WARN_GB exceeded but holds
|
||
# ssd — always uses SSD, never flips to ramdisk
|
||
TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd
|
||
|
||
# Daily statistics log — read by weekly_health_digest.sh for transcode summary
|
||
TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db"
|
||
TRANSCODE_LOG_RETENTION=90 # days before old entries purged
|
||
|
||
# ━━━ Transcode Server Array ━━━
|
||
# All media servers sharing the ramdisk transcode space.
|
||
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
|
||
# Entries with placeholder API keys are skipped automatically.
|
||
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD.
|
||
TRANSCODE_SERVERS=(
|
||
"${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby"
|
||
# "${HOST2_EMBY_CONTAINER}|${HOST2_EMBY_URL}|${HOST2_EMBY_API_KEY}|emby"
|
||
)
|
||
|
||
TRANSCODE_CHECK_EMBY=true
|
||
|
||
# ==============================================================================================
|
||
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# ━━━ Certificate Monitor ━━━
|
||
# Checks SSL certificate expiry via direct openssl connection — no NPM dependency.
|
||
# Checks the actual certificate served by each domain, not what NPM thinks it has.
|
||
# CERT_WARN_DAYS = notify this many days before expiry
|
||
# CERT_CRIT_DAYS = escalate to critical this many days before expiry
|
||
# CERT_TIMEOUT = seconds before giving up on the openssl connection
|
||
CERT_MONITOR_DOMAINS=(
|
||
"Gmer4Lfe.com"
|
||
"Gmer4Lfe.us"
|
||
)
|
||
CERT_WARN_DAYS=30 # warn when cert expires within this many days
|
||
CERT_CRIT_DAYS=7 # critical alert within this many days
|
||
CERT_TIMEOUT=10 # seconds per domain check
|
||
|
||
# ━━━ Backup Verify ━━━
|
||
# Verifies rsync mirror health by comparing random file checksums between servers.
|
||
# Catches silent corruption or incomplete syncs that rsync itself wouldn't detect.
|
||
# Leave BACKUP_VERIFY_SHARES empty to use HOST*_DAILY_SYNC_SHARES automatically.
|
||
BACKUP_VERIFY_SHARES=(
|
||
# leave empty to use HOST*_DAILY_SYNC_SHARES automatically
|
||
)
|
||
BACKUP_VERIFY_SAMPLE=10 # random files to check per share
|
||
BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample
|
||
|
||
# ━━━ SMART Health ━━━
|
||
# Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*.
|
||
# Thresholds read from /boot/config/plugins/dynamix/dynamix.cfg at runtime
|
||
# (hot/max/hotssd/maxssd) — these vars are fallback only if dynamix.cfg not found.
|
||
SMART_TEMP_WARN=45 # fallback — Celsius warn threshold
|
||
SMART_TEMP_CRIT=55 # fallback — Celsius critical threshold
|
||
SMART_IGNORE_DRIVES=(
|
||
"sda" # boot USB — SMART not meaningful on flash drives
|
||
)
|
||
|
||
# ━━━ ZFS Memory Snapshot ━━━
|
||
# Weekly ZFS pool health and memory diagnostic report — informational only.
|
||
# ZFS_REPORT_ARC_WARN_PCT = warn if ARC is using more than this % of its max
|
||
# ZFS_REPORT_FREE_WARN_GB = warn if less than this GB free RAM
|
||
# ZFS_REPORT_AVAIL_WARN_GB = warn if less than this GB available on ZFS pool
|
||
# ZFS_REPORT_DOCKER_TOP = how many top Docker containers to show by memory usage
|
||
# ZFS_REPORT_IGNORE_POOLS = individual disk pools to skip (unRAID array disks as ZFS)
|
||
ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log"
|
||
ZFS_REPORT_ARC_WARN_PCT=90
|
||
ZFS_REPORT_FREE_WARN_GB=10
|
||
ZFS_REPORT_AVAIL_WARN_GB=20
|
||
ZFS_REPORT_DOCKER_TOP=10
|
||
ZFS_REPORT_IGNORE_POOLS=(
|
||
"disk5"
|
||
"disk6"
|
||
"disk8"
|
||
"disk9"
|
||
"disk10"
|
||
)
|
||
|
||
# ━━━ Bandwidth Monitor ━━━
|
||
# Called automatically by rsync.sh after each sync — one bounded write per run.
|
||
# Tracks transfer size, duration and profile per sync for weekly summary reporting.
|
||
BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db"
|
||
BANDWIDTH_LOG_RETENTION=90 # days before old entries purged
|
||
BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report
|
||
|
||
# Stats files — written by cleanup and recovery scripts, read by coffee report
|
||
# All in DATA_DIR — array always running when these are written
|
||
ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats
|
||
ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats
|
||
|
||
# ━━━ Health Digest ━━━
|
||
# Aggregated system health summary — reads existing state files, no new writes.
|
||
# Three profiles control when the digest is sent:
|
||
# always — sends every run regardless of findings
|
||
# smart — sends only when DIGEST_SMART_ON_* conditions are found
|
||
# weekly — sends once per week on DIGEST_DAY only
|
||
DIGEST_PROFILE="weekly" # always | smart | weekly
|
||
DIGEST_DAY="Sunday"
|
||
DIGEST_SMART_ON_WATCHDOG=true # send if any watchdog strikes are active
|
||
DIGEST_SMART_ON_FAILOVER=true # send if failover state is not NORMAL
|
||
DIGEST_SMART_ON_CERT_WARN=true # send if any cert is under CERT_WARN_DAYS
|
||
DIGEST_SMART_ON_BANDWIDTH=true # send if any transfer exceeded BANDWIDTH_WARN_GB
|
||
|
||
# ━━━ Emby Session Report ━━━
|
||
# Weekly Emby usage statistics via API — no persistent writes, queries fresh each run.
|
||
# URL and API key pulled from HOST1/HOST2_EMBY_URL and HOST1/HOST2_EMBY_API_KEY.
|
||
EMBY_REPORT_DAYS=7 # days to include in the report period
|
||
EMBY_REPORT_TOP_N=10 # number of top content items to show
|
||
|
||
# ==============================================================================================
|
||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Continuous system health monitoring — last line of defense before a crash.
|
||
# Started by array_start.sh — runs until array stops.
|
||
# Re-sources Master.conf each cycle — config changes take effect on next cycle.
|
||
# Strike system: sustained threshold hits trigger reboot — single spikes ignored.
|
||
# Reboot loop protection: shuts down instead if reboot limit hit in rolling window.
|
||
|
||
# ━━━ State Files ━━━
|
||
SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" # /tmp resets on reboot ✅
|
||
SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db"
|
||
SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db"
|
||
|
||
# ━━━ Strike and Reboot Loop Settings ━━━
|
||
# Strike system: a check must fail this many consecutive cycles before action is taken
|
||
# Single spikes (one bad reading) are ignored — sustained problems trigger reboot
|
||
SYS_WATCHDOG_STRIKE_LIMIT=2 # consecutive failures before reboot trigger
|
||
|
||
# How often checks run — 300s = 5 minutes
|
||
# At STRIKE_LIMIT=2 and INTERVAL=300: problem must persist 10min before reboot
|
||
SYSTEM_WATCHDOG_INTERVAL=300
|
||
|
||
# Reboot loop protection — if system keeps rebooting something is seriously wrong
|
||
# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS → shutdown instead of reboot
|
||
SYS_WATCHDOG_REBOOT_LIMIT=3
|
||
SYS_WATCHDOG_REBOOT_WINDOW_HRS=12
|
||
|
||
# Heartbeat — proof of life logged periodically even when everything is healthy
|
||
SYSTEM_WATCHDOG_HEARTBEAT=true
|
||
SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1
|
||
|
||
# ━━━ Thresholds ━━━
|
||
# Set at "about to become unstable" levels — not "things are a bit high"
|
||
# These should be high enough that normal operation never triggers them
|
||
|
||
# rootfs (/) usage — when array is down rsync writes land on rootfs, fills rapidly
|
||
SYS_WATCHDOG_ROOTFS_PCT=95
|
||
|
||
# /var/log usage — log spam can fill rootfs, indicates something broken
|
||
SYS_WATCHDOG_LOG_PCT=95
|
||
|
||
# Free RAM in GB — below this is critically low, OOM or swap imminent
|
||
SYS_WATCHDOG_MEM_GB=4
|
||
|
||
# ZFS ARC pinned percentage — ARC not releasing after reclaim = memory stuck
|
||
SYS_WATCHDOG_ARC_PINNED_PCT=98
|
||
SYS_WATCHDOG_ARC_RELEASE_PCT=95
|
||
|
||
# Load average multiplier — threshold = MULTIPLIER × CPU core count
|
||
# MULTIPLIER=3 on 16-core = load average of 48 before triggering
|
||
SYS_WATCHDOG_LOAD_MULTIPLIER=3
|
||
|
||
# Zombie process count — large numbers indicate serious process management failure
|
||
SYS_WATCHDOG_ZOMBIE_LIMIT=50
|
||
|
||
# CPU temperature — sustained high temp causes throttling or kernel panic
|
||
SYS_WATCHDOG_CPU_TEMP_MAX=95
|
||
|
||
# ━━━ Check Toggles ━━━
|
||
# Disable individual checks without disabling the whole watchdog
|
||
SYS_WATCHDOG_CHECK_ROOTFS=true
|
||
SYS_WATCHDOG_CHECK_LOG=true
|
||
SYS_WATCHDOG_CHECK_RAM=true
|
||
SYS_WATCHDOG_CHECK_ARC=true
|
||
SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
||
SYS_WATCHDOG_CHECK_LOAD=false # disabled — load spikes during transcoding are normal
|
||
SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||
SYS_WATCHDOG_CHECK_CONTAINERS=true # checks docker_watchdog persistent skip list
|
||
SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
|
||
|
||
# ━━━ Abort Toggles ━━━
|
||
# Conditions that prevent reboot even when a threshold is hit
|
||
# true = abort reboot if this condition is active (conservative — avoid data loss)
|
||
# false = reboot anyway (aggressive — a clean reboot beats a hard crash)
|
||
SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # rebooting with bad pool risks data loss
|
||
SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity is better than crashing mid-check
|
||
SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting move is better than crashing mid-move
|
||
|
||
# ==============================================================================================
|
||
# ──────────────────────── End Of User Variables ───────────────────────────────────────────────
|
||
# ============================================================================================== |