652 lines
33 KiB
Bash
652 lines
33 KiB
Bash
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ================================= MASTER CONFIGURATION =======================================
|
|
# ==============================================================================================
|
|
# All user-facing variables for the unRAID script ecosystem.
|
|
# Scripts source this file — edit here, changes apply everywhere on next git pull.
|
|
#
|
|
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
|
#
|
|
# Section Description
|
|
# ───────────────────────────────────────────────────────────────────────────────────────────
|
|
# HOST CONFIGURATION Server hostnames and SSH key paths
|
|
# LOGGING Enable or disable verbose logging
|
|
# NOTIFICATIONS unRAID native and Discord webhook settings
|
|
# GIT / REPO Gitea repository and SSH settings
|
|
#
|
|
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
|
# RSYNC DEFAULTS Global fallback rsync settings
|
|
# REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort
|
|
# DAILY SYNC SHARES Media shares synced by daily_sync.sh
|
|
# RSYNC PROFILE SYSTEM Per-profile overrides (appdata profiles)
|
|
#
|
|
# ── FAILOVER ───────────────────────────────────────────────────────────────────────────────
|
|
# FAILOVER Mutual container failover between two servers
|
|
#
|
|
# ── DOCKER ESSENTIALS ──────────────────────────────────────────────────────────────────────
|
|
# DOCKER DAILY RESTART Containers restarted daily
|
|
# DOCKER WEEKLY RESTART Containers restarted weekly
|
|
# DOCKER WATCHDOG Container health monitoring — memory, CPU, HTTP
|
|
# DOCKER NETWORK CONNECT Connect containers to extra networks on boot
|
|
#
|
|
# ── UNRAID ESSENTIALS ──────────────────────────────────────────────────────────────────────
|
|
# REBOOT User warning delay before scheduled reboot
|
|
# MOVER Mover stop timeout
|
|
# SYSLOG FILTER Docker veth noise filter file path
|
|
# PHP-FPM PHP-FPM max children config
|
|
# CLEAR LOGS System log file paths
|
|
# WEBGUI WATCHDOG WebGUI nginx + emhttp monitoring and restart
|
|
# ZFS MEMORY SNAPSHOT Weekly ZFS health and memory diagnostic report
|
|
#
|
|
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
|
# MEDIA PERMISSIONS Share list, mode and owner for permissions script
|
|
# MEDIA CLEANER Anime and media folder lists and file patterns
|
|
#
|
|
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
|
# TRANSCODE MANAGER Ramdisk and SSD fallback transcode management
|
|
#
|
|
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
|
# SYSTEM WATCHDOG System health monitoring — last line of defense
|
|
#
|
|
# ==============================================================================================
|
|
|
|
# ━━━ Host Configuration ━━━
|
|
# Hostnames must match Tailscale machine names exactly — case sensitive
|
|
HOST1="unRAID-Gmer4Lfe"
|
|
HOST2="unRAID-Jayred365"
|
|
# SSH keys for server-to-server rsync and failover operations
|
|
# Each server authenticates with its own key — both must be authorised on the remote
|
|
HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key"
|
|
HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key"
|
|
|
|
# ━━━ Logging ━━━
|
|
# true = verbose [LOG] lines in all script output / false = user-facing output only
|
|
ENABLE_LOGGING=true
|
|
|
|
# ━━━ Notifications ━━━
|
|
# unRAID native — fires through Settings → Notification Settings
|
|
# Recommended: configure unRAID to send errors/warnings only so normal completions stay quiet
|
|
NOTIFY_UNRAID=true
|
|
# Discord webhook — paste full webhook URL to enable, leave blank to disable
|
|
DISCORD_WEBHOOK=""
|
|
|
|
# ━━━ Git / Repo ━━━
|
|
# Gitea self-hosted repository for the script ecosystem
|
|
# git_pull_execute.sh uses these to clone or pull the latest version on both servers
|
|
REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git"
|
|
TARGET_DIR="/mnt/user/appdata/unraid_scripts"
|
|
GITEA_SSH_KEY="/root/.ssh/unraid_gitea"
|
|
SSH_PORT=221
|
|
|
|
# ==============================================================================================
|
|
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
|
|
# ━━━ Rsync Defaults ━━━
|
|
# Global fallback values used when no profile match is found for a directory.
|
|
# Shares in DAILY_SYNC_SHARES always use these globals — no profile is defined for them.
|
|
# Appdata shares (Arrs_Stack, Critical-Data etc.) match profiles by directory basename.
|
|
|
|
# Network transfer speed cap in KB/s — 12500 ≈ 100Mbit
|
|
BW_LIMIT=12500
|
|
# Number of retry attempts if rsync fails before giving up
|
|
RETRY_COUNT=3
|
|
# Seconds to wait between retry attempts
|
|
SLEEP=300
|
|
# Containers to stop on remote before rsync and restart after — empty by default
|
|
# Profiles below override this per appdata share
|
|
CRITICAL_CONTAINER_NAMES=()
|
|
# Containers that need a delay before starting — e.g. Authelia needs its DB ready first
|
|
DELAYED_CONTAINERS=()
|
|
# Seconds to wait before starting delayed containers
|
|
CONTAINER_DELAY=5
|
|
# Directories to exclude from transfer — empty by default, profiles override per share
|
|
EXCLUDE_DIRS=()
|
|
# Default rsync options used when no profile match is found
|
|
# --delete removes files on remote that no longer exist on source
|
|
# --inplace writes directly to destination file rather than temp file — better for large files
|
|
# --no-whole-file forces delta transfer even on local-like connections
|
|
DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file)
|
|
|
|
# ━━━ Remote Health Checks ━━━
|
|
# Abort rsync if remote rootfs (/) usage is at or above this percentage.
|
|
# When the remote array is down or drives are missing, rsync writes land on rootfs instead
|
|
# of /mnt/user — this fills the filesystem rapidly and can crash the remote server.
|
|
# 75% gives enough headroom to detect the problem before it becomes critical.
|
|
ROOTFS_WARN=75
|
|
|
|
# ━━━ Daily Sync Shares ━━━
|
|
# Media shares synced once daily by Orchestrators/daily_sync.sh
|
|
# These shares have no profile — all use DEFAULT_RSYNC_OPTS above
|
|
# Add or remove paths to control what gets synced each night
|
|
DAILY_SYNC_SHARES=(
|
|
/mnt/user/Anime_Movies-Old
|
|
/mnt/user/Anime_Shows-Old
|
|
/mnt/user/Anime_Shows
|
|
/mnt/user/Books
|
|
/mnt/user/Intros
|
|
/mnt/user/Kids_Movies
|
|
/mnt/user/Kids_Tv_Shows
|
|
/mnt/user/Movies
|
|
/mnt/user/Music_Videos
|
|
/mnt/user/Nextcloud
|
|
/mnt/user/stand-up_comedy
|
|
/mnt/user/Tv_Shows
|
|
)
|
|
|
|
# ━━━ Rsync Profile System ━━━
|
|
# Profiles allow per-share rsync behaviour without touching script logic.
|
|
# Each profile is matched automatically by the basename of the directory passed to rsync.sh
|
|
# (lowercased). Example: /mnt/user/appdata-Failover/Arrs_Stack → profile key = arrs_stack
|
|
#
|
|
# How fallthrough works:
|
|
# If a key exists in a profile array → that value is used for this run
|
|
# If a key is missing → the global default above is used instead
|
|
# Shares in DAILY_SYNC_SHARES → always use globals, no profile defined
|
|
#
|
|
# To add a new profile:
|
|
# 1. Add a key to each array below with your chosen profile name
|
|
# 2. Call rsync.sh with a directory whose basename matches that key
|
|
# 3. Any array you omit falls back to its global default automatically
|
|
#
|
|
# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit from DEFAULT_RSYNC_OPTS —
|
|
# if you define a profile entry you must list all desired options explicitly
|
|
#
|
|
# Profile descriptions:
|
|
# arrs_stack — Sonarr, Radarr, Lidarr, Readarr, Prowlarr, Bazarr, Pinchflat appdata
|
|
# Lower bandwidth — runs alongside media syncs, containers stopped during sync
|
|
# critical-data — Auth stack: NPM, Authelia, Mariadb, Redis, LLDAP
|
|
# High bandwidth — small data, synced frequently, Authelia needs delayed start
|
|
# gmer4lfe — Server-specific appdata: Organizr, UptimeKuma, VaultWarden
|
|
# Medium bandwidth — personal services, no container stop needed
|
|
# important-data — NextCloud + Postgres database
|
|
# High bandwidth — NextCloud needs graceful stop before sync
|
|
# emby — Emby media server appdata and metadata
|
|
# Medium bandwidth — large appdata, no containers stopped (metadata only)
|
|
|
|
# SPACE-SEPARATED STRINGS — rsync options per profile
|
|
# If defined for a profile, these replace DEFAULT_RSYNC_OPTS entirely for that run
|
|
declare -A PROFILE_RSYNC_OPTS=(
|
|
[arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace"
|
|
[critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete"
|
|
[gmer4lfe]="-av --info=progress2 --bwlimit=$BW_LIMIT"
|
|
[important-data]="-av --human-readable --bwlimit=$BW_LIMIT"
|
|
[emby]="-av --human-readable --bwlimit=$BW_LIMIT"
|
|
)
|
|
|
|
# Bandwidth limit in KB/s per profile — overrides global BW_LIMIT for this profile
|
|
declare -A PROFILE_BW_LIMIT=(
|
|
[arrs_stack]=5000 # lower — runs alongside other jobs
|
|
[critical-data]=9500 # high — small data, sync fast
|
|
[gmer4lfe]=8000 # medium
|
|
[important-data]=9500 # high — database sync, prioritise speed
|
|
[emby]=8000 # medium — large files, steady transfer
|
|
)
|
|
|
|
# Retry attempts per profile — overrides global RETRY_COUNT
|
|
declare -A PROFILE_RETRY_COUNT=(
|
|
[arrs_stack]=3
|
|
[critical-data]=3
|
|
[gmer4lfe]=3
|
|
[important-data]=3
|
|
[emby]=3
|
|
)
|
|
|
|
# Sleep between retries in seconds — overrides global SLEEP
|
|
declare -A PROFILE_SLEEP=(
|
|
[arrs_stack]=300
|
|
[critical-data]=300
|
|
[gmer4lfe]=300
|
|
[important-data]=300
|
|
[emby]=300
|
|
)
|
|
|
|
# Containers stopped on REMOTE before rsync and restarted after — SPACE-SEPARATED STRINGS
|
|
# Only containers that need to be stopped for data consistency — running containers are fine
|
|
# for most media but databases and auth stacks need clean state during sync
|
|
declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
|
|
[arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
|
|
[critical-data]="Mariadb-Authelia Redis-Authelia Lldap-Gmer4Lfe NginxProxyManager Authelia"
|
|
[gmer4lfe]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
|
|
[important-data]="Postgres-NextCloud NextCloud"
|
|
[emby]=""
|
|
)
|
|
|
|
# Containers that need a delay before starting after rsync — SPACE-SEPARATED STRINGS
|
|
# Authelia needs its database containers (Mariadb, Redis) to be ready before it starts
|
|
# NextCloud needs Postgres to be accepting connections before it starts
|
|
declare -A PROFILE_DELAYED_CONTAINERS=(
|
|
[arrs_stack]=""
|
|
[critical-data]="Authelia"
|
|
[gmer4lfe]=""
|
|
[important-data]="NextCloud"
|
|
[emby]=""
|
|
)
|
|
|
|
# Seconds to wait before starting delayed containers — per profile
|
|
declare -A PROFILE_CONTAINER_DELAY=(
|
|
[arrs_stack]=5
|
|
[critical-data]=10 # Authelia needs DB ready — 10s gives Mariadb/Redis time to start
|
|
[gmer4lfe]=5
|
|
[important-data]=10 # NextCloud needs Postgres ready
|
|
[emby]=5
|
|
)
|
|
|
|
# Directories excluded from transfer per profile — SPACE-SEPARATED STRINGS
|
|
# logs and *.tmp are excluded universally — they are ephemeral and regenerated on start
|
|
declare -A PROFILE_EXCLUDE_DIRS=(
|
|
[arrs_stack]="logs *.tmp"
|
|
[critical-data]="logs *.tmp"
|
|
[gmer4lfe]="logs *.tmp"
|
|
[important-data]="logs *.tmp"
|
|
[emby]="logs *.tmp"
|
|
)
|
|
|
|
# ━━━ Profile Disk Check Toggle ━━━
|
|
# true = skip per-disk check for this profile — use when share lives on a ZFS pool
|
|
# false = run per-disk check — use for traditional unRAID array with individual disks
|
|
# Per-disk check looks for /mnt/disk*/sharename — ZFS pools don't have this structure
|
|
# The share existence and content checks still run regardless of this setting
|
|
declare -A PROFILE_SKIP_DISK_CHECK=(
|
|
[arrs_stack]=true
|
|
[critical-data]=true
|
|
[gmer4lfe]=true
|
|
[important-data]=true
|
|
[emby]=true
|
|
)
|
|
# ==============================================================================================
|
|
# ── FAILOVER ──────────────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
# Mutual container failover between two unRAID servers 50 miles apart.
|
|
# Each server runs failover.sh independently — no coordination between servers.
|
|
# All decisions are based solely on two ping checks: remote reachable + internet reachable.
|
|
#
|
|
# States:
|
|
# NORMAL — remote up, internet up — own containers only, silent operation
|
|
# FAILOVER — remote down, internet up — start remote's containers locally (additive)
|
|
# NO_INTERNET — internet down — stop public-facing containers, wait
|
|
# DARK — remote down + internet down — same as NO_INTERNET
|
|
#
|
|
# Handback sequence when remote returns after FAILOVER:
|
|
# Strike confirmation (FAILOVER_HANDBACK_STRIKES consecutive remote-up checks)
|
|
# → pre-flight checks (remote array, docker daemon, rootfs)
|
|
# → rsync data back via rsync.sh (uses profile system)
|
|
# → start failover containers on remote via SSH
|
|
# → stop failover containers locally
|
|
# → return to NORMAL
|
|
#
|
|
# Comment out any container or rsync job to disable without removing the entry.
|
|
# Script runs on BOTH servers — detect_hosts() selects the correct arrays automatically.
|
|
|
|
# External IP to ping for internet connectivity check — Google DNS, reliable and fast
|
|
EXTERNAL_IP="8.8.8.8"
|
|
# Seconds between state checks — 120s = 2 minute polling interval
|
|
# With 1 minute DNS TTL this means failover is visible to users within ~3 minutes
|
|
FAILOVER_CHECK_INTERVAL=120
|
|
# Consecutive remote-up confirmations required before initiating handback
|
|
# Prevents handing back during a brief network blip — 2 strikes = 4 minutes confirmation
|
|
FAILOVER_HANDBACK_STRIKES=2
|
|
# State file path — on /boot/ so it survives reboots
|
|
# Script re-evaluates from scratch on restart using live pings — state file is reference only
|
|
FAILOVER_STATE_FILE="/boot/config/failover_state.db"
|
|
|
|
# ━━━ HOST1 Failover Config (unRAID-Gmer4Lfe — Primary) ━━━
|
|
|
|
# Containers HOST1 starts locally when HOST2 goes down
|
|
# These run on top of HOST1's normal containers — additive, not replacement
|
|
# HOST2's DDNS containers must start here so DNS stays pointing at HOST1 during HOST2 outage
|
|
FAILOVER_HOST1_STARTS_FOR_HOST2=(
|
|
"Gmer4Lfe.com"
|
|
"Gmer4Lfe.us"
|
|
)
|
|
|
|
# Containers HOST1 stops when it loses internet
|
|
# No point serving DDNS or media if HOST1 itself has no internet — stops unnecessary churn
|
|
# HOST2 will independently detect HOST1 is gone and start its own failover
|
|
FAILOVER_HOST1_STOP_ON_NO_NET=(
|
|
"Gmer4Lfe.com"
|
|
"Gmer4Lfe.us"
|
|
)
|
|
|
|
# Rsync jobs HOST1 runs before handing containers back to HOST2
|
|
# Uses rsync.sh with existing profile system — basename matched to profile keys
|
|
# Comment out jobs that don't need syncing back or aren't ready yet
|
|
FAILOVER_HOST1_RSYNC_JOBS=(
|
|
# "/mnt/user/appdata-Failover/Jayred365"
|
|
# "/mnt/user/Media_Server/Emby-Jayred"
|
|
)
|
|
|
|
# ━━━ HOST2 Failover Config (unRAID-Jayred365 — Secondary) ━━━
|
|
|
|
# Containers HOST2 starts locally when HOST1 goes down
|
|
# Emby starts on HOST2 so media keeps working during HOST1 outage
|
|
# HOST1's DDNS containers start here so DNS updates to point at HOST2
|
|
FAILOVER_HOST2_STARTS_FOR_HOST1=(
|
|
"Emby"
|
|
"Gmer4Lfe.com"
|
|
"Gmer4Lfe.us"
|
|
)
|
|
|
|
# Containers HOST2 stops when it loses internet
|
|
# HOST2's DDNS containers serve no purpose without internet connectivity
|
|
FAILOVER_HOST2_STOP_ON_NO_NET=(
|
|
"Gmer4Lfe.com"
|
|
"Gmer4Lfe.us"
|
|
)
|
|
|
|
# Rsync jobs HOST2 runs before handing containers back to HOST1
|
|
# Comment out jobs that don't need syncing back or aren't ready yet
|
|
FAILOVER_HOST2_RSYNC_JOBS=(
|
|
# "/mnt/user/appdata-Failover/Gmer4Lfe"
|
|
)
|
|
|
|
# ==============================================================================================
|
|
# ── DOCKER ESSENTIALS ─────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
|
|
# ━━━ Docker Daily Restart ━━━
|
|
# Containers restarted every day — keeps services fresh, clears memory leaks
|
|
# Case-sensitive — must match exact Docker container names
|
|
DAILY_RESTART_CONTAINERS=(
|
|
"NginxProxyManager"
|
|
"Authelia"
|
|
"Dispatcharr-Iptv-Users"
|
|
"Dispatcharr"
|
|
"Dispatcharr-Basic"
|
|
"ErsatzTV-Emby"
|
|
)
|
|
|
|
# ━━━ Docker Weekly Restart ━━━
|
|
# Containers restarted once per week — less critical services that benefit from periodic restart
|
|
WEEKLY_RESTART_CONTAINERS=(
|
|
"NextCloud"
|
|
"Organizrv2-Gmer4Lfe"
|
|
"AdGuard-Home"
|
|
"Immich-Gmer4Lfe"
|
|
)
|
|
|
|
# ━━━ Docker Watchdog ━━━
|
|
# First line of defense — monitors containers for memory, CPU and HTTP responsiveness.
|
|
# Restarts containers that exceed thresholds using a strike system to avoid false positives.
|
|
# Works alongside system_watchdog.sh — containers first, system reboot second.
|
|
|
|
# Containers to monitor with their memory hard limits in MB
|
|
# Strike system used for CPU and responsiveness — immediate restart for memory hard limit
|
|
# 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240
|
|
# 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024
|
|
declare -A WATCHDOG_CONTAINERS=(
|
|
["Emby"]=16384 # 16GB — large media server, transcoding can spike
|
|
["LidaTube"]=6144 # 6GB — YouTube downloader
|
|
["Tdarr"]=6144 # 6GB — transcoding node
|
|
["Code-Server"]=1024 # 1GB — VS Code server
|
|
)
|
|
|
|
# Containers to check HTTP responsiveness — curl check per interval
|
|
# Omit a container entirely to skip its HTTP check
|
|
declare -A WATCHDOG_CONTAINER_URLS=(
|
|
["Emby"]="http://localhost:8096"
|
|
)
|
|
|
|
# Containers that should always be running — monitored for unexpected stops
|
|
# Strike system used — persistent skip list on /boot/ prevents reboot loops
|
|
# Auto-clears from skip list when container recovers after reboot or manual fix
|
|
WATCHDOG_REQUIRED_CONTAINERS=(
|
|
"NginxProxyManager"
|
|
"Lldap-Gmer4Lfe"
|
|
"Authelia"
|
|
"Mariadb-Authelia"
|
|
"Redis-Authelia"
|
|
"Authelia-Secondary"
|
|
"Redis-Authelia-Secondary"
|
|
)
|
|
|
|
# Strike state file — /tmp resets on reboot which is correct for strike tracking
|
|
WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db"
|
|
|
|
# CPU thresholds — normalised against total core count at runtime
|
|
SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU
|
|
HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU
|
|
CPU_FAIL_LIMIT=2 # consecutive strikes before container restart
|
|
|
|
# Memory — warn at this % of per-container hard limit defined in WATCHDOG_CONTAINERS
|
|
SOFT_MEM_THRESHOLD=80
|
|
|
|
# HTTP responsiveness check
|
|
RESP_FAIL_LIMIT=2 # consecutive failed curl checks before restart
|
|
CURL_TIMEOUT=5 # seconds before curl gives up per check
|
|
|
|
# ━━━ Docker Network Connect ━━━
|
|
# Connects containers to extra Docker networks on array start — many-to-many
|
|
# Every container in the list connects to every network in the list
|
|
# Useful when containers need to communicate across networks they weren't configured with
|
|
# Comment out entries to disable without removing them
|
|
NETWORK_CONNECT_CONTAINERS=(
|
|
"memcached"
|
|
"Npm-CrowdSec"
|
|
)
|
|
|
|
NETWORK_CONNECT_NETWORKS=(
|
|
"nextcloud-aio"
|
|
)
|
|
|
|
# ==============================================================================================
|
|
# ── UNRAID ESSENTIALS ─────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
|
|
# ━━━ Reboot ━━━
|
|
# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots
|
|
REBOOT_SLEEP=300
|
|
|
|
# ━━━ Mover ━━━
|
|
# Seconds to wait after warning users before mover_stop.sh kills the mover process
|
|
MOVER_STOP_TIMEOUT=300
|
|
|
|
# ━━━ Syslog Filter ━━━
|
|
# Path for the rsyslog filter file that suppresses Docker veth/docker0 noise
|
|
FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"
|
|
|
|
# ━━━ PHP-FPM ━━━
|
|
# Config file path and max children value for php_fpm_max_children.sh
|
|
PHP_CONF="/etc/php-fpm.d/www.conf"
|
|
PHP_MAX_CHILDREN=250
|
|
|
|
# ━━━ Clear Logs ━━━
|
|
# System log files cleared by clear_logs.sh — Docker logs cleared automatically too
|
|
LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg)
|
|
|
|
# ━━━ WebGUI Watchdog ━━━
|
|
# Monitors unRAID WebGUI and restarts services if unresponsive
|
|
# Escalation: nginx restart → recheck → emhttp restart → recheck → notify warning
|
|
WEBGUI_URL="http://localhost" # adjust port if non-standard e.g. http://localhost:8080
|
|
WEBGUI_TIMEOUT=5 # seconds before curl gives up
|
|
WEBGUI_NGINX_WAIT=15 # seconds to wait after nginx restart before recheck
|
|
WEBGUI_EMHTTP_WAIT=30 # seconds to wait after emhttp restart before recheck
|
|
|
|
# ━━━ ZFS Memory Snapshot ━━━
|
|
# Weekly ZFS pool health and memory diagnostic report — informational only, no action taken
|
|
# system_watchdog.sh handles threshold-based intervention
|
|
ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log"
|
|
ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC utilization above this %
|
|
ZFS_REPORT_FREE_WARN_GB=10 # warn if free RAM below this GB
|
|
ZFS_REPORT_AVAIL_WARN_GB=20 # warn if available RAM below this GB
|
|
ZFS_REPORT_DOCKER_TOP=10 # number of top Docker memory users to show in report
|
|
|
|
# ==============================================================================================
|
|
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
|
|
# ━━━ Media Permissions ━━━
|
|
# Mode and owner applied recursively to all shares in MEDIA_PERMISSION_SHARES
|
|
PERMISSIONS_MODE="777"
|
|
PERMISSIONS_OWNER="nobody:users"
|
|
|
|
MEDIA_PERMISSION_SHARES=(
|
|
/mnt/user/Anime_Movies
|
|
/mnt/user/Anime_Movies-Old
|
|
/mnt/user/Anime_Shows
|
|
/mnt/user/Anime_Shows-Old
|
|
/mnt/user/appcache
|
|
/mnt/user/Books
|
|
/mnt/user/Downloads
|
|
/mnt/user/Games
|
|
/mnt/user/Intros
|
|
/mnt/user/Kids_Movies
|
|
/mnt/user/Kids_Tv_Shows
|
|
/mnt/user/Movie_Recordings
|
|
/mnt/user/Movies
|
|
/mnt/user/Music
|
|
/mnt/user/Music_Videos
|
|
/mnt/user/Photo
|
|
/mnt/user/Sports
|
|
/mnt/user/stand-up_comedy
|
|
/mnt/user/Temp_Storage
|
|
/mnt/user/Tv_Recordings
|
|
/mnt/user/Tv_Shows
|
|
/mnt/user/YouTube
|
|
)
|
|
|
|
# ━━━ Media Cleaner ━━━
|
|
# Two profiles: anime and media — passed as argument to media_cleaner.sh
|
|
# Run via Orchestrators/media_management.sh for permissions + both cleaners in one job
|
|
# Usage: media_cleaner.sh anime or media_cleaner.sh media
|
|
|
|
ANIME_CLEAN_FOLDERS=(
|
|
/mnt/user/Anime_Movies
|
|
/mnt/user/Anime_Movies-Old
|
|
/mnt/user/Anime_Shows
|
|
/mnt/user/Anime_Shows-Old
|
|
)
|
|
|
|
MEDIA_CLEAN_FOLDERS=(
|
|
/mnt/user/Kids_Movies
|
|
/mnt/user/Kids_Tv_Shows
|
|
/mnt/user/Movies
|
|
/mnt/user/Music
|
|
/mnt/user/Sports
|
|
/mnt/user/stand-up_comedy
|
|
/mnt/user/Tv_Shows
|
|
)
|
|
|
|
# Junk file patterns common in anime downloads
|
|
ANIME_FILE_PATTERNS=(
|
|
'*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk'
|
|
'*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*'
|
|
'*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp'
|
|
'*.log' '*.json'
|
|
)
|
|
|
|
# Junk file patterns for general media — includes *.iso and *.lrc not needed in anime
|
|
MEDIA_FILE_PATTERNS=(
|
|
'*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk'
|
|
'*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*'
|
|
'*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp'
|
|
'*.log' '*.json' '*.iso' '*.lrc'
|
|
)
|
|
|
|
# ━━━ Media Management Orchestrator ━━━
|
|
# Scripts run sequentially by Orchestrators/media_management.sh
|
|
# Order matters — permissions runs first so cleaner operates on correctly owned files
|
|
# Format: "script_path [optional_argument]"
|
|
# Add a new line to run another script — comment out to disable without removing
|
|
MEDIA_MAINTENANCE_JOBS=(
|
|
"Media/media_shares_permissions.sh"
|
|
"Media/media_cleaner.sh anime"
|
|
"Media/media_cleaner.sh media"
|
|
)
|
|
|
|
# ==============================================================================================
|
|
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
# Session-based storage allocator using filesystem indirection.
|
|
# ffmpeg resolves the symlink path once at session start — existing sessions are never affected.
|
|
# New sessions land wherever TRANSCODE_LINK points at the moment they start.
|
|
#
|
|
# Flow:
|
|
# ramdisk_setup.sh — run once at array start, creates ramdisk and sets symlink
|
|
# transcode_manager.sh — every 3 min, monitors usage and flips symlink if needed
|
|
# transcode_cleanup.sh — every 5 min, removes old inactive files from both locations
|
|
#
|
|
# Hysteresis gap between RAMDISK_WARN_GB and RAMDISK_LOW_GB prevents flip-flop
|
|
# when usage hovers near the threshold — gap should be at least 0.5-1GB
|
|
|
|
RAMDISK_PATH="/mnt/ramdisk_transcodes" # tmpfs mount point — created at array start
|
|
RAMDISK_SIZE="8G" # ceiling — tmpfs is dynamic, only uses what's needed
|
|
TRANSCODE_LINK="/mnt/ram-transcode" # symlink Emby points at — never changes location
|
|
TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" # SSD fallback for edge cases
|
|
|
|
# Thresholds in GB — flip symlink to SSD at WARN, flip back to ramdisk at LOW
|
|
RAMDISK_WARN_GB=6.8 # flip to SSD at or above this — "getting full"
|
|
RAMDISK_LOW_GB=5.5 # flip back to ramdisk when cleanup brings usage here
|
|
RAMDISK_SSD_MIN_GB=20 # minimum free GB on SSD before allowing flip — safety net
|
|
|
|
# Cleanup — files must be older than MAX_AGE and not open by any process to be deleted
|
|
TRANSCODE_MAX_AGE=20 # minutes before a file is eligible for cleanup
|
|
TRANSCODE_ORPHAN_AGE=30 # minutes before an orphaned file is eligible — extra caution
|
|
|
|
# Flip frequency monitoring — notify if symlink flips too often (indicates sizing issue)
|
|
TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in one hour
|
|
|
|
# Permissions — must match your Emby container user
|
|
TRANSCODE_OWNER="nobody:users"
|
|
TRANSCODE_MODE="755"
|
|
|
|
# ==============================================================================================
|
|
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
# Last line of defense — reboots cleanly if the system is about to become unstable.
|
|
# Works alongside docker_watchdog.sh — containers first, system reboot second.
|
|
# Thresholds set at "about to fall over" levels — not just high usage.
|
|
# Strike system prevents rebooting on single spikes — sustained threshold hits trigger action.
|
|
|
|
# ━━━ State Files ━━━
|
|
# Strike counts reset on reboot — /tmp is correct for this
|
|
SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db"
|
|
# Persistent container skip list — survives reboots, auto-clears when container recovers
|
|
SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db"
|
|
# Reboot timestamp log — survives reboots for loop detection
|
|
SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db"
|
|
|
|
# ━━━ Strike and Reboot Loop Settings ━━━
|
|
# Consecutive threshold hits before triggering reboot
|
|
SYS_WATCHDOG_STRIKE_LIMIT=2
|
|
# Maximum reboots allowed in window before shutdown instead — prevents reboot loops
|
|
SYS_WATCHDOG_REBOOT_LIMIT=3
|
|
# Window in hours — controls both reboot count window AND rolling log purge
|
|
# 12 = entries older than 12hrs purge automatically, fresh window starts
|
|
SYS_WATCHDOG_REBOOT_WINDOW_HRS=12
|
|
|
|
# ━━━ Thresholds ━━━
|
|
# Set these at "I am about to become unstable" levels — not just "things are a bit tight"
|
|
SYS_WATCHDOG_ROOTFS_PCT=95 # rootfs % — at 95% something is seriously wrong
|
|
SYS_WATCHDOG_LOG_PCT=95 # /var/log % — log spam filling disk
|
|
SYS_WATCHDOG_MEM_GB=4 # free RAM GB — 4GB on 128GB system is critical
|
|
SYS_WATCHDOG_ARC_PINNED_PCT=98 # ZFS ARC % of max before attempting reclaim
|
|
SYS_WATCHDOG_ARC_RELEASE_PCT=95 # ZFS ARC % after reclaim that still triggers reboot
|
|
SYS_WATCHDOG_LOAD_MULTIPLIER=3 # strike if load avg > cores x this value
|
|
SYS_WATCHDOG_ZOMBIE_LIMIT=50 # zombie process count before strike
|
|
SYS_WATCHDOG_CPU_TEMP_MAX=95 # degrees C — adjust for your CPU tjmax
|
|
|
|
# ━━━ Check Toggles ━━━
|
|
# true = run this check / false = skip entirely
|
|
# Disable checks that are not relevant to your hardware or cause false positives
|
|
SYS_WATCHDOG_CHECK_ROOTFS=true
|
|
SYS_WATCHDOG_CHECK_LOG=true
|
|
SYS_WATCHDOG_CHECK_RAM=true
|
|
SYS_WATCHDOG_CHECK_ARC=true
|
|
SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
|
SYS_WATCHDOG_CHECK_LOAD=false
|
|
SYS_WATCHDOG_CHECK_ZOMBIES=true
|
|
SYS_WATCHDOG_CHECK_CONTAINERS=true
|
|
SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
|
|
|
|
# ━━━ Abort Toggles ━━━
|
|
# true = abort reboot if condition is active / false = reboot anyway
|
|
# Default true = conservative — set false only when "reboot no matter what" is wanted
|
|
# Goal: graceful reboot before crash is always better than hard crash mid-operation
|
|
SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # unhealthy pool + reboot = potential data loss
|
|
SYS_WATCHDOG_ABORT_ON_PARITY=true # aborting parity better than crashing mid-check
|
|
SYS_WATCHDOG_ABORT_ON_MOVER=true # aborting move better than crashing mid-move
|
|
|
|
# ==============================================================================================
|
|
# ──────────────────────── End Of User Variables ───────────────────────────────────────────────
|
|
# ============================================================================================== |