From 55f8842dbd49f90fc683dfbae0829dbaa68f1b62 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sun, 14 Jun 2026 11:07:18 -0400 Subject: [PATCH] Move remainder container updates from weekly to monthly Daily updates daily containers, weekly updates its sync-window containers, monthly now handles everything else via MONTHLY_MAINTENANCE_SCRIPTS. --- Configurations/master.conf | 1452 ++++++++++++++++++++++ Docker_Essentials/docker_update.sh | 8 +- Orchestrators/weekly_sync_maintenance.sh | 26 - Plugin/unraid/user_script_plug-in.sh | 11 +- 4 files changed, 1461 insertions(+), 36 deletions(-) create mode 100644 Configurations/master.conf diff --git a/Configurations/master.conf b/Configurations/master.conf new file mode 100644 index 0000000..20f15b9 --- /dev/null +++ b/Configurations/master.conf @@ -0,0 +1,1452 @@ +#!/bin/bash +# ============================================================================================== +# ================================= MASTER CONFIGURATION ======================================= +# ============================================================================================== +# Shared configuration for the unRAID script ecosystem. +# Contains all settings that apply to every server — thresholds, toggles, profiles, job lists. +# +# ── HOW THE THREE-FILE SYSTEM WORKS ────────────────────────────────────────────────────────── +# Scripts source all three files at startup: +# source master.conf ← shared config (this file) +# source host1.conf ← HOST1 credentials, shares, container lists +# source host2.conf ← HOST2 credentials, shares, container lists +# +# Sparse checkout (git) ensures each server only pulls its own host*.conf. +# HOST2 never sees HOST1 credentials. HOST1 never sees HOST2 credentials. +# +# What belongs here: thresholds, toggles, intervals, profiles, job lists +# What belongs in HOST*: hostnames, SSH keys, API keys, passwords, share paths, +# container names, failover lists, watchdog containers +# +# ── HOW THIS FILE WORKS ─────────────────────────────────────────────────────────────────────── +# Every script sources all three conf files and common.sh at startup. +# Change a value here and it affects all scripts that use it — no hunting through files. +# To disable something: comment it out with # rather than deleting it. +# To add a new rsync profile: add a key to each PROFILE_* array in the RSYNC section. +# To add or remove orchestrator jobs: edit the arrays in the ORCHESTRATORS section. +# +# ── INDEX ───────────────────────────────────────────────────────────────────────────────────── +# +# Section Description +# ─────────────────────────────────────────────────────────────────────────────────────────── +# SHARED HOST CONFIGURATION DATA_DIR and shared connection settings +# PARTNERSHIP Mirror relationship lifecycle — onboard/offboard/transfer +# LOGGING Enable or disable verbose logging +# NOTIFICATIONS unRAID native and Discord webhook settings +# GIT / REPO Gitea repository and SSH settings +# +# ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────── +# ARRAY START Scripts launched at array start (array_started.sh) +# ARRAY STOP Scripts run at planned shutdown (array_stopping.sh) +# WATCHDOG ORCHESTRATOR Per-minute watchdog runner (watchdog_orchestrator.sh) +# SYSTEM WATCHDOG Sub-scripts called by watchdog_orchestrator.sh +# CRITICAL SYNC MAINTENANCE 30-minute jobs + sync shares + partnership check (critical_sync_maintenance.sh) +# INTERMEDIATE SYNC MAINTENANCE Arr sync + optional mid-day rsync (intermediate_sync_maintenance.sh) +# DAILY SYNC MAINTENANCE Job list + media shares (daily_sync_maintenance.sh) +# WEEKLY SYNC MAINTENANCE Job list + sync shares + update toggles (weekly_sync_maintenance.sh) +# MONTHLY MAINTENANCE Uptime-triggered heavy tasks — ZFS scrub, SMART tests (monthly_maintenance.sh) +# +# ── RSYNC ────────────────────────────────────────────────────────────────────────────────── +# RSYNC ENABLE/DISABLE Two-tier toggle system — global gate + per-orchestrator +# RSYNC DEFAULTS Global fallback rsync options and limits +# REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort +# RSYNC PROFILE SYSTEM Per-profile overrides for appdata syncs +# +# ── FALLBACK ─────────────────────────────────────────────────────────────────────────────── +# FALLBACK Mutual container failover shared settings +# FALLBACK TEST Simulated outage settings for fallback_test.sh +# +# ── DOCKER ESSENTIALS ────────────────────────────────────────────────────────────────────── +# DOWNLOADERS RESET Retention and thresholds for slskd, SABnzbd, qBittorrent +# DOCKER DAILY RESTART Containers restarted daily +# DOCKER WEEKLY RESTART Containers restarted weekly +# DOCKER WATCHDOG Thresholds and toggles for container monitoring +# DOCKER NETWORK CONNECT Networks to ensure + containers to connect +# +# ── UNRAID ESSENTIALS ────────────────────────────────────────────────────────────────────── +# INOTIFY TUNING inotify limits — raised at array start by inotify_tuning.sh +# SYSTEM TUNING MONITOR Tracks inotify + php-fpm usage over time +# REBOOT User warning delay before scheduled reboot +# MOVER Mover stop timeout +# SYSLOG FILTER Docker veth noise filter file path +# PHP-FPM PHP-FPM max children config +# CLEAR LOGS System log file paths +# WEBGUI WATCHDOG WebGUI nginx + emhttp monitoring and restart +# +# ── MEDIA ────────────────────────────────────────────────────────────────────────────────── +# MEDIA PERMISSIONS Share list, mode and owner for permissions script +# MEDIA CLEANER Anime and media folder lists and file patterns +# ARR CLEANUP Lidarr, Sonarr, Radarr orphan file cleanup settings +# ARR FAILED/STALLED RECOVERY Auto blocklist + re-search settings +# +# ── TRANSCODES ───────────────────────────────────────────────────────────────────────────── +# TRANSCODE MANAGER Ramdisk and SSD fallback transcode management +# TRANSCODE SERVER ARRAY Multi-server session monitoring (Emby, Jellyfin, Plex) +# +# ── MONITORS ─────────────────────────────────────────────────────────────────────────────── +# CERTIFICATE MONITOR SSL certificate expiry monitoring +# BACKUP VERIFY Random sample checksum verification against remote +# SMART HEALTH Drive SMART attribute monitoring +# ZFS MEMORY SNAPSHOT Weekly ZFS health and memory diagnostic report +# BANDWIDTH MONITOR Daily rsync transfer logging and weekly summary +# HEALTH DIGEST Aggregated system health digest — always/smart/weekly +# EMBY SESSION REPORT Weekly Emby usage statistics via API +# +# ── RESOURCE MANAGER ─────────────────────────────────────────────────────────────────────── +# RESOURCE MANAGER Containers paused/stopped under RAM pressure +# +# ── SYSTEM WATCHDOG ──────────────────────────────────────────────────────────────────────── +# SYSTEM WATCHDOG Continuous system health monitoring — last line of defense +# +# ============================================================================================== + +# ============================================================================================== +# ── HOST IDENTITIES ─────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Hostnames for every server in the ecosystem — not credentials, safe for all machines. +# Must match the exact unRAID hostname AND Tailscale device name (case sensitive). +# detect_hosts() in common.sh matches the local hostname against these to set MY_ID / REMOTE_ID. +# Tailscale IP resolution uses these names — no hardcoded IPs needed. +# To add a new server: add HOST3="unRAID-NewServer" here + create host3.conf. + HOST1="unRAID-Gmer4Lfe" + HOST2="unRAID-Jayred365" + +# ============================================================================================== +# ── SHARED HOST CONFIGURATION ───────────────────────────────────────────────────────────────── +# ============================================================================================== +# Requirement: internal NVMe/SSD boot drive. Everything lives in the plugin folder on flash — +# scripts, state, and data are all available before the array mounts. +# Both directories are created automatically if they don't exist. +# +# DATA_DIR — historical logs, statistics, discovery histories, blocklists +# STATE_DIR — runtime state files for all scripts (watchdogs, fallback, transcode, etc.) +# Requirement: ALL state files MUST use $STATE_DIR. No /tmp, no /boot/config root. + DATA_DIR="${SCRIPTS_DIR}/data" + STATE_DIR="${SCRIPTS_DIR}/State_Files" + +# ── Version Parity ── +# Controls behaviour when local and remote unRAID versions differ. +# Major version mismatch always aborts regardless of this setting. +# "warn" — log warning and continue (safe for minor/patch differences) +# "abort" — refuse to continue (strict — ensures both sides always match) + UNRAID_VERSION_MISMATCH_ACTION="warn" + +# ── Arr Sync ── +# arr_sync.sh syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally. +# Runs before rsync — all nodes agree on tracked library before files are transferred. +# Remote API keys are read live from each node's config.xml via SSH — never stored here. + CONF_SYNC_ENABLED=false # HOST2 rebuild in progress — re-enable when HOST2 is back online + + ARR_SYNC_ENABLED=true + ARR_SYNC_BLOCKLIST="${DATA_DIR}/arr_sync_blocklist.tsv" + ARR_SYNC_CONNECT_TIMEOUT=10 # seconds — SSH connect timeout per node + ARR_SYNC_API_TIMEOUT=60 # seconds — curl timeout for library fetches + DOCKER_APPDATA_BASE="${DOCKER_APPDATA_BASE:-}" # set by platform adapter; override in host*.conf if needed + ARR_SYNC_LIDARR_PORT=8686 + ARR_SYNC_SONARR_PORT=8989 + ARR_SYNC_RADARR_PORT=7878 + +# ── Remote Docker Daemon ── +# Strike system for remote Docker daemon health checks. +# Scripts that issue remote container commands check the remote daemon first. +# Below limit → skip operation this run | At limit → notify critical + exit + REMOTE_DOCKER_STRIKE_LIMIT=3 # consecutive failures before critical notify + +# ============================================================================================== +# ── PARTNERSHIP ─────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Manages the relationship lifecycle between two unRAID servers. +# HOST1 is always the owner (source of truth) — HOST2 is always the mirror. +# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a --transfer operation. +# All identity vars (hostnames, SSH keys) live in host*.conf. +# Hostnames already match Tailscale device names — IP resolution is automatic. +# +# State files in STATE_DIR — survives reboots, array must be up (scripts run after array start): +# STATE_DIR/partnership_HOST1.db ← HOST1 writes only +# STATE_DIR/partnership_HOST2.db ← HOST2 writes only +# STATE_DIR/partnership_blocklist.db ← shared blocklist +# STATE_DIR/varaverk_setup.db ← onboarding progress state +# Propagated via SSH — no rsync needed +# +# critical_sync_maintenance.sh runs --check every 30min: +# Reads both state files via SSH +# Detects offboard requests → finalises from owner side +# Increments offline counter → auto-offboards after threshold +# Silent when healthy ✅ +# +# See README-Partnership.md for full lifecycle documentation. + + PARTNERSHIP_ENABLED=false + PARTNERSHIP_OWNER_HOST="HOST1" # "HOST1" or "HOST2" — flips on --transfer + +# Auth containers reconfigured on onboard/offboard — defined per host in host*.conf. +# Format: "ContainerName|WebUIPort" +# HOST1_PARTNERSHIP_AUTH_WEBUIS / HOST2_PARTNERSHIP_AUTH_WEBUIS +# On onboard → WebUI pointed at owner's Tailscale IP +# On offboard → WebUI pointed back at localhost + +# Paths to collect during the grace window after offboard — defined per host in host*.conf. +# HOST1_PARTNERSHIP_MIRROR_BACKUPS / HOST2_PARTNERSHIP_MIRROR_BACKUPS +# Notified on offboard — no auto-deletion, manual collection. + +# Timing — single var controls both Tailscale removal and backup access expiry. +# Both expire at the same time — keeping backups accessible beyond Tailscale removal is pointless. + PARTNERSHIP_GRACE_HOURS=6 # hours after offboard before Tailscale removal + # backup access expires at the same time + PARTNERSHIP_OFFLINE_THRESHOLD=30 # days either server unreachable before auto-offboard + # works both directions independently + +# Partnership and setup state files — all in STATE_DIR per the project requirement. +# Scripts use these variables; do not hardcode /boot/config paths in scripts. + PARTNERSHIP_BLOCKLIST_FILE="$STATE_DIR/partnership_blocklist.db" + VARAVERK_SETUP_FILE="$STATE_DIR/varaverk_setup.db" + +# Tailscale removal on offboard. + PARTNERSHIP_REMOVE_TAILSCALE=true # remove mirror from Tailscale tailnet on offboard + # false = skip removal (manual or testing) + +# Tailscale API — required when PARTNERSHIP_REMOVE_TAILSCALE=true. +# Stays in shared conf — only owner uses it, and owner is always running this script. +# API key: https://login.tailscale.com/admin/settings/keys → Devices write scope + TAILSCALE_API_KEY="" # tskey-api-... + TAILSCALE_TAILNET="" # your tailnet name (e.g. yourname.github) + +# Transfer safety. + PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership" + PARTNERSHIP_TRANSFER_STRIKES=3 # consecutive health checks required + PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20 # max health check attempts before giving up + +# Onboard settings. + PARTNERSHIP_ONBOARD_VERIFY=true # verify WebUI reachable after reconfiguration + PARTNERSHIP_ONBOARD_NOTIFY=true # notify both servers on completion + PARTNERSHIP_SYNC_INTERVAL=15 # minutes — informational, actual schedule in cron + +# SSH key lifecycle — managed by Partnership/ssh_setup.sh. +# Key named after this server: hostname lowercased, unraid- prefix stripped. +# unRAID-Gmer4Lfe → /root/.ssh/gmer4lfe_rsync_automation +# Run Partnership/partnership_onboard.sh to generate, copy, and update conf automatically. + SSH_MAX_STRIKES=5 # consecutive SSH auth failures before critical notify + SSH_STRIKE_RESET_HRS=24 # hours since last failure before strike counter resets + +# FolderView3 plugin integration — chodeus/VladoPortos/scolcipitato. +# Creates a "{Partner}-Failover" folder on onboard, removes + cleans containers on offboard. +# Folder name derived from remote hostname (strip unraid- prefix case-insensitively). +# Plugin config: /boot/config/plugins/folder.view3/docker.json + PARTNERSHIP_FOLDERVIEW3=true # create/remove FolderView3 folder on onboard/offboard + PARTNERSHIP_FOLDERVIEW3_URL="" # CA plugin URL — leave empty to skip auto-install + +# ============================================================================================== +# ── LOGGING ─────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# Controls verbose [LOG] output across all scripts. +# true = show detailed [LOG] lines — useful for debugging or first-time setup +# false = show only user-facing output — cleaner for scheduled runs + ENABLE_LOGGING=false + +# ============================================================================================== +# ── NOTIFICATIONS ───────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# unRAID native notification system — integrates with the bell icon in the WebGUI. +# normal = job completed successfully / warning = something failed or needs attention + NOTIFY_UNRAID=true + +# Discord webhook URL — defined per host in host*.conf. +# HOST1_DISCORD_WEBHOOK / HOST2_DISCORD_WEBHOOK +# Allows different webhooks per server, or only one server notifying. + +# ============================================================================================== +# ── GIT / REPO ──────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# Gitea self-hosted repository — used by git_pull_execute.sh. +# Detects Gitea container location at runtime — works through failover automatically. +# Falls back to GITEA_DOMAIN if local and Tailscale both fail. + GITEA_CONTAINER="Gitea" + GITEA_REPO_PATH="FailedProxy/Varaverk.git" + GITEA_DOMAIN="" # e.g. git.yourdomain.com — requires NPM + DNS + TARGET_DIR="/boot/config/plugins/varaverk" + GITEA_SSH_KEY="/root/.ssh/unraid_gitea" + SSH_PORT=221 # Gitea SSH port (default 22, Gitea often uses 221/222) + GITEA_HTTP_PORT=3000 # Gitea web/API port — used by gitea_ssh_setup.sh + +# ============================================================================================== +# ── ORCHESTRATORS ───────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# All orchestrator job lists live here — edit arrays to add/remove scripts. +# No changes to orchestrator scripts needed when adding or removing jobs. +# Sections ordered by run frequency: array events first, then shortest interval to longest. + +# ━━━ Array Start ━━━ +# Scripts launched by array_started.sh when the array comes online. +# Launched in order — each as a background process. +# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally. +# Continuous scripts (failover) run until array stops. +# Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via +# watchdog_orchestrator.sh — NOT launched here. + ARRAY_START_SCRIPTS=( + "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh" # re-register Varaverk API key — registry is ephemeral + "System_Essentials/conf_sync.sh" # pull partner confs + push own conf into /tmp/.vv/ RAM cache + "Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts + "System_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill + "Plugin/unraid/System_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning + "System_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted + "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers + "Tools/claude_startup.sh" # persist Claude data + binary to appdata; re-symlink on boot + "Fallback/fallback.sh" # mutual failover — continuous + ) + +# ━━━ Array Stop ━━━ +# Scripts run by array_stopping.sh for a planned shutdown — stops everything cleanly in order. +# Run sequentially (foreground) — each must complete before the next starts. +# Order matters: user scripts first (prevents new ops), then data movement, then containers. + ARRAY_STOP_SCRIPTS=( + "Plugin/unraid/System_Essentials/user_scripts_stop.sh" # stop background scripts before they start new ops + "Fallback/fallback.sh --stop" # gracefully stop fallback (not caught by user_scripts_stop) + "System_Essentials/rsync_stop.sh --rsync-only" # kill rsync; skip container recovery (handled below) + "Plugin/unraid/System_Essentials/mover_stop.sh" # stop mover after rsync (they conflict on same files) + "Docker_Essentials/docker_container_stop.sh" # stop all containers last + ) + +# ━━━ Watchdog Orchestrator ━━━ +# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle. +# Schedule: * * * * * (every minute) +# NOT in ARRAY_START_SCRIPTS — has its own cron entry. +# Order matters — resource first (frees pressure), docker second (heals with freed resources), +# system third (storage + webgui component health), stability last (last line of defense). + WATCHDOG_ORCHESTRATOR_SCRIPTS=( + "Watchdogs/resource_watchdog.sh" # reduce system pressure before healing attempts + "Watchdogs/docker_watchdog.sh" # heal containers with freed resources + "Watchdogs/system_watchdog.sh" # system component health — storage + webgui + "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh" # re-register API key if registry lost it + "Watchdogs/stability_watchdog.sh" # reboot if all else fails — last line of defense + ) + +# ━━━ System Watchdog ━━━ +# system_watchdog.sh runs SYSTEM_WATCHDOG_SCRIPTS sequentially each cycle. +# Called by watchdog_orchestrator.sh — not scheduled directly. + SYSTEM_WATCHDOG_SCRIPTS=( + "Watchdogs/System/storage_watchdog.sh" # pool growth + runaway log detection + "Plugin/unraid/Watchdogs/System/webgui_watchdog.sh" # WebGUI availability — nginx → php-fpm → emhttp + "Watchdogs/System/network_watchdog.sh" # internet, DDNS, Tailscale, NPM proxy + ) + +# ━━━ Critical Sync Maintenance ━━━ +# critical_sync_maintenance.sh runs every 30 minutes. +# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check +# partnership --check always runs last regardless of rsync gate. +# Comment out entries to disable without removing. + CRITICAL_MAINTENANCE_SCRIPTS=( + "Docker_Essentials/downloaders_reset.sh" # clear stuck download states every 30min + "Media/play_state_sync.sh" # sync watched/resume state across Emby + Jellyfin + ) + +# Shares synced every 30 minutes — defined per host in host*.conf. +# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES +# Format: "/path/to/share" or "/path/to/share|profile-name" +# Order matters — Critical-Data first (auth stack), then Emby dirty sync. + +# ━━━ Intermediate Sync Maintenance ━━━ +# intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch, +# and optional mid-day rsync for any shares that need sub-daily propagation. +# Schedule: 0 */4 * * * + # INTERMEDIATE_SYNC_SHARES is host-specific — configure HOST*_INTERMEDIATE_SYNC_SHARES in host*.conf. + + INTERMEDIATE_MAINTENANCE_SCRIPTS=( + "Media/arrs_failed_stalled_recovery.sh" # blocklist + re-search failed/stalled arr queue items + ) + # arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here. + # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above). + +# ━━━ Daily Sync Maintenance ━━━ +# daily_sync_maintenance.sh runs media share sync first, then iterates +# DAILY_MAINTENANCE_SCRIPTS for all jobs. +# Schedule: 0 1 * * * (1am daily) + DAILY_MAINTENANCE_SCRIPTS=( + "git_pull_execute.sh" # pull latest scripts — always runs first + "Media/media_shares_permissions.sh" # apply permissions — runs before cleaners + "Media/media_cleaner.sh anime" # remove junk from anime shares + "Media/media_cleaner.sh media" # remove junk from media shares + #"Media/lidarr_cleanup.sh" # remove orphaned music files — enable when ready + #"Media/sonarr_cleanup.sh" # remove orphaned TV files — enable when ready + "Media/radarr_cleanup.sh" # remove orphaned movie files + "Media/lidarr_missing_art.sh" # fetch missing album/artist artwork (HOST1 only — self-guards) + "Media/radarr_tmdb_removed.sh" # remove movies dropped from TMDb + "Media/sonarr_tvdb_removed.sh" # remove series dropped from TVDB + "Docker_Essentials/docker_update.sh" # pull container image updates before restart + "Docker_Essentials/docker_daily_restart.sh" # daily container restarts — runs last + ) + +# Pull latest images for DAILY_RESTART_CONTAINERS before the daily restart. +# Containers keep running during pull — no extra downtime. +# Set false to skip updates while still running the daily restart. + DAILY_CONTAINER_UPDATES=true + +# Media shares synced daily by daily_sync_maintenance.sh. +# Defined per-host in host*.conf — HOST1_DAILY_SYNC_SHARES and HOST2_DAILY_SYNC_SHARES. +# Mesh model: every node pushes every media share. rsync has no --delete so pushes are additive. +# arr_sync (union) ensures all arr libraries converge first. arr_cleanup removes true orphans. +# Any node can download content to any share — it propagates to all nodes on the next cycle. +# Adding HOST3: list every media share in HOST3_DAILY_SYNC_SHARES. No ownership to track. +# These shares use DEFAULT_RSYNC_OPTS — no profile entry needed. +# For shares needing custom options or container stops — create a profile in RSYNC section. + +# Personal encrypted shares defined per-host in host*.conf. +# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content. +# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting. + +# ━━━ Weekly Sync Maintenance ━━━ +# weekly_sync_maintenance.sh stops containers both sides → pulls updates → +# syncs WEEKLY_SYNC_SHARES → restarts → then iterates WEEKLY_MAINTENANCE_SCRIPTS. +# Schedule: 30 2 * * 0 (Sunday 2:30am) + WEEKLY_MAINTENANCE_SCRIPTS=( + "Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync + "System_Essentials/clear_logs.sh" # purge aged logs — Sunday only, low priority + "Media/playback_aware_lidarr_discovery.sh" # behavior-driven music discovery using weekly Emby playback history + "Media/playback_aware_radarr_discovery.sh" # behavior-driven movie discovery using TMDB recommendations + "Media/playback_aware_sonarr_discovery.sh" # behavior-driven TV discovery using TMDB recommendations + ) + +# Pull updates for all running containers NOT in daily/weekly managed lists. +# Ensures every deployed container receives at least one image pull per month. +# Set false to skip. + MONTHLY_REMAINING_UPDATES=true + +# Shares synced during the weekly maintenance window — defined per host in host*.conf. +# HOST1_WEEKLY_SYNC_SHARES / HOST2_WEEKLY_SYNC_SHARES +# Containers stopped both sides before sync — full clean state guaranteed. +# Profiles drive container stops, excludes, and options — configure in RSYNC section. +# Order matters — Emby first (larger), then Critical-Data (auth stack). + +# Container update toggles for the weekly sync window. +# Containers already stopped for sync — updates pull at no extra downtime. +# Both false → sync only, no updates. + WEEKLY_SYNC_UPDATES=true # pull container updates locally during weekly window + WEEKLY_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH + +# ━━━ Monthly Maintenance ━━━ +# monthly_maintenance.sh fires only when BOTH gates pass: +# 1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days +# 2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run) +# Cron: 0 3 * * * (daily 3am check — script self-gates, calling daily is safe) +# NOT in WATCHDOG_ORCHESTRATOR_SCRIPTS — has its own cron entry. +# Add scripts that require a long-stable settled system — scrubs, extended drive tests. +# State file on /boot/config — survives reboots (interval gate independent of uptime gate). + MONTHLY_MAINTENANCE_SCRIPTS=( + "Docker_Essentials/docker_update.sh --remainder" # update all containers not in daily/weekly managed lists + "Tools/zfs_pool_scrub.sh" # ZFS pool integrity scrub — all pools, skip ZFS_REPORT_IGNORE_POOLS + "Tools/smart_long_test.sh" # SMART extended self-test — all drives, sequential + ) + + MONTHLY_UPTIME_THRESHOLD_DAYS=30 # minimum uptime in days before maintenance fires + MONTHLY_RUN_INTERVAL_DAYS=30 # minimum days since last run before running again + MONTHLY_LAST_RUN_FILE="$STATE_DIR/monthly_maintenance_last_run.db" + +# ━━━ Sunday Morning Coffee Report ━━━ +# Orchestrator that runs all Sunday monitor scripts in sequence. +# Schedule: 0 7 * * 0 (Sunday 7am — after weekly_sync_maintenance.sh finishes at ~3am) +# Each script runs independently and notifies on its own findings. + COFFEE_REPORT_SCRIPTS=( + "Monitors/zfs_memory_snapshot.sh" # ZFS pool health + ARC + Docker memory snapshot + "Monitors/smart_health.sh" # drive SMART attributes — reallocated, pending, temp + "Monitors/cert_monitor.sh" # SSL certificate expiry for all configured domains + "Monitors/backup_verify.sh" # rsync mirror integrity via independent MD5 checksums + "Monitors/bandwidth_monitor.sh" # weekly rsync transfer totals and per-share breakdown + "Monitors/emby_session_report.sh" # Emby usage — streams, users, library, transcode ratio + "Monitors/weekly_health_digest.sh" # aggregated digest — watchdogs, fallback, skip list + ) + +# ============================================================================================== +# ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Rsync Enable/Disable ━━━ +# Two-tier toggle system — Tier 1 overrides Tier 2. +# +# Tier 1 — Global gate: +# RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions +# Use when: remote completely offline, major maintenance, disaster recovery +# +# Tier 2 — Per-orchestrator (only applies when Tier 1 is true): +# Fine grained control — disable specific orchestrators while keeping others +# Use when: rebuilding secondary, testing, per-window bandwidth management +# +# Example — HOST2 data rebuild: +# RSYNC_ENABLED=true ← rsync works, individual scripts run fine +# CRITICAL_RSYNC_ENABLED=true ← 30min auth stack sync still runs +# INTERMEDIATE_RSYNC_ENABLED=false ← skip 4h arr/mid-day rsync during rebuild +# DAILY_RSYNC_ENABLED=false ← skip daily HDD syncs during rebuild +# WEEKLY_RSYNC_ENABLED=true ← Emby + Critical-Data still sync (NVMe) +# FALLBACK_RSYNC_ENABLED=true ← handback writeback still works +# → Run individual: bash Rsync/rsync.sh /mnt/user/Movies +# → When ready: INTERMEDIATE_RSYNC_ENABLED=true DAILY_RSYNC_ENABLED=true + RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below + CRITICAL_RSYNC_ENABLED=true # Tier 2 — critical_sync_maintenance.sh rsync section + INTERMEDIATE_RSYNC_ENABLED=true # Tier 2 — intermediate_sync_maintenance.sh rsync section + DAILY_RSYNC_ENABLED=true # Tier 2 — daily_sync_maintenance.sh rsync section + WEEKLY_RSYNC_ENABLED=true # Tier 2 — weekly_sync_maintenance.sh rsync section + FALLBACK_RSYNC_ENABLED=true # Tier 2 — fallback.sh writeback jobs on handback + +# ━━━ Rsync Defaults ━━━ +# Global fallback values used when no profile match is found. +# Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed. +# Appdata shares match profiles by directory basename (lowercased). + BW_LIMIT=12500 # KB/s — 12500 ≈ 100Mbit + RETRY_COUNT=3 # retry attempts before giving up + SLEEP=300 # seconds between retry attempts + CRITICAL_CONTAINER_NAMES=() # containers stopped on REMOTE before rsync — profiles override + DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override + CONTAINER_DELAY=5 # seconds before starting delayed containers + RESTART_VERIFY_WAIT=3 # seconds to wait after restart before checking container is running + EXCLUDE_DIRS=() # directories excluded from transfer — profiles override + +# --inplace writes directly to destination — delta against existing file, better for large media +# --partial keep partial file on interrupted transfer so next run resumes, not re-transfers +# --timeout kill stalled transfers instead of hanging indefinitely +# --numeric-ids use UIDs/GIDs numerically — prevents ownership mismatches between servers +# --delete intentionally omitted — arr_cleanup.sh enforces media truth post-rsync. +# Media shares use this default; rsync spreads files only, never removes them. +# Note: --no-whole-file removed — redundant over SSH (delta transfer is already the default). +# Note: arr_cleanup only catches true orphans under the union model — intentional removals +# require arr_sync.sh --blocklist-add first, then manual file deletion. + DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --inplace --partial --timeout=60 --numeric-ids) + +# ━━━ Remote Health Checks ━━━ +# Pre-flight — aborts if remote rootfs (/) usage is at or above this percentage. +# When remote array is down, rsync writes land on rootfs and fill it rapidly. + ROOTFS_WARN=75 + +# ━━━ Rsync Profile System ━━━ +# Profiles allow per-share rsync behaviour without touching script logic. +# Profile key matched by basename of directory passed to rsync.sh (lowercased). +# Override with --profile=name flag. +# +# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit DEFAULT_RSYNC_OPTS. +# List ALL desired options explicitly when defining a profile. +# +# Current profiles: +# arrs_stack — arr databases — lower bandwidth, containers stopped for consistency +# critical-data — auth stack — full stop both sides, Authelia delayed start +# called by weekly_sync_maintenance.sh — full clean sync weekly +# critical-fallback — dirty sync — auth stays running both sides, WAL excluded +# called by critical_sync_maintenance.sh every 30min +# host1-appdata — HOST1 server-specific appdata — defined in host1.conf +# host2-appdata — HOST2 server-specific appdata — defined in host2.conf +# important-data — NextCloud + Postgres — NextCloud delayed start after Postgres +# emby — weekly clean sync — both Emby stopped, full mirror +# called by weekly_sync_maintenance.sh only — do NOT schedule separately +# emby-fallback — dirty sync — Emby stays running, WAL excluded +# called by critical_sync_maintenance.sh every 30min + + declare -A PROFILE_RSYNC_OPTS=( + [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" + [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete" + [critical-fallback]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" + [important-data]="-av --human-readable --bwlimit=$BW_LIMIT" + [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" + [emby-fallback]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" + ) + +# Per-profile bandwidth limits in KB/s + declare -A PROFILE_BW_LIMIT=( + [arrs_stack]=5000 # lower — runs alongside other syncs + [critical-data]=9500 # high — small dataset, sync fast + [critical-fallback]=9500 # high — small dataset, sync fast + [important-data]=9500 # high — database sync + [emby]=8000 # medium — large full mirror + [emby-fallback]=9500 # high — small critical dataset + ) + +# Retry attempts per profile + declare -A PROFILE_RETRY_COUNT=( + [arrs_stack]=3 + [critical-data]=3 + [critical-fallback]=3 + [important-data]=3 + [emby]=3 + [emby-fallback]=3 + ) + +# Seconds between retry attempts + declare -A PROFILE_SLEEP=( + [arrs_stack]=300 + [critical-data]=300 + [critical-fallback]=120 # shorter — frequent dirty sync, retry faster + [important-data]=300 + [emby]=300 + [emby-fallback]=120 # shorter — frequent dirty sync, retry faster + ) + +# Containers stopped on BOTH LOCAL and REMOTE before rsync. +# Local stops first — flushes databases cleanly. Remote stops next — prevents writes. +# Only running containers get restarted — stopped containers stay stopped. +# SPACE-SEPARATED STRINGS — converted to array at runtime + declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( + [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" + [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary" + [critical-fallback]="" # dirty sync — auth stays running both sides + [important-data]="Postgres-NextCloud NextCloud" + [emby]="Emby" + [emby-fallback]="" # dirty sync — Emby stays running both sides + ) + +# Containers needing a delay after rsync before starting. +# SPACE-SEPARATED STRINGS — converted to array at runtime + declare -A PROFILE_DELAYED_CONTAINERS=( + [arrs_stack]="" + [critical-data]="Authelia Authelia-Secondary" # wait for Mariadb + Redis + [critical-fallback]="" + [important-data]="NextCloud" # wait for Postgres + [emby]="" + [emby-fallback]="" + ) + +# Seconds before starting delayed containers + declare -A PROFILE_CONTAINER_DELAY=( + [arrs_stack]=5 + [critical-data]=15 # Mariadb + Redis need time to accept connections + [critical-fallback]=5 + [important-data]=10 # Postgres needs time before NextCloud + [emby]=5 + [emby-fallback]=5 + ) + +# Directories excluded from rsync per profile. +# SPACE-SEPARATED STRINGS — converted to array at runtime + declare -A PROFILE_EXCLUDE_DIRS=( + [arrs_stack]="logs *.tmp" + [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt" + [critical-fallback]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt *.db-wal *.db-shm" + [important-data]="logs *.tmp" + [emby]="logs transcodes cache crash*" + [emby-fallback]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root" + ) + +# Remote restart after dirty sync — restart these on remote IF they were running before sync. +# Same logic as stop/start — was stopped = stays stopped, was running = gets restarted. +# Used by dirty sync profiles (critical-fallback, emby-fallback) so remote picks up changes. +# SPACE-SEPARATED STRINGS — converted to array at runtime + declare -A PROFILE_REMOTE_RESTART_CONTAINERS=( + [critical-fallback]="NginxProxyManager Authelia Authelia-Secondary Lldap-Gmer4Lfe Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary" + [emby-fallback]="Emby" + ) + +# Note: disk check is auto-detected from disks.ini — no PROFILE_SKIP_DISK_CHECK needed. +# check_remote_disks() reads fsType per disk and handles XFS, ZFS, and cache pools automatically. + +# ============================================================================================== +# ── FALLBACK ────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Mutual container failover between two unRAID servers. +# Each server runs Fallback/fallback.sh independently via array_started.sh. +# All decisions based on two pings: remote reachable + internet reachable. +# +# States: NORMAL | FALLBACK | NO_INTERNET | DARK +# +# DDNS rules — absolute: +# Internet loss → stop own DDNS immediately +# Failover → start remote DDNS first (Tier 1) +# Handback → stop remote DDNS → rsync → start containers → start local DDNS last +# +# Per-host container lists and tier delays live in host*.conf. +# Shared settings (intervals, state file, thresholds) live here. + + EXTERNAL_IP="8.8.8.8" + FALLBACK_CHECK_INTERVAL=30 # seconds between fallback state checks + FALLBACK_HANDBACK_STRIKES=3 # consecutive healthy checks before initiating handback (3×30s = 90s) + FALLBACK_STATE_FILE="$STATE_DIR/fallback_state.db" + FALLBACK_ENABLED=true # HOST2 back online + # false = suppresses "not running" warnings in status scripts + FALLBACK_PARTNERSHIP_REQUIRED=true # gate fallback on an active partnership + # false = standalone mode, no partnership dependency + FALLBACK_PARTNERSHIP_SUSPEND_AFTER=120 # minutes without active partnership before suspending + # 0 = suspend immediately when partnership goes inactive + +# ━━━ Failover Test ━━━ +# Controlled simulation of a failover event — run manually via fallback_test.sh. + FALLBACK_TEST_BLOCK_WAIT=150 # seconds to wait after blocking connectivity + FALLBACK_TEST_HANDBACK_WAIT=360 # seconds to wait before initiating handback + +# ============================================================================================== +# ── DOCKER ESSENTIALS ───────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Downloaders Reset ━━━ +# Runs every 30 minutes via CRITICAL_MAINTENANCE_SCRIPTS. +# Clears stuck states, purges old history, prepares each download client for a clean cycle. +# Per-host URLs and API keys live in host*.conf. + DOWNLOADER_RETENTION_DAYS=7 # days — purge history older than this + +# qBittorrent failsafe — removes torrents older than threshold regardless of ratio +# deleteFiles=false — removes from qBit but leaves files on disk (arr manages files) + QBIT_FAILSAFE_MIN_DAYS=180 # days — minimum age before failsafe deletion + QBIT_FAILSAFE_MIN_RATIO=0 # 0 = age only, no ratio requirement + +# ━━━ Docker Daily Restart ━━━ +# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS. +# Per-host lists live in host*.conf: +# HOST1_DAILY_RESTART_CONTAINERS +# HOST2_DAILY_RESTART_CONTAINERS +# detect_hosts() sets DAILY_RESTART_CONTAINERS to the correct host array at runtime. + +# ━━━ Docker Weekly Restart ━━━ +# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am). +# Containers already stopped for weekly sync — restart adds zero extra downtime. +# Per-host lists live in host*.conf: +# HOST1_WEEKLY_RESTART_CONTAINERS +# HOST2_WEEKLY_RESTART_CONTAINERS + +# ━━━ Docker Watchdog ━━━ +# Continuous two-tier self-healing container monitoring. +# Started by array_started.sh — runs until array stops. +# Re-sources all three conf files each cycle — add/remove containers without restarting watchdog. +# +# Tier 1 — strict monitoring of explicitly configured containers: +# Memory hard limits — immediate restart if exceeded +# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT sustained strikes +# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT failed checks +# Required containers — must always be running, strike + skip list with auto-clear +# +# Tier 2 — global health scan of ALL running containers: +# Unhealthy status — Docker HEALTHCHECK unhealthy → restart +# OOM killed — kernel killed → restart + notify +# Crash loop detection — RestartCount climbing → notify, critical above limit +# Dead containers — remove and restart +# Unexpected exits — non-zero exit code → restart +# +# All per-host container lists live in host*.conf: +# HOST*_WATCHDOG_CONTAINERS — memory hard limits per container +# HOST*_WATCHDOG_CONTAINER_URLS — HTTP health check URLs +# HOST*_WATCHDOG_REQUIRED_CONTAINERS — must always be running +# HOST*_WATCHDOG_SCAN_IGNORE — skip in Tier 2 scan +# HOST*_WATCHDOG_DEPENDENCIES — dependency ordering for restart decisions + +# Strike state file — persistent in STATE_DIR + WATCHDOG_STATE_FILE="$STATE_DIR/container_watchdog_state.db" + +# CPU thresholds — normalised against total core count at runtime + SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU + HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU + CPU_FAIL_LIMIT=2 # consecutive hard CPU strikes before container restart + +# Memory soft threshold — warn when container reaches this % of its hard limit + SOFT_MEM_THRESHOLD=80 + +# HTTP responsiveness + RESP_FAIL_LIMIT=2 # consecutive failed checks before restart + CURL_TIMEOUT=5 # seconds per check before timeout + +# Tier 2 master toggle + WATCHDOG_SCAN_ALL=true # false = only WATCHDOG_CONTAINERS + required containers + +# Individual Tier 2 check toggles + WATCHDOG_RESTART_UNHEALTHY=true + WATCHDOG_RESTART_DEAD=true + WATCHDOG_RESTART_CRASHED=true + WATCHDOG_NOTIFY_OOM=true + WATCHDOG_NOTIFY_CRASHLOOP=true + +# Crash loop threshold + WATCHDOG_CRASH_LIMIT=5 # RestartCount above this = critical crash loop + +# Startup grace period — skip restarts while system is still booting + WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures + +# Restart loop protection — prevents watchdog from endlessly restarting a broken container + WATCHDOG_CONTAINER_RESTART_LIMIT=3 + WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours + WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db" + +# Notification batching — one summary per cycle instead of one ping per event + WATCHDOG_BATCH_NOTIFY=true + +# Appdata size monitoring — two-part catch-all for runaway growth and oversized log files. +# +# Part 1 — Growth rate (zero-config): +# Reads per-container dir totals each cycle via du, compares to previous cycle. +# Any container growing more than WATCHDOG_APPDATA_GROWTH_GB triggers a focused *.log scan +# inside that container. No per-container config required — new containers are covered +# automatically. Baseline built on first run after boot; growth detection starts cycle 2. +# +# Part 2 — Absolute log size: +# Finds *.log / *.log.* files over WATCHDOG_APPDATA_LOG_MAX_GB across all appdata paths. +# Catches logs that have already stabilised at a large size and are no longer actively growing. +# +# Strike system (reuses existing watchdog infrastructure): +# Strike 1 — warn + notify: condition first detected +# Strike 2 — warn + escalated notify: still present next cycle +# Strike 3 (WATCHDOG_APPDATA_STRIKE_LIMIT) — action cycle: +# If WATCHDOG_APPDATA_TRUNCATE_LOGS=true: truncate *.log files in-place, clear strikes +# If false: critical notify only, strikes held until condition resolves +# Condition resolves (growth stops / log drops below threshold) → strikes auto-clear +# +# HOST*_WATCHDOG_APPDATA_SIZES (in host*.conf) suppresses growth warnings for a +# container until its dir exceeds the configured ceiling. Only needed when a container +# legitimately has large stable data and you want to guarantee it never triggers a false alarm. + WATCHDOG_CHECK_APPDATA=true + WATCHDOG_APPDATA_PATHS=("/mnt/docker-unraid/appdata") + WATCHDOG_APPDATA_GROWTH_GB=2 # flag containers growing more than this per cycle + WATCHDOG_APPDATA_LOG_MAX_GB=2 # flag *.log files exceeding this size (absolute) + WATCHDOG_APPDATA_TRUNCATE_LOGS=false # set true to auto-truncate oversized *.log files on action cycle + WATCHDOG_APPDATA_STRIKE_LIMIT=3 # cycles before action fires (matches existing watchdog pattern) + WATCHDOG_APPDATA_GROWTH_FILE="$STATE_DIR/watchdog_appdata_growth.db" + STORAGE_WATCHDOG_STATE_FILE="$STATE_DIR/storage_watchdog_state.db" + +# ━━━ Docker Network Connect ━━━ +# Ensures custom networks exist and connects containers at array start. +# Runs once via ARRAY_START_SCRIPTS — idempotent, safe to re-run. +# Container and network lists are host-specific — defined in host*.conf: +# HOST1_NETWORK_CONNECT_CONTAINERS / HOST2_NETWORK_CONNECT_CONTAINERS +# HOST1_NETWORK_CONNECT_NETWORKS / HOST2_NETWORK_CONNECT_NETWORKS + +# ============================================================================================== +# ── UNRAID ESSENTIALS ───────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ inotify Tuning ━━━ +# Linux inotify limits — applied at every array start by inotify_tuning.sh. +# Default unRAID values are very low — with many Docker containers watching files +# you can silently exhaust the limit causing containers to miss file events. +# Symptoms: containers miss file events, downloads not detected, library not updated. +# Live TV stutter is a known symptom of inotify exhaustion. +# These settings are lost on reboot — reapplied automatically at array start. + INOTIFY_MAX_INSTANCES=1024 # default: 128 — max inotify instances per user + # 1024 handles ~20-30 containers watching files simultaneously + INOTIFY_MAX_WATCHES=1048576 # default: 8192 — SHARED budget across ALL users/containers + # 524288 (512K) was previous value — raised to 1M (1048576) + # VSCode/Code-Server alone needs ~50K-200K for large workspaces + # with node_modules. All arr containers + Emby + VSCode share + # this budget. 1M is safe on 128GB RAM (~128MB kernel memory) + # If VSCode shows "unable to watch for file changes" → too low + INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — max events queued before dropping + +# ━━━ System Tuning Monitor ━━━ +# Tracks inotify and php-fpm usage over time. +# Snapshot written every 6 hours by system_tuning_monitor.sh. +# Read by sunday_morning_coffee_report.sh for weekly peak/avg/warning summary. + INOTIFY_WARN_PCT=80 # warn if inotify instances exceed this % of limit + PHP_FPM_WARN_PCT=80 # warn if php-fpm workers exceed this % of max_children + TUNING_MONITOR_LOG="$DATA_DIR/system_tuning_history.db" + TUNING_LOG_RETENTION=30 # days before old entries are purged + +# ━━━ Reboot ━━━ +# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots. +# Gives users time to save work — 300s = 5 minutes. + REBOOT_SLEEP=300 + +# ━━━ Mover ━━━ +# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process. +# Gives mover time to finish current file transfer before being interrupted. + MOVER_STOP_TIMEOUT=300 + +# ━━━ Syslog Filter ━━━ +# Path for the rsyslog filter file that suppresses Docker veth interface noise. +# Docker creates a new veth interface for each container — generates hundreds of +# log lines per hour that have no diagnostic value. Filter removes them at source. + FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" + +# ━━━ PHP-FPM ━━━ +# Higher max_children allows more concurrent PHP requests to the unRAID WebGUI. +# Default is very low — increasing it prevents WebGUI slowdowns under load. +# 250 is safe for servers with 32GB+ RAM. + PHP_CONF="/etc/php-fpm.d/www.conf" + PHP_MAX_CHILDREN=250 + +# ━━━ Clear Logs ━━━ +# System log files cleared weekly to prevent rootfs fill over time. +# These grow continuously — without clearing they eventually consume all rootfs space. +# clear_logs.sh runs in WEEKLY_MAINTENANCE_SCRIPTS — Sunday 2:30am. +# +# Size threshold approach — only clear if log exceeds threshold. +# Avoids destroying useful recent diagnostic context when logs are small. +# LOG_MIN_SIZE_MB: skip clearing if log is under this size (not worth clearing) +# LOG_DOCKER_MAX_MB: clear a container log only if it exceeds this size +# Docker logs grow fastest on active containers (Emby, SABnzbd, Sonarr) +# 100MB per container × 30 containers = 3GB before clearing kicks in + LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) + LOG_MIN_SIZE_MB=10 # skip system log if under this size (already small) + LOG_DOCKER_MAX_MB=100 # clear Docker container log only if over this size (MB) + +# ━━━ Network Watchdog ━━━ +# Services-layer connectivity — internet reachability, DDNS sync, Tailscale, NPM proxy. +# Host-specific values (domain, container, NPM URL) live in host*.conf. + NETWORK_WATCHDOG_ENABLED=true + NETWORK_WATCHDOG_INTERNET_URL="https://1.1.1.1" + NETWORK_WATCHDOG_INTERNET_TIMEOUT=5 + NETWORK_WATCHDOG_CHECK_TAILSCALE=true + NETWORK_WATCHDOG_NPM_TIMEOUT=10 + NETWORK_WATCHDOG_NPM_STRIKE_LIMIT=2 + NETWORK_WATCHDOG_NPM_STATE_FILE="$STATE_DIR/network_watchdog_state.db" + +# ━━━ WebGUI Watchdog ━━━ +# Monitors unRAID WebGUI responsiveness — escalates through nginx restart → emhttp restart. +# Separate from docker_watchdog — this monitors the unRAID UI itself, not containers. + WEBGUI_URL="http://localhost" + WEBGUI_TIMEOUT=5 # seconds before curl gives up on WebGUI check + WEBGUI_NGINX_WAIT=15 # seconds after nginx restart before rechecking + WEBGUI_EMHTTP_WAIT=30 # seconds after emhttp restart before rechecking + +# ============================================================================================== +# ── MEDIA ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Media Permissions ━━━ +# Applied recursively by media_shares_permissions.sh. +# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership. +# +# Why split directory vs file permissions: +# Directories need execute bit to enter — 755 allows owner+group+others to traverse +# Files should NOT be executable — 664 allows owner+group read/write, others read +# This is POSIX best practice for media servers, not a blanket 777 band-aid +# +# Why nobody:users ownership: +# All ecosystem containers run as PUID=99 (nobody) PGID=100 (users) on unRAID +# Files owned by nobody:users are accessible to all containers without world-write +# If this script fixes many files each run → a container has wrong PUID/PGID set +# Fix: add PUID=99 PGID=100 to the container's environment variables +# +# This script runs daily as a failsafe — even with correct container config: +# rsync may bring files with source ownership if not run with --chown +# Manual admin copies create root:root files +# New containers may not have PUID/PGID set yet + PERMISSIONS_DIR_MODE="755" # directories — traverse + list, no world-write + PERMISSIONS_FILE_MODE="664" # files — group read/write, no execute + PERMISSIONS_OWNER="nobody:users" + +# Share list defined per host in host*.conf: +# HOST1_MEDIA_PERMISSION_SHARES / HOST2_MEDIA_PERMISSION_SHARES + +# ━━━ Media Cleaner ━━━ +# Removes junk files from media shares — two profiles: anime and media. +# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media +# Folder lists defined per host in host*.conf: +# HOST1_ANIME_CLEAN_FOLDERS / HOST2_ANIME_CLEAN_FOLDERS +# HOST1_MEDIA_CLEAN_FOLDERS / HOST2_MEDIA_CLEAN_FOLDERS + + ANIME_FILE_PATTERNS=( + # ── Checksums ──────────────────────────────────────────────────────────────────────── + '*.sfv' '*.md5' '*.sha1' # verification files — useless post-download + + # ── Scene / download metadata ───────────────────────────────────────────────────────── + '*.url' '*.lnk' # scene links + '*.info' '*.diz' # scene description files + '*.nzb' # usenet download files + '*.torrent' # torrent files left by download clients + + # ── Archives and segments ───────────────────────────────────────────────────────────── + '*.rar' '*.zip' '*.7z' '*.ace' # archives — source files not needed after extract + '*.r00' '*.r01' '*.r02' '*.r03' # multi-part rar segments + '*.r04' '*.r05' '*.r06' '*.r07' + '*.r08' '*.r09' '*.srr' # scene repair files + '*.001' '*.002' '*.003' # split archive parts + '*.gz' '*.tar' '*.bz2' # linux archives + + # ── Scene samples and proofs ────────────────────────────────────────────────────────── + '*.sample*' '*.proof*' # scene samples — never needed in library + + # ── Executables and scripts ─────────────────────────────────────────────────────────── + '*.exe' '*.scr' '*.com' # Windows executables + '*.bat' '*.cmd' '*.vbs' '*.ps1' # Windows scripts + '*.msi' '*.dll' '*.sys' # Windows installers and system files + '*.sh' # shell scripts in media folders = suspicious + + # ── Incomplete downloads ────────────────────────────────────────────────────────────── + '*.!ut' '*.!qB' # uTorrent / qBittorrent incomplete markers + '*.crdownload' '*.opdownload' # Chrome / Opera incomplete downloads + '*.part' # partial download files + + # ── Sync conflicts ──────────────────────────────────────────────────────────────────── + '*sync-conflict*' # Syncthing conflict copies + ) + + MEDIA_FILE_PATTERNS=( + # ── Checksums ──────────────────────────────────────────────────────────────────────── + '*.sfv' '*.md5' '*.sha1' # verification files — useless post-download + + # ── Scene / download metadata ───────────────────────────────────────────────────────── + '*.url' '*.lnk' # scene links + '*.info' '*.diz' # scene description files + '*.nzb' # usenet download files + '*.torrent' # torrent files left by download clients + + # ── Archives and segments ───────────────────────────────────────────────────────────── + '*.rar' '*.zip' '*.7z' '*.ace' # archives — source files not needed after extract + '*.r00' '*.r01' '*.r02' '*.r03' # multi-part rar segments + '*.r04' '*.r05' '*.r06' '*.r07' + '*.r08' '*.r09' '*.srr' # scene repair files + '*.001' '*.002' '*.003' # split archive parts + '*.gz' '*.tar' '*.bz2' # linux archives + + # ── Scene samples and proofs ────────────────────────────────────────────────────────── + '*.sample*' '*.proof*' # scene samples — never needed in library + + # ── Executables and scripts ─────────────────────────────────────────────────────────── + '*.exe' '*.scr' '*.com' # Windows executables + '*.bat' '*.cmd' '*.vbs' '*.ps1' # Windows scripts + '*.msi' '*.dll' '*.sys' # Windows installers and system files + '*.sh' # shell scripts in media folders = suspicious + + # ── Incomplete downloads ────────────────────────────────────────────────────────────── + '*.!ut' '*.!qB' # uTorrent / qBittorrent incomplete markers + '*.crdownload' '*.opdownload' # Chrome / Opera incomplete downloads + '*.part' # partial download files + + # ── Sync conflicts ──────────────────────────────────────────────────────────────────── + '*sync-conflict*' # Syncthing conflict copies + + # ── Media-specific extras ───────────────────────────────────────────────────────────── + '*.iso' # disc images — extracted content is in library + '*.lrc' # lyrics files — handled by music apps not Emby + ) + +# ━━━ Arr Cleanup ━━━ +# Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs. +# Per-host URLs, API keys, and path maps live in host*.conf. +# detect_hosts() selects correct host vars at runtime. +# +# API versions — update MAJOR version here when script is updated for a new arr version: +# Sonarr v4 → /api/v3/series → /api/v3/episodefile?seriesId=X +# Radarr v6 → /api/v3/movie → /api/v3/moviefile?movieId=X +# Lidarr v3 → /api/v1/artist → /api/v1/trackFile?artistId=X + SONARR_VERSION_MAJOR=4 + RADARR_VERSION_MAJOR=6 + LIDARR_VERSION_MAJOR=3 + +# Lidarr shared settings + LIDARR_LOCK_WARN_AGE=3600 # 1hr — large libraries take time, not stuck + LIDARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion + LIDARR_MAX_DELETE_GB=5 # require --i-know-what-im-doing if deletion exceeds this + LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run + # protects against API returning partial data on a bad day + LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count" + LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") + LIDARR_PROTECTED_PATTERNS=( + # Metadata + "*.nfo" "*.tbn" + # Images — album art, artist images, Emby artwork + "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" + "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" + "banner.*" "thumb.*" "landscape.*" + "folder.*" "cover.*" "album.*" "artist.*" "disc.*" + # Lyrics + "*.lrc" + # Cue sheets and rip logs + "*.cue" "*.log" + # Theme music — Emby downloads artist themes into artist folders + "theme.mp3" "theme.flac" "theme.m4a" "theme.wav" + ) + +# Lidarr artwork fetcher settings + LIDARR_ART_MIN_SIZE=10000 # bytes — reject downloads smaller than this + LIDARR_ART_MAX_PARALLEL=4 # concurrent background download jobs + LIDARR_ART_RETRIES=2 # download retry attempts per image + LIDARR_ART_SLEEP_BETWEEN=0.2 # seconds between fanart.tv API calls +# HOST*_FANART_API_KEY / HOST*_LASTFM_API_KEY — set in host*.conf + +# Lidarr discovery settings (playback_aware_lidarr_discovery.sh) + LIDARR_DISCOVERY_THRESHOLD=70 # score to accept candidate (0-100) + LIDARR_DISCOVERY_LOOKBACK_DAYS=7 # Emby play history window in days + LIDARR_DISCOVERY_MIN_PLAYS=3 # min plays in window before evaluating an artist + LIDARR_DISCOVERY_USER_CAP_PCT=35 # max % any single user can contribute to play score (prevents one listener dominating) + LIDARR_DISCOVERY_MAX_ADDS=5 # max artists to add per run — quality over bulk + LIDARR_DISCOVERY_REJECT_COOLDOWN=30 # days before re-evaluating a rejected artist + LIDARR_DISCOVERY_HISTORY="$DATA_DIR/lidarr_discovery_history.db" + +# Sonarr discovery settings (playback_aware_sonarr_discovery.sh) + SONARR_DISCOVERY_THRESHOLD=52 # score to accept candidate (0-100) + SONARR_DISCOVERY_LOOKBACK_DAYS=14 # Emby episode play history window in days (shorter than movies — TV watched more frequently) + SONARR_DISCOVERY_MAX_SEEDS=5 # max seed series from Stage 1 + SONARR_DISCOVERY_MAX_ADDS=3 # max shows to add per run — TV is a larger commitment than movies + SONARR_DISCOVERY_MIN_VOTE_COUNT=50 # min TMDB votes (TV has fewer votes than movies at same popularity) + SONARR_DISCOVERY_MIN_RATING=65 # min TMDB vote_average × 10 (65 = 6.5/10) + SONARR_DISCOVERY_REJECT_COOLDOWN=60 # days before re-evaluating a rejected show + SONARR_DISCOVERY_USER_EPISODE_CAP=8 # max episodes any one user contributes to seed volume score + SONARR_DISCOVERY_MONITOR_MODE="all" # Sonarr monitor mode on add: all | future | first | latest | none + SONARR_DISCOVERY_HISTORY="$DATA_DIR/sonarr_discovery_history.db" + +# Radarr discovery shared settings + RADARR_DISCOVERY_THRESHOLD=52 # score to accept candidate (0-100) — lower than Lidarr since diverse seeds rarely overlap + RADARR_DISCOVERY_LOOKBACK_DAYS=30 # Emby watch history window in days (movies rewatched less often) + RADARR_DISCOVERY_MAX_SEEDS=5 # max seed movies from Stage 1 + RADARR_DISCOVERY_MAX_ADDS=5 # max movies to add per run + RADARR_DISCOVERY_MIN_VOTE_COUNT=100 # min TMDB votes to be considered a candidate + RADARR_DISCOVERY_MIN_RATING=60 # min TMDB vote_average × 10 (60 = 6.0/10) + RADARR_DISCOVERY_REJECT_COOLDOWN=60 # days before re-evaluating a rejected movie + RADARR_DISCOVERY_SEED_LIBRARIES=("Movies") # Emby libraries to draw seed movies from + RADARR_DISCOVERY_HISTORY="$DATA_DIR/radarr_discovery_history.db" + +# Emby → arr sync library allowlists +# Only these Emby library names will be considered by the sync tools. +# Names must match exactly as shown in Emby > Dashboard > Libraries. + SONARR_EMBY_LIBRARIES=("Anime" "Kids Shows" "Stand-Up Comedy" "TV shows") + RADARR_EMBY_LIBRARIES=("Movies" "Kid's Movies") + +# Sonarr shared settings + SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion + SONARR_MAX_DELETE_GB=10 # require --i-know-what-im-doing if deletion exceeds this + SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") + SONARR_PROTECTED_PATTERNS=( + # Subtitles + "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" + # Metadata + "*.nfo" "*.tbn" + # Images + "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" + "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" + "banner.*" "thumb.*" "landscape.*" + # Kodi/Emby extras — not tracked by Sonarr API + "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" + "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" + "*-clip.*" "*-other.*" + # Theme songs — stored in show folder, not tracked by arr + "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" + "theme.mkv" "theme.mp4" + "trailer.*" "trailer *.*" + ) + +# Radarr shared settings + RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion + RADARR_MAX_DELETE_GB=15 # require --i-know-what-im-doing if deletion exceeds this + RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") + RADARR_PROTECTED_PATTERNS=( + # Subtitles + "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" + # Metadata + "*.nfo" "*.tbn" + # Images + "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" + "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" + "banner.*" "thumb.*" "landscape.*" + # Kodi/Emby extras — not tracked by Radarr API + "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" + "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" + "*-clip.*" "*-other.*" + # Theme songs — stored in movie folder, not tracked by arr + "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" + "theme.mkv" "theme.mp4" + "trailer.*" "trailer *.*" + ) + +# Radarr/Sonarr TMDb/TVDB removed entry cleanup + RADARR_DROPPED_ADD_EXCLUSION=true # add removed movies to import exclusion list + SONARR_DROPPED_ADD_EXCLUSION=true # add removed series to import exclusion list + +# ━━━ Arr Failed/Stalled Recovery ━━━ +# Auto blocklist + re-search failed imports and stalled downloads. +# Runs every 6 hours — schedule: 0 */6 * * * +# +# Targets four problem types: +# importFailed — downloaded but arr couldn't import +# importPending — downloaded, stuck waiting to import (won't self-resolve) +# error status — serious failure not covered above +# stalled — download stuck with no connections or progress +# +# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first. +# Per-host recovery toggles (HOST1_SONARR_RECOVERY etc.) live in host*.conf. + ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this + # matches cron interval — items eligible after one missed cycle + +# ============================================================================================== +# ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Session-based storage allocator using filesystem symlink indirection. +# ffmpeg resolves the symlink ONCE at session start — existing sessions never affected. +# Symlink flips between ramdisk and SSD are transparent to active streams. +# +# ⚠️ Docker mount — must use shared propagation so symlink flips work inside container: +# --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared +# Standard path mappings use rprivate — symlink changes are NOT visible inside container. + +# ━━━ Transcode Manager ━━━ +# tmpfs mount point — created at array start by ramdisk_setup.sh. +# Must exist before Emby starts so the symlink resolves correctly. + RAMDISK_PATH="/mnt/ramdisk_transcodes" + +# Ramdisk size and flip thresholds — defined per host in host*.conf. +# All three are coupled — if size changes, thresholds must change with it. +# HOST1_RAMDISK_SIZE / HOST2_RAMDISK_SIZE +# HOST1_RAMDISK_WARN_GB / HOST2_RAMDISK_WARN_GB ← flip to SSD at this usage +# HOST1_RAMDISK_LOW_GB / HOST2_RAMDISK_LOW_GB ← flip back to ramdisk at this usage + +# Symlink that Emby points at — this path NEVER changes regardless of ramdisk/SSD state. +# Emby resolves the symlink once per session at start — symlink flips are transparent. +# Must match the container path configured in Emby's Extra Parameters. + TRANSCODE_LINK="/mnt/ram-transcode" + +# SSD fallback location — defined per host in host*.conf (cache path differs per server): +# HOST1_TRANSCODE_SSD / HOST2_TRANSCODE_SSD + +# Minimum free GB on SSD before allowing flip from ramdisk to SSD. + RAMDISK_SSD_MIN_GB=20 + + TRANSCODE_MAX_AGE=20 # minutes — HLS segment age before cleanup eligibility + TRANSCODE_ORPHAN_AGE=30 # minutes — files with no matching active session + TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in one hour + + TRANSCODE_OWNER="nobody:users" + TRANSCODE_CHMOD="755" + +# Operating mode — controls symlink direction behaviour. +# smart — auto-flips between ramdisk and SSD based on thresholds (default) +# ramdisk — always uses ramdisk, warns if RAMDISK_WARN_GB exceeded but holds +# ssd — always uses SSD, never flips to ramdisk + TRANSCODE_MANAGER_MODE="smart" + +# Daily statistics log — read by weekly_health_digest.sh for transcode summary. + TRANSCODE_STATE_FILE="$STATE_DIR/transcode_state.db" + TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db" + TRANSCODE_LOG_RETENTION=90 # days before old entries purged + + TRANSCODE_CHECK_EMBY=true + +# ━━━ Transcode Server Array ━━━ +# All media servers sharing the ramdisk transcode space. +# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex +# Entries with placeholder API keys are skipped automatically. +# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk. +# Defined per host in host*.conf — Emby container names and keys differ per server: +# HOST1_TRANSCODE_SERVERS / HOST2_TRANSCODE_SERVERS + +# ============================================================================================== +# ── MONITORS ────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Certificate Monitor ━━━ +# Checks SSL certificate expiry via direct openssl connection — no NPM dependency. +# Checks the actual certificate served by each domain, not what NPM thinks it has. +# Domains defined per host in host*.conf — each server monitors its own domains: +# HOST1_CERT_MONITOR_DOMAINS / HOST2_CERT_MONITOR_DOMAINS + CERT_WARN_DAYS=30 # warn when cert expires within this many days + CERT_CRIT_DAYS=7 # critical alert within this many days + CERT_TIMEOUT=10 # seconds per domain before giving up + +# ━━━ Backup Verify ━━━ +# Verifies rsync mirror health by comparing random file checksums between servers. +# Catches silent corruption or incomplete syncs that rsync itself wouldn't detect. +# Defined per host in host*.conf — leave empty to use HOST*_DAILY_SYNC_SHARES automatically: +# HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES + BACKUP_VERIFY_SAMPLE=10 # random files to check per share + BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample + +# ━━━ SMART Health ━━━ +# Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*. +# Thresholds read from /boot/config/plugins/dynamix/dynamix.cfg at runtime +# (hot/max/hotssd/maxssd) — these vars are fallback only if dynamix.cfg not found. + SMART_TEMP_WARN=45 # fallback — Celsius warn threshold + SMART_TEMP_CRIT=55 # fallback — Celsius critical threshold +# Drives to ignore defined per host in host*.conf — hardware is server-specific: +# HOST1_SMART_IGNORE_DRIVES / HOST2_SMART_IGNORE_DRIVES + +# ━━━ ZFS Memory Snapshot ━━━ +# Weekly ZFS pool health and memory diagnostic report — informational only. + ZFS_REPORT_LOG="$DATA_DIR/zfs-weekly-health.log" + ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC using more than this % of its max + ZFS_REPORT_FREE_WARN_GB=10 # warn if less than this GB free RAM + ZFS_REPORT_AVAIL_WARN_GB=20 # warn if less than this GB available on ZFS pool + ZFS_REPORT_DOCKER_TOP=10 # how many top Docker containers to show by memory +# Pool ignore list defined per host in host*.conf — pool names are server-specific: +# HOST1_ZFS_REPORT_IGNORE_POOLS / HOST2_ZFS_REPORT_IGNORE_POOLS + +# ━━━ Bandwidth Monitor ━━━ +# Called automatically by rsync.sh after each sync — one bounded write per run. +# Tracks transfer size, duration and profile per sync for weekly summary reporting. + BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db" + BANDWIDTH_LOG_RETENTION=90 # days before old entries purged + BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report + +# Stats files — written by cleanup and recovery scripts, read by coffee report. +# All in DATA_DIR — array always running when these are written. + ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats + ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats + +# ━━━ Health Digest ━━━ +# Aggregated system health summary — reads existing state files, no new writes. +# Three profiles control when the digest is sent: +# always — sends every run regardless of findings +# smart — sends only when DIGEST_SMART_ON_* conditions are found +# weekly — sends once per week on DIGEST_DAY only + DIGEST_PROFILE="weekly" # always | smart | weekly + DIGEST_DAY="Sunday" + DIGEST_SMART_ON_WATCHDOG=true # send if any watchdog strikes are active + DIGEST_SMART_ON_FALLBACK=true # send if fallback state is not NORMAL + DIGEST_SMART_ON_CERT_WARN=true # send if any cert is under CERT_WARN_DAYS + DIGEST_SMART_ON_BANDWIDTH=true # send if any transfer exceeded BANDWIDTH_WARN_GB + +# ━━━ Emby Session Report ━━━ +# Weekly Emby usage statistics via API — no persistent writes, queries fresh each run. +# URL and API key pulled from HOST*_EMBY_URL and HOST*_EMBY_API_KEY in host*.conf. + EMBY_REPORT_DAYS=7 # days to include in the report period + EMBY_REPORT_TOP_N=10 # number of top content items to show + +# ============================================================================================== +# ── RESOURCE MANAGER ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Pressure reduction layer — keeps the system comfortable before things break. +# Called by watchdog_orchestrator.sh. Single-pass, not a continuous loop. +# +# ── PRESSURE LEVELS ─────────────────────────────────────────────────────────────────────────── +# Level 1 (soft) — throttle SABnzbd + qBit download speeds +# Level 2 (medium) — further throttle + docker pause background containers +# Level 3 (hard) — docker stop optional containers, signal docker_watchdog to defer +# +# ── PER-HOST CONTAINER LISTS ────────────────────────────────────────────────────────────────── +# HOST*_RW_PAUSE_CONTAINERS — docker pause at medium pressure (in host*.conf) +# HOST*_RW_STOP_CONTAINERS — docker stop at hard pressure (in host*.conf) + + RW_ENABLED=true + RW_STATE_FILE="$STATE_DIR/resource_watchdog_state.db" + +# ━━━ Pressure Thresholds ━━━ +# Graduated RAM response — resource_watchdog acts before system_watchdog reboots. +# RW_RAM_SOFT_GB > RW_RAM_MEDIUM_GB > RW_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always + RW_RAM_SOFT_GB=12 # throttle start — reduce background load + RW_RAM_MEDIUM_GB=8 # pause background containers + RW_RAM_HARD_GB=6 # stop optional containers (was SYS_WATCHDOG_MEM_SHUTDOWN_GB) + RW_RAM_RECOVER_GB=20 # RAM must reach this before restoring hard-stopped containers + +# Load average thresholds — multiplier × core count + RW_LOAD_SOFT_MULTIPLIER=2.0 # soft pressure: 2× cores sustained + RW_LOAD_MEDIUM_MULTIPLIER=3.0 # medium pressure: 3× cores sustained + +# Consecutive runs at lower pressure before de-escalating + RW_RECOVER_CYCLES=3 + +# ━━━ SABnzbd Throttle ━━━ +# Speed values: "50M" = 50 MB/s, "0" = unlimited + RW_SABNZBD_ENABLED=true + RW_SABNZBD_SPEED_SOFT="50M" + RW_SABNZBD_SPEED_MEDIUM="10M" + +# ━━━ qBittorrent Throttle ━━━ +# KB/s — 0 = unlimited + RW_QBIT_ENABLED=true + RW_QBIT_DL_SOFT=51200 # 50 MB/s + RW_QBIT_DL_MEDIUM=10240 # 10 MB/s + +# ━━━ Critical Containers ━━━ +# Never paused or stopped regardless of pressure level. +# Keep DNS, auth, media serving, and live TV always running. + RW_CRITICAL_CONTAINERS=( + "NginxProxyManager" # reverse proxy — internet access + "Authelia" # auth — nothing accessible without it + "Authelia-Secondary" + "Mariadb-Authelia" # Authelia dependency + "Mariadb-Authelia-Secondary" + "Redis-Authelia" # Authelia dependency + "Redis-Authelia-Secondary" + "AdGuard-Home" # DNS — all LAN resolution + "Emby" # media server — Live TV buffering + "Dispatcharr" # Live TV scheduler — loses state if stopped + "Dispatcharr-Basic" + "Dispatcharr-Iptv-Users" + ) + +# ============================================================================================== +# ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Single-pass system health check — last line of defense before a crash. +# Called by watchdog_orchestrator.sh every minute. NOT started by array_started.sh. +# Re-sources config at each orchestrator run — config changes take effect immediately. +# +# ── THREE-TIER RESPONSE SYSTEM ──────────────────────────────────────────────────────────────── +# CRITICAL — bypass ALL strikes, reboot immediately +# Docker daemon down, rootfs 100%, kernel oops, FD exhaustion, /boot read-only +# +# URGENT — bypass strikes only when OOM confirms active crisis +# RAM < MEM_GB AND OOM kills >= OOM_LIMIT in this cycle → reboot NOW +# Without OOM confirmation → normal strike system +# +# STANDARD — strike system (N consecutive failures → reboot) +# RAM tiers, high load, CPU temp, zombies, /var/log, /tmp, containers +# +# ── RAM ─────────────────────────────────────────────────────────────────────────────────────── +# SYS_WATCHDOG_MEM_GB — strike system → reboot (or OOM bypass) +# Warn/shutdown/recover RAM tiers are handled by resource_watchdog.sh + +# ━━━ State Files ━━━ + SYS_WATCHDOG_STATE_FILE="$STATE_DIR/system_watchdog_state.db" + DOCKER_WATCHDOG_FAILED_FILE="$STATE_DIR/docker_watchdog_failed.db" + SYS_WATCHDOG_REBOOT_LOG="$STATE_DIR/system_watchdog_reboots.db" + SYS_WATCHDOG_OOM_FILE="$STATE_DIR/system_watchdog_oom.db" + +# ━━━ Strike and Reboot Loop Settings ━━━ +# Strike system: a check must fail this many consecutive cycles before action is taken. +# Single spikes (one bad reading) are ignored — sustained problems trigger reboot. + SYS_WATCHDOG_STRIKE_LIMIT=2 # consecutive failures before reboot trigger + +# Reboot loop protection — if system keeps rebooting something is seriously wrong. +# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS → shutdown instead of reboot. + SYS_WATCHDOG_REBOOT_LIMIT=3 + SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 + +# ━━━ RAM Reboot Threshold ━━━ +# Reboot trigger only — warn/shutdown/recover handled by resource_watchdog.sh +# RW_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always (RM acts before watchdog reboots) + SYS_WATCHDOG_MEM_GB=4 # strike system → reboot + +# ━━━ OOM Bypass Settings ━━━ +# OOM bypass: if RAM is critically low AND kernel OOM kills exceed this threshold +# in a single cycle → bypass strike system and reboot immediately. +# Rate-based: kills per 5-minute cycle, not absolute count. +# Rationale: 1-2 kills = docker_watchdog handles it ✅ +# 3+ kills while RAM critical = system dying faster than watchdogs can heal ✅ + SYS_WATCHDOG_OOM_LIMIT=3 # OOM kills in one cycle to trigger bypass + +# ━━━ Thresholds ━━━ +# Set at "about to become unstable" — not "things are a bit high". + +# rootfs (/) usage — two levels: strike at 95%, critical bypass at 99% + SYS_WATCHDOG_ROOTFS_PCT=95 # standard strike threshold + SYS_WATCHDOG_ROOTFS_CRITICAL_PCT=99 # bypass strikes — truly full, writes failing + +# /var/log usage — log spam indicates something broken + SYS_WATCHDOG_LOG_PCT=95 + +# /tmp usage — tmpfs fills from downloads, lock files fail at 100% + SYS_WATCHDOG_TMP_PCT=90 # warn + attempt clear + SYS_WATCHDOG_TMP_CRITICAL_PCT=98 # bypass strikes if clear failed + +# ZFS ARC pinned percentage — ARC not releasing after reclaim = memory stuck + SYS_WATCHDOG_ARC_PINNED_PCT=98 + SYS_WATCHDOG_ARC_RELEASE_PCT=95 + +# Load average multiplier — threshold = MULTIPLIER × CPU core count +# MULTIPLIER=3 on 16-core = load of 48 before triggering + SYS_WATCHDOG_LOAD_MULTIPLIER=3 + +# Zombie process count — large numbers = serious process management failure + SYS_WATCHDOG_ZOMBIE_LIMIT=50 + +# CPU temperature — sustained high temp causes throttling or kernel panic + SYS_WATCHDOG_CPU_TEMP_MAX=95 + +# File descriptor exhaustion — system-wide FD limit near exhaustion +# New connections fail silently, Docker can't spawn processes, SSH fails + SYS_WATCHDOG_FD_CRITICAL_PCT=95 # bypass strikes — critical tier + +# Runaway process — single non-container process consuming excessive CPU +# Multiple strikes before action — single spikes are normal + SYS_WATCHDOG_RUNAWAY_CPU_PCT=90 # % single process must sustain + SYS_WATCHDOG_RUNAWAY_STRIKES=3 # consecutive cycles before warning + +# Array disk errors — accumulating mdstat errors = disk failing NOW + SYS_WATCHDOG_MDSTAT_ERROR_LIMIT=5 # new errors in one cycle before acting + +# ━━━ Check Toggles ━━━ +# Per-host — moved to host*.conf +# Different servers may have different hardware, NICs, and check requirements +# See HOST*_SYS_WATCHDOG_CHECK_* in host*.conf + +# ━━━ Abort Toggles ━━━ +# Conditions that prevent reboot even when a threshold is hit. +# CRITICAL tier bypasses these — a truly critical condition reboots regardless. +# true = abort standard reboot if this condition is active +# false = reboot anyway + SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # rebooting with bad pool risks data loss + SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity better than crashing mid-check + SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting move better than crashing mid-move + +# ============================================================================================== +# ──────────────────────── End Of User Variables ─────────────────────────────────────────────── +# ============================================================================================== diff --git a/Docker_Essentials/docker_update.sh b/Docker_Essentials/docker_update.sh index 2d0173d..8911ead 100755 --- a/Docker_Essentials/docker_update.sh +++ b/Docker_Essentials/docker_update.sh @@ -93,7 +93,7 @@ # Enable or disable normal mode. docker_daily_restart.sh runs regardless. # (default: true) # -# WEEKLY_REMAINING_UPDATES +# MONTHLY_REMAINING_UPDATES # Enable or disable remainder mode. (default: true) # # PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data] @@ -117,7 +117,7 @@ # docker_update.sh --remainder # Remainder mode — pull all running containers not in managed lists, # restart those that received updates, prune dangling images -# Called by weekly_sync_maintenance.sh at the end of the Sunday window +# Called by monthly_maintenance.sh # # docker_update.sh --dry-run # Preview which containers would be pulled without making changes @@ -170,8 +170,8 @@ detect_hosts # ━━━ Container Discovery ━━━ # ============================================================================================== if [[ "$REMAINDER_MODE" == true ]]; then - if [[ "${WEEKLY_REMAINING_UPDATES:-true}" != "true" ]]; then - echo "WEEKLY_REMAINING_UPDATES=false — skipping remainder container updates" + if [[ "${MONTHLY_REMAINING_UPDATES:-true}" != "true" ]]; then + echo "MONTHLY_REMAINING_UPDATES=false — skipping remainder container updates" exit 0 fi diff --git a/Orchestrators/weekly_sync_maintenance.sh b/Orchestrators/weekly_sync_maintenance.sh index 4a34b09..af6dd43 100755 --- a/Orchestrators/weekly_sync_maintenance.sh +++ b/Orchestrators/weekly_sync_maintenance.sh @@ -14,7 +14,6 @@ # 6. Start remote containers — correct order, delayed start respected # 7. Start local containers — rebuild if new image pulled, docker start otherwise # 8. WEEKLY_MAINTENANCE_SCRIPTS — weekly restarts etc. (docker_weekly_restart.sh) -# 9. docker_update.sh --remainder — update all containers not in daily or weekly sync window # # ── WHY WEEKLY NOT NIGHTLY FOR EMBY ────────────────────────────────────────────────────────── # Emby builds a warm image cache on HOST2 throughout the week. @@ -370,31 +369,6 @@ if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then done fi -# ============================================================================================== -# ━━━ Remainder Container Updates ━━━ -# ============================================================================================== -# Updates all running containers not already covered by daily or the weekly sync window. -# Runs last — weekly sync-window containers are back up before this pulls their peers. -echo "" -echo "━━━ $ICON_CONTAINERS Remainder Container Updates ━━━" - -DOCKER_UPDATE_SCRIPT="$SCRIPTS_ROOT/Docker_Essentials/docker_update.sh" -if [[ ! -f "$DOCKER_UPDATE_SCRIPT" ]]; then - warn "docker_update.sh not found — skipping remainder updates" - JOB_FAIL+=("docker_update.sh --remainder") -else - _remainder_args=("--remainder") - [[ "$DRY_RUN" == true ]] && _remainder_args+=("--dry-run") - if bash "$DOCKER_UPDATE_SCRIPT" "${_remainder_args[@]}"; then - echo "Remainder updates complete ✅" - JOB_PASS+=("docker_update.sh --remainder") - else - warn "Remainder updates completed with errors" - JOB_FAIL+=("docker_update.sh --remainder") - fi - unset _remainder_args -fi - WINDOW_END=$(date +%s) # ============================================================================================== diff --git a/Plugin/unraid/user_script_plug-in.sh b/Plugin/unraid/user_script_plug-in.sh index 3c04d3f..0c68ecd 100755 --- a/Plugin/unraid/user_script_plug-in.sh +++ b/Plugin/unraid/user_script_plug-in.sh @@ -71,7 +71,7 @@ # Correct schedules verified against script headers. sunday_morning_coffee_report # and rsync_emby_fallback added as separate scheduled entries. # v2.3 — intermediate_sync_maintenance.sh new 4-hour orchestrator (arr_sync + artwork). -# docker_update.sh (normal + --remainder): daily/weekly image pulls. +# docker_update.sh (normal + --remainder): daily/monthly image pulls. # downloaders_reset.sh: slskd + SABnzbd + qBittorrent maintenance. # arr_sync.sh: bidirectional Lidarr/Sonarr/Radarr library mesh. # lidarr_missing_art.sh: fetch missing album/artist artwork. @@ -330,8 +330,7 @@ # 7. Start remote containers dependency order, new image, verify each container up # 8. Start local containers dependency order, new image, verify each container up # 9. docker_weekly_restart.sh restart less-critical services: NextCloud, AdGuard, Immich -# 10. docker_update.sh --remainder pull latest images for all containers not in daily/weekly lists -# 11. WEEKLY_MAINTENANCE_SCRIPTS — clear_logs.sh + playback_aware_*.sh discovery scripts. +# 10. WEEKLY_MAINTENANCE_SCRIPTS — clear_logs.sh + playback_aware_*.sh discovery scripts. # playback_aware_lidarr_discovery.sh: 0–5 music adds/week # based on your Emby play history + Last.fm similar artists. # playback_aware_radarr_discovery.sh: 0–5 movie adds/week @@ -660,10 +659,10 @@ # Containers stay running during the pull. Rebuilds containers that received a new image. # docker_daily_restart.sh restarts the rest. Toggle: DAILY_CONTAINER_UPDATES in master.conf. # -# Remainder (weekly) — targets all running containers not in managed lists. -# Excludes daily list, weekly restart list, emby+critical-data sync-window containers, +# Remainder (monthly) — targets all running containers not in managed lists. +# Excludes daily list, emby+critical-data sync-window containers, # and fallback containers owned by the remote server's update cycle. -# Rebuilds containers that received a new image. Toggle: WEEKLY_REMAINING_UPDATES in master.conf. +# Rebuilds containers that received a new image. Toggle: MONTHLY_REMAINING_UPDATES in master.conf. # # bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh --dry-run # bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh --status