From 3937d5e33b32dfa0bc45283419d1c57056b2002a Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Fri, 12 Jun 2026 21:35:58 -0400 Subject: [PATCH] =?UTF-8?q?Consolidate=20conf=20templates=20into=20Configu?= =?UTF-8?q?rations/=20=E2=80=94=20remove=20stale=20Deployment/conf=5Ftempl?= =?UTF-8?q?ates/?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deployment/conf_templates/ had two stale files: host.conf.template was a copy of host1.conf (not a HOSTN template), and master.conf had hardcoded hostnames and an old appdata DATA_DIR path. The correct templates already exist in Configurations/ (master.conf.template, host.conf.template). Updated varaverk.plg and git_pull_execute.sh to read from Configurations/ directly. Removed TMPL_DIR variable from git_pull_execute.sh — CONF_DIR covers both. --- Deployment/conf_templates/host.conf.template | 769 ---------- Deployment/conf_templates/master.conf | 1433 ------------------ Deployment/conf_upgrade.sh | 6 +- Plugin/varaverk.plg | 4 +- git_pull_execute.sh | 11 +- 5 files changed, 10 insertions(+), 2213 deletions(-) delete mode 100644 Deployment/conf_templates/host.conf.template delete mode 100644 Deployment/conf_templates/master.conf diff --git a/Deployment/conf_templates/host.conf.template b/Deployment/conf_templates/host.conf.template deleted file mode 100644 index a763285..0000000 --- a/Deployment/conf_templates/host.conf.template +++ /dev/null @@ -1,769 +0,0 @@ -#!/bin/bash -# ============================================================================================== -# ========================== HOST1 CONFIGURATION — unRAID-Gmer4Lfe ============================ -# ============================================================================================== -# HOST1-specific variables — credentials, container names, share paths, failover lists. -# Sourced after master.conf — values here extend shared profile arrays and add HOST1-specific -# identity, credentials, and container configuration. -# -# Sparse checkout (git) ensures HOST2 never receives this file. -# HOST2 never sees HOST1 credentials — clean separation at the file level. -# -# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf. -# DO NOT put HOST2 variables here — they belong in host2.conf. -# -# ── INDEX ───────────────────────────────────────────────────────────────────────────────────── -# -# ── IDENTITY & CONNECTIVITY ──────────────────────────────────────────────────────────────── -# IDENTITY hostname, SSH key -# EMBY container name, URL, API key -# NOTIFICATIONS Discord webhook -# PARTNERSHIP auth containers, backup paths -# -# ── RSYNC ────────────────────────────────────────────────────────────────────────────────── -# DAILY SYNC SHARES media shares HOST1 owns and pushes to HOST2 -# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am) -# CRITICAL SYNC SHARES appdata shares synced every 30 minutes -# BACKUP VERIFY shares for checksum verification against remote -# HOST1 RSYNC PROFILE host1-appdata profile for HOST1-specific appdata syncs -# -# ── DOCKER ───────────────────────────────────────────────────────────────────────────────── -# DOCKER DAILY RESTART containers restarted daily -# DOCKER WEEKLY RESTART containers restarted weekly -# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list -# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh -# -# ── FALLBACK ─────────────────────────────────────────────────────────────────────────────── -# DDNS DDNS containers managed by HOST1 -# INTERNET LOSS containers stopped when internet is lost -# FALLBACK TIERS what HOST1 runs for HOST2 per tier -# TIER DELAYS how long HOST1 must be down before each tier activates on HOST2 -# RSYNC WRITEBACK HOST1 appdata synced back on handback -# -# ── MEDIA ────────────────────────────────────────────────────────────────────────────────── -# MEDIA PERMISSIONS share list for media_shares_permissions.sh -# MEDIA CLEANER folder lists for media_cleaner.sh -# -# ── MONITORS ─────────────────────────────────────────────────────────────────────────────── -# CERTIFICATE MONITOR domains checked for SSL expiry -# SMART HEALTH drives to skip in SMART monitoring -# ZFS REPORT pools to exclude from ZFS health report -# -# ── TRANSCODES ───────────────────────────────────────────────────────────────────────────── -# TRANSCODES ramdisk size, thresholds, SSD path, server array -# -# ── ARR STACK ────────────────────────────────────────────────────────────────────────────── -# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs -# LIDARR URL, API key, path map -# SONARR URL, API key, path map -# RADARR URL, API key, path map -# ARR RECOVERY per-arr recovery toggles -# -# ── SYSTEM WATCHDOG ──────────────────────────────────────────────────────────────────────── -# SYSTEM WATCHDOG per-host check toggles and NIC configuration -# -# ── RESOURCE MANAGER ─────────────────────────────────────────────────────────────────────── -# RESOURCE MANAGER containers paused/stopped under memory pressure -# -# ============================================================================================== - -# ============================================================================================== -# ── IDENTITY & CONNECTIVITY ─────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ Identity ━━━ -# HOST1 hostname lives in master.conf (not a credential — safe for all servers). -# SSH key used for all server-to-server operations — rsync, failover container commands. -# Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys. - HOST1_SSH_KEY="" # /root/.ssh/your_key_name - HOST1_OWNER="" # your Gitea/GitHub username - HOST1_OWNER_EMAIL="" # your email - -# ━━━ Emby ━━━ -# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh, -# weekly_sync_maintenance.sh, and HOST1_TRANSCODE_SERVERS below. -# API key: Emby Dashboard → API Keys → + New Key - HOST1_EMBY_CONTAINER="Emby" - HOST1_EMBY_URL="http://localhost:8096" - HOST1_EMBY_API_KEY="" # Emby Dashboard → API Keys → + New Key - -# ━━━ Jellyfin ━━━ -# API key: Jellyfin Dashboard → Administration → API Keys → + New Key - HOST1_JELLYFIN_CONTAINER="Jellyfin" - HOST1_JELLYFIN_URL="http://localhost:8096" - HOST1_JELLYFIN_API_KEY="" # Jellyfin Dashboard → API Keys → + New Key - -# ━━━ Gitea ━━━ -# Personal access token for gitea_ssh_setup.sh — registers this server's SSH public key -# with Gitea so git operations use key auth instead of passwords. -# Create in Gitea: Settings → Applications → Generate Token → scope: write:user - HOST1_GITEA_API_TOKEN="" # Gitea Settings → Applications → Generate Token - -# ━━━ Notifications ━━━ -# Discord webhook — leave blank to disable. -# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies. - HOST1_DISCORD_WEBHOOK="" # Discord channel → Integrations → Webhooks - -# ━━━ Partnership ━━━ -# HOST1 is always the owner (source of truth) unless --transfer has been run. -# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs. - -# Auth containers reconfigured on onboard/offboard. -# Format: "ContainerName|WebUIPort" -# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth) -# On offboard → WebUI pointed back at localhost - HOST1_PARTNERSHIP_AUTH_WEBUIS=( - "NginxProxyManager|81" - "Lldap-Gmer4Lfe|17170" - "Authelia|9091" - "Authelia-Secondary|9092" - ) - -# XML templates (from this server's templates-user/) pushed to mirror during onboard. -# These become the mirror's active auth stack, backed by the rsync-synced appdata. -# Update filename if Lldap is renamed to drop the host suffix. - HOST1_PARTNERSHIP_AUTH_STACK=( - # Dependencies first — Mariadb/Redis must be healthy before Authelia starts - "my-Mariadb-Authelia.xml" - "my-Mariadb-Authelia-Secondary.xml" - "my-Redis-Authelia.xml" - "my-Redis-Authelia-Secondary.xml" - # Auth apps — deployed after their deps are confirmed healthy - "my-Authelia.xml" - "my-Authelia-Secondary.xml" - "my-NginxProxyManager.xml" - "my-Lldap-Gmer4Lfe.xml" - # Source of truth — must be available on HOST2 independently of the auth stack - "my-Gitea.xml" - ) - -# XML templates pushed to mirror for the arr stack during onboard. -# Deps (e.g. databases) first if any — same ordering rule as auth stack. - HOST1_PARTNERSHIP_ARR_STACK=( - # "my-Sonarr.xml" - # "my-Radarr.xml" - # "my-Lidarr.xml" - # "my-Prowlarr.xml" - # "my-Bazarr.xml" - ) - -# Paths HOST2 should collect during the grace window after offboard. -# Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS. - HOST1_PARTNERSHIP_MIRROR_BACKUPS=( - # "/mnt/user/appdata-Fallback/Jayred365-Emby" - ) - -# Containers parked on this server when partnership is active. -# Stopped on onboard (owner deploys its stack instead), restarted on offboard. - HOST1_PARTNERSHIP_OWN_CONTAINERS=( - # "Emby" - # "NginxProxyManager" - ) - -# Emby admin provisioning — toggle is owner-only, credentials are per-host. -# Owner enables/disables the feature. Each host sets the account they want on the shared Emby. -# On onboard: owner reads mirror's HOST*_PARTNERSHIP_EMBY_ADMIN_* and creates that account. -# On offboard: account is deleted. Username collision → onboard exits with error. - HOST1_PARTNERSHIP_PROVISION_EMBY_ADMIN=false # owner controls whether Emby is shared - HOST1_PARTNERSHIP_EMBY_PORT=8096 - HOST1_PARTNERSHIP_EMBY_ADMIN_USER="" # this server's desired Emby username - HOST1_PARTNERSHIP_EMBY_ADMIN_PASS="" # this server's desired Emby password - -# ============================================================================================== -# ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ Daily Sync Shares ━━━ -# Shares HOST1 pushes to all other nodes every night (1am via daily_sync_maintenance.sh). -# Mesh model: every node pushes every media share — no ownership, no mirrors. -# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete). -# arr_cleanup removes true orphans based on local arr state. -# Any node can download content to any share — it propagates to all nodes on the next cycle. -# Nextcloud is intentionally one-directional (HOST1→HOST2 offsite backup — not arr-managed). -# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed. -# For shares needing container stops or custom options — add a profile in master.conf. - HOST1_DAILY_SYNC_SHARES=( - /mnt/user/Books - /mnt/user/Intros - /mnt/user/Kids_Movies - /mnt/user/Kids_Tv_Shows - /mnt/user/Movies - /mnt/user/Music - /mnt/user/Music_Videos - /mnt/user/Nextcloud - /mnt/user/stand-up_comedy - /mnt/user/Sports - /mnt/user/Tv_Shows - /mnt/user/Anime_Shows-Old - /mnt/user/Anime_Movies-Old - /mnt/user/Anime_Movies - /mnt/user/Anime_Shows - ) - -# Personal encrypted shares — synced for offsite backup, independent of media shares. -# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content. -# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting. - HOST1_PERSONAL_SHARES=( - # /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset - ) - -# ━━━ Weekly Sync Shares ━━━ -# Appdata shares synced during the weekly maintenance window (Sunday 2:30am). -# Containers stopped both sides before sync — full clean state guaranteed. -# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section. -# Order matters — Emby first (larger transfer), then Critical-Data (auth stack). - HOST1_WEEKLY_SYNC_SHARES=( - "/mnt/user/Media_Server/Emby" # emby profile — full clean mirror - "/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — auth stack - ) - -# ━━━ Intermediate Sync Shares ━━━ -# Shares synced every 4 hours by intermediate_sync_maintenance.sh. -# Uses DEFAULT_RSYNC_OPTS (no --delete) — for sub-daily propagation of metadata or watch state. -# Full media share sync stays in the daily window. Leave empty to skip mid-day rsync entirely. - HOST1_INTERMEDIATE_SYNC_SHARES=( - # Add shares here to enable mid-day rsync - # Example: "/mnt/user/Emby_Metadata" - ) - -# ━━━ Critical Sync Shares ━━━ -# Appdata shares synced every 30 minutes by critical_sync_maintenance.sh. -# Format: "/path/to/share" or "/path/to/share|profile-name" -# Order matters — Critical-Data first (auth stack), then Emby dirty sync. - HOST1_CRITICAL_SYNC_SHARES=( - "/mnt/user/appdata-Fallback/Critical-Data|critical-fallback" # auth dirty sync — stays running - "/mnt/user/Media_Server/Emby|emby-fallback" # Emby dirty sync — stays running - ) - -# ━━━ Backup Verify ━━━ -# Shares verified by backup_verify.sh — random file checksum comparison against remote. -# Leave empty to use HOST1_DAILY_SYNC_SHARES automatically. -# Sample size and minimum file size defined in master.conf. - HOST1_BACKUP_VERIFY_SHARES=( - # leave empty to use HOST1_DAILY_SYNC_SHARES automatically - ) - -# ━━━ HOST1 Rsync Profile — host1-appdata ━━━ -# HOST1-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf. -# Use for appdata unique to HOST1 (Organizrv2, VaultWarden, UptimeKuma etc.) -# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf. -# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Fallback/HOST1-Appdata --profile=host1-appdata - PROFILE_RSYNC_OPTS[host1-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host1-appdata]:-8000}" - PROFILE_BW_LIMIT[host1-appdata]=8000 - PROFILE_RETRY_COUNT[host1-appdata]=3 - PROFILE_SLEEP[host1-appdata]=300 - PROFILE_CRITICAL_CONTAINER_NAMES[host1-appdata]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe" - PROFILE_DELAYED_CONTAINERS[host1-appdata]="" - PROFILE_CONTAINER_DELAY[host1-appdata]=5 - PROFILE_EXCLUDE_DIRS[host1-appdata]="logs *.tmp" - -# ============================================================================================== -# ── DOCKER ──────────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ Docker Daily Restart ━━━ -# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS. -# Dispatcharr degrades over time without restart — daily is intentional, not just housekeeping. -# Order matters — auth stack first, then media services. - HOST1_DAILY_RESTART_CONTAINERS=( - "NginxProxyManager" - "Lldap-Gmer4Lfe" - "Authelia" - "Authelia-Secondary" - "Dispatcharr-Iptv-Users" - "Dispatcharr" # Live TV scheduler — degrades without daily restart - "Dispatcharr-Basic" - "ErsatzTV-Emby" - "Slskd" # Soulseek connection drops after extended uptime; restart refreshes share index - ) - -# ━━━ Docker Weekly Restart ━━━ -# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am). -# Containers already stopped for weekly sync — restart adds zero extra downtime. - HOST1_WEEKLY_RESTART_CONTAINERS=( - "NextCloud" - "Organizrv2-Gmer4Lfe" - "AdGuard-Home" - "Immich-Gmer4Lfe" - ) - -# ━━━ Docker Watchdog ━━━ -# Per-HOST1 container configuration for docker_watchdog.sh. -# Shared thresholds and toggles live in master.conf. - -# Memory hard limits in MB — immediate restart if exceeded. -# Set at "container is clearly broken" not "container is busy". -# 20GB=20480 18GB=18432 16GB=16384 12GB=12288 8GB=8192 4GB=4096 2GB=2048 1GB=1024 - declare -A HOST1_WATCHDOG_CONTAINERS=( - ["Emby"]=18432 # 18GB — large library + active transcodes - ["LidaTube"]=6144 # 6GB — memory leak over time - ["Tdarr"]=6144 # 6GB — encoding is memory intensive - ["Code-Server"]=1024 # 1GB — should never need more - ) - -# HTTP health check URLs — checked every cycle, strike system before restart. -# Only add containers with a meaningful web interface to check. - declare -A HOST1_WATCHDOG_CONTAINER_URLS=( - ["Emby"]="http://localhost:8096" - ) - -# Required containers — must always be running on HOST1. -# Strike system before restart — repeated failures go on skip list, auto-clears on recovery. -# Listed in dependency order — dependencies before dependents. - HOST1_WATCHDOG_REQUIRED_CONTAINERS=( - "NginxProxyManager" - "Lldap-Gmer4Lfe" - "Mariadb-Authelia" - "Mariadb-Authelia-Secondary" - "Redis-Authelia" - "Redis-Authelia-Secondary" - "Authelia" - "Authelia-Secondary" - ) - -# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting. -# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking. - HOST1_WATCHDOG_SCAN_IGNORE=( - "DashGate" - "PIA-WG-Config-Generator" - "Aperture" - "Aperture-Kids" - "pgvector-18-Apeture-Kids" - "Pgvector18-Aperture" - ) - -# Dependency ordering — skip restarting a container if its dependency is also down. -# Prevents watchdog from restarting Authelia before Mariadb is back up. -# SPACE-SEPARATED STRINGS — converted to array at runtime. - declare -A HOST1_WATCHDOG_DEPENDENCIES=( - ["Authelia"]="Mariadb-Authelia Redis-Authelia" - ["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary" - ["NextCloud"]="Postgres-NextCloud" - ) - -# Per-container appdata growth suppress ceilings in MB. -# ONLY needed in specific cases — growth rate detection covers all containers automatically. -# Use this when a container legitimately has large stable data and you want to guarantee -# it never triggers a false-positive growth alert. Growth warnings are suppressed while the -# container's dir stays below this ceiling; above it, warnings resume as normal. -# 50GB=51200 25GB=25600 20GB=20480 15GB=15360 10GB=10240 5GB=5120 - declare -A HOST1_WATCHDOG_APPDATA_SIZES=( - ["Tdarr"]="25600" # 25GB — transcode cache grows legitimately during active jobs - ["7dtd"]="20480" # 20GB — game server world data, expected to be large - ) - -# ━━━ Network Watchdog ━━━ -# Host-specific connectivity config for Watchdogs/System/network_watchdog.sh. - HOST1_NETWORK_WATCHDOG_DDNS_DOMAIN="gmer4lfe.com" - HOST1_NETWORK_WATCHDOG_DDNS_CONTAINER="Gmer4Lfe.com" - HOST1_NETWORK_WATCHDOG_NPM_URL="https://gmer4lfe.com" - -# ━━━ Docker Network Connect ━━━ -# Containers connected to custom networks at array start by docker_network_connect.sh. -# Networks created if they don't exist — idempotent, safe to re-run. - HOST1_NETWORK_CONNECT_CONTAINERS=( - "memcached" - "Npm-CrowdSec" - ) - - HOST1_NETWORK_CONNECT_NETWORKS=( - "high-availability" - ) - -# ============================================================================================== -# ── FALLBACK ────────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ DDNS ━━━ -# DDNS containers HOST1 manages — started/stopped by fallback.sh per DDNS absolute rules: -# Internet loss → stop immediately -# Failover → HOST2 starts HOST1's DDNS as Tier 1 (before any other containers) -# Handback → stop HOST1's DDNS on HOST2 → rsync → start containers → start local DDNS last - HOST1_DDNS_CONTAINERS=( - "Gmer4Lfe.com" - ) - -# ━━━ Internet Loss ━━━ -# Containers stopped immediately on HOST1 when internet connection is lost. -# Prevents external-facing services from operating without connectivity. - FALLBACK_HOST1_STOP_ON_NO_NET=( - "Gmer4Lfe.com" - ) - -# ━━━ Fallback Tiers — HOST1 Runs for HOST2 ━━━ -# Containers HOST1 starts when HOST2 goes down. -# Tier 1 is always immediate — vital services cannot wait. -# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in host2.conf). - FALLBACK_HOST1_COVERS_HOST2_TIER1=( - "Gmer4Lfe.us" - "VaultWarden-Jayred365" - ) - - FALLBACK_HOST1_COVERS_HOST2_TIER2=( - # "container-placeholder" - ) - - FALLBACK_HOST1_COVERS_HOST2_TIER3=( - # "container-placeholder" - ) - - FALLBACK_HOST1_COVERS_HOST2_TIER4=( - # "container-placeholder" - ) - -# ━━━ Tier Delays — HOST1's Containers on HOST2 ━━━ -# How long HOST1 must be down before each tier activates on HOST2 — in minutes. -# Tier 1 is always immediate — no delay var needed. - HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich - HOST1_TIER3_DELAY=720 # 12 hours — secondary services - HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders - -# ━━━ Rsync Writeback — HOST1 Appdata Back on Handback ━━━ -# Syncs HOST1 appdata BACK to HOST1 when it comes back online after a failover. -# Containers stopped before writeback — clean source, no competing writes. -# -# HOST1_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state -# is more reliable than dirty sync data for brief outages. - HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr - -# Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES — only list paths NOT in that array. - FALLBACK_HOST1_WRITEBACK_TIER1=( - "/mnt/user/Media_Server/Emby" # watch states built up during outage - ) - - FALLBACK_HOST1_WRITEBACK_TIER2=( - "/mnt/user/appdata-Fallback/Important-Data" # NextCloud + Postgres — files added during outage - ) - - FALLBACK_HOST1_WRITEBACK_TIER3=( - # "location-placeholder" - ) - - FALLBACK_HOST1_WRITEBACK_TIER4=( - "/mnt/user/appdata-Fallback/Arrs_Stack" # arr databases — downloads queued during outage - ) - -# ============================================================================================== -# ── MEDIA ───────────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ Media Permissions ━━━ -# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to. -# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership. - HOST1_MEDIA_PERMISSION_SHARES=( - /mnt/user/Anime_Movies - /mnt/user/Anime_Movies-Old - /mnt/user/Anime_Shows - /mnt/user/Anime_Shows-Old - /mnt/user/appcache - /mnt/user/Books - /mnt/user/Downloads - /mnt/user/Games - /mnt/user/Intros - /mnt/user/Kids_Movies - /mnt/user/Kids_Tv_Shows - /mnt/user/Movie_Recordings - /mnt/user/Movies - /mnt/user/Music - /mnt/user/Music_Videos - /mnt/user/Photo - /mnt/user/Sports - /mnt/user/stand-up_comedy - /mnt/user/Temp_Storage - /mnt/user/Tv_Recordings - /mnt/user/Tv_Shows - /mnt/user/YouTube - ) - -# ━━━ Media Cleaner ━━━ -# Folder lists for media_cleaner.sh — two profiles: anime and media. -# File patterns shared across all servers — defined in master.conf. -# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media - HOST1_ANIME_CLEAN_FOLDERS=( - /mnt/user/Anime_Movies - /mnt/user/Anime_Movies-Old - /mnt/user/Anime_Shows - /mnt/user/Anime_Shows-Old - ) - - HOST1_MEDIA_CLEAN_FOLDERS=( - /mnt/user/Kids_Movies - /mnt/user/Kids_Tv_Shows - /mnt/user/Movies - /mnt/user/Music - /mnt/user/Sports - /mnt/user/stand-up_comedy - /mnt/user/Tv_Shows - ) - -# ============================================================================================== -# ── MONITORS ────────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ Certificate Monitor ━━━ -# Domains checked via direct openssl connection — not relying on NPM's certificate state. -# Checks the actual certificate served, not what NPM thinks it has. -# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf. - HOST1_CERT_MONITOR_DOMAINS=( - "Gmer4Lfe.com" - "Gmer4Lfe.us" - ) - -# ━━━ SMART Health ━━━ -# Drives skipped in SMART attribute monitoring — hardware is server-specific. -# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf. - HOST1_SMART_IGNORE_DRIVES=( - "sda" # boot USB — SMART not meaningful on flash drives - ) - -# ━━━ ZFS Report ━━━ -# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools. -# These are individual array disks formatted as ZFS — converting to XFS over time via unBalance. -# Pool health thresholds defined in master.conf. - HOST1_ZFS_REPORT_IGNORE_POOLS=( - "disk5" - "disk6" - "disk8" - "disk9" - "disk10" - ) - -# ============================================================================================== -# ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront. -# Real-world: 9 streams peaked at ~5.5GB — 10G gives generous headroom on 128GB RAM. - HOST1_RAMDISK_SIZE="10G" - -# Usage thresholds — coupled to HOST1_RAMDISK_SIZE, adjust all three together if size changes. -# Hysteresis gap (8.5 - 7 = 1.5GB) prevents flip-flop between ramdisk and SSD. - HOST1_RAMDISK_WARN_GB=8.5 # flip to SSD when ramdisk usage reaches this - HOST1_RAMDISK_LOW_GB=7 # flip back to ramdisk when usage drops to this - -# SSD fallback path — where transcodes land when ramdisk exceeds HOST1_RAMDISK_WARN_GB. -# Must be on cache pool — array disks too slow for active transcode writes. - HOST1_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" - -# Media servers sharing the ramdisk transcode space on HOST1. -# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex -# Entries with placeholder API keys are skipped automatically. -# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk. - HOST1_TRANSCODE_SERVERS=( - "${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby" - "${HOST1_JELLYFIN_CONTAINER}|${HOST1_JELLYFIN_URL}|${HOST1_JELLYFIN_API_KEY}|jellyfin" - ) - -# ============================================================================================== -# ── ARR STACK ───────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh. -# detect_hosts() selects HOST1 vars when running on HOST1. -# -# PATH MAPS — container path → host path translation. -# Arr stores file paths using container-internal paths — scripts need host paths to scan. -# Add one entry per root folder in arr Settings → Media Management → Root Folders. - -# ━━━ Downloaders ━━━ -# Used by downloaders_reset.sh — runs every 30min via CRITICAL_MAINTENANCE_SCRIPTS. -# Clears stuck states, purges old history, prepares each client for a clean cycle. - -# slskd — clears stuck searches, dead transfers, purges expired failed imports. -# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected. - HOST1_SLSKD_URL="http://localhost:8980" - HOST1_SLSKD_API_KEY="" # slskd Settings → API key - HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports" - -# SABnzbd - HOST1_SABNZBD_URL="http://localhost:8180" - HOST1_SABNZBD_API_KEY="" # SABnzbd Config → General → API Key - -# qBittorrent — deleteFiles=false removes torrent from qBit but leaves files on disk. -# Radarr/Sonarr manage actual files independently. - HOST1_QBIT_URL="http://localhost:8080" - HOST1_QBIT_USERNAME="root" - HOST1_QBIT_PASSWORD="" # qBittorrent WebUI password - -# ━━━ Lidarr — HOST1 only ━━━ -# HOST2 does not run Lidarr — HOST1_LIDARR_RECOVERY flag handles the exit cleanly. - HOST1_LIDARR_URL="http://localhost:8686" - HOST1_LIDARR_API_KEY="" # Lidarr Settings → General → API Key - HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New" - HOST1_FANART_API_KEY="" # fanart.tv account → API Key - HOST1_LASTFM_API_KEY="" # last.fm API → Create API Account - - declare -A HOST1_LIDARR_PATH_MAP=( - ["/ext-music"]="/mnt/user/Music-New" - ) - -# ━━━ Sonarr ━━━ - HOST1_SONARR_URL="http://localhost:8989" - HOST1_SONARR_API_KEY="" # Sonarr Settings → General → API Key - HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows" - -# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder - declare -A HOST1_SONARR_PATH_MAP=( - ["/tv"]="/mnt/user/Tv_Shows" - ["/ext-standup-comedy"]="/mnt/user/stand-up_comedy" - ["/kids tv"]="/mnt/user/Kids_Tv_Shows" - ["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old" - ) - -# ━━━ Radarr ━━━ - HOST1_RADARR_URL="http://localhost:7878" - HOST1_RADARR_API_KEY="" # Radarr Settings → General → API Key - HOST1_TMDB_API_KEY="" # themoviedb.org → Settings → API - HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies" - -# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder - declare -A HOST1_RADARR_PATH_MAP=( - ["/movies"]="/mnt/user/Movies" - ["/kids movies"]="/mnt/user/Kids_Movies" - ["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy" - ["/anime-movies"]="/mnt/user/Anime_Movies-Old" - ) - -# ━━━ Arr Recovery Toggles ━━━ -# false = skip that arr on this host — exits cleanly without error - HOST1_SONARR_RECOVERY=true - HOST1_RADARR_RECOVERY=true - HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2 - -# ============================================================================================== -# ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Per-host check toggles and NIC config for system_watchdog.sh. -# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names. -# HOST1: TR1950X 128GB — full media server, active transcoding, ZFS cache pools. -# -# Three-tier response — all critical checks enabled by default on HOST1: -# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot -# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle -# Tier 3 (standard strike system): everything else -# -# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section. - -# ━━━ Primary NIC ━━━ -# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]" -# Common values: eth0, bond0, br0, eno1 - HOST1_SYS_WATCHDOG_NIC="eth0" - -# ━━━ Tier 1 — Critical Checks ━━━ -# These bypass the strike system — a single hit triggers immediate reboot. -# Disabling any of these is not recommended — they protect against acute system failure. - -# Docker daemon unresponsive → try restart, reboot if restart fails. -# Without a working daemon docker_watchdog.sh is blind and containers cannot be managed. - HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true - -# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately. -# At 99% rootfs writes fail silently — logs stop, Docker errors out, SSH may stop working. -# Standard 95% threshold still uses strike system — only 99%+ is critical tier. - HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true - -# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately. -# A kernel oops means the kernel ran with a corrupted state — stability is not guaranteed. - HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true - -# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately. -# At 95% FD: new connections fail, Docker can't spawn processes, SSH drops. - HOST1_SYS_WATCHDOG_CHECK_FD=true - -# /boot read-only detected → reboot immediately. -# Unexpected read-only /boot means state files and config writes are silently failing. -# Fallback state, watchdog reboot log, and lock files all go stale silently. - HOST1_SYS_WATCHDOG_CHECK_BOOT=true - -# ━━━ Tier 2 — Urgent OOM Check ━━━ -# Bypass strikes when RAM is critically low AND OOM kill rate confirms active crisis. -# Both must be enabled for Tier 2 bypass to function — disable either to always use strikes. - -# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta. -# Also provides diagnostic context in reboot messages (which processes were killed). - HOST1_SYS_WATCHDOG_CHECK_OOM=true - -# Free RAM check — required for both Tier 2 bypass and RAM tier logic. -# Tiers: MEM_WARN_GB(10) → notify | MEM_SHUTDOWN_GB(6) → stop containers | MEM_GB(4) → strikes - HOST1_SYS_WATCHDOG_CHECK_RAM=true - -# ━━━ Tier 3 — Standard Checks (strike system) ━━━ -# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action is taken. -# Single spikes are ignored — sustained problems trigger reboot. - -# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT. -# Log spam (Docker log storms, syslog loops) fills rootfs — indicates something broken. - HOST1_SYS_WATCHDOG_CHECK_LOG=true - -# ZFS ARC memory pinned above SYS_WATCHDOG_ARC_PINNED_PCT after cache drop. -# Enabled on HOST1 — ZFS cache pools actively used. Disable on hosts without ZFS. - HOST1_SYS_WATCHDOG_CHECK_ARC=true - -# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C). -# Sustained high temp causes kernel throttling or panic. Requires lm-sensors. - HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true - -# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count. -# DISABLED on HOST1 — Tdarr and Emby cause legitimate sustained load spikes during encoding. -# Enable on idle servers or adjust SYS_WATCHDOG_LOAD_MULTIPLIER if load is always high. - HOST1_SYS_WATCHDOG_CHECK_LOAD=false - -# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50). -# Large zombie counts indicate serious process management failure — something is stuck. - HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true - -# Check docker_watchdog.sh persistent skip list — required containers on skip list. -# Cross-watchdog coordination: if docker_watchdog gave up, system_watchdog escalates. -# ENABLED — HOST1 fully built and operational, skip list is meaningful. - HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true - -# /tmp filesystem usage above SYS_WATCHDOG_TMP_PCT with auto-clear attempt. -# Script tries to clear aged /tmp files first — only strikes if clear fails. -# Lock files, rsync temp files, and Docker ops use /tmp — 100% means lock failures. - HOST1_SYS_WATCHDOG_CHECK_TMP=true - -# Array disk error count delta in /proc/mdstat — accumulating errors = disk failing now. -# Triggers on SYS_WATCHDOG_MDSTAT_ERROR_LIMIT new errors in one cycle. - HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true - -# Primary NIC operstate — detects NIC going down (physical or driver failure). -# Uses HOST1_SYS_WATCHDOG_NIC above. Strike system — brief flaps don't trigger reboot. - HOST1_SYS_WATCHDOG_CHECK_NETWORK=true - -# sshd running check — attempts restart before escalating. -# sshd down = no remote access. Script tries rc.sshd start, notifies, strikes on failure. - HOST1_SYS_WATCHDOG_CHECK_SSHD=true - -# Runaway process detection — single process above SYS_WATCHDOG_RUNAWAY_CPU_PCT sustained. -# DISABLED — Tdarr encoding and Emby transcoding legitimately peg CPU for extended periods. -# Enable only if HOST1 has no CPU-intensive workloads. - HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false - -# ============================================================================================== -# ── RESOURCE MANAGER ────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Containers to manage under pressure — see master.conf RW_CRITICAL_CONTAINERS for exclusions. - -# docker pause at medium pressure (RAM < RW_RAM_MEDIUM_GB or load > medium threshold) -# Suspended in-place — instant to pause/unpause, no state lost, no restart delay. - HOST1_RW_PAUSE_CONTAINERS=( - "Huntarr" # arr search automation — safe to suspend - "Cleanuparr" # download cleanup — safe to suspend - "Healarr" # arr health checks — safe to suspend - "Soularr" # Slskd automation — background only - "ChannelTube" # YouTube archiver — background only - "Pinchflat" # YouTube archiver — background only - ) - -# docker stop at hard pressure (RAM < RW_RAM_HARD_GB) -# Full stop — these are optional/heavy services that free significant RAM when stopped. -# resource_watchdog.sh restarts them when pressure fully clears (RAM >= RW_RAM_RECOVER_GB). - HOST1_RW_STOP_CONTAINERS=( - "LocalAI" # GPU/CPU heavy — largest RAM consumer when idle - "7DaysToDie" # game server — optional - "V-Rising" # game server — optional - "Code-Server" # IDE — not needed during pressure events - ) - -# ============================================================================================== -# ──────────────────────── End Of HOST1 Variables ────────────────────────────────────────────── -# ============================================================================================== \ No newline at end of file diff --git a/Deployment/conf_templates/master.conf b/Deployment/conf_templates/master.conf deleted file mode 100644 index ba51bf7..0000000 --- a/Deployment/conf_templates/master.conf +++ /dev/null @@ -1,1433 +0,0 @@ -#!/bin/bash -# ============================================================================================== -# ================================= MASTER CONFIGURATION ======================================= -# ============================================================================================== -# Shared configuration for the unRAID script ecosystem. -# Contains all settings that apply to every server — thresholds, toggles, profiles, job lists. -# -# ── HOW THE THREE-FILE SYSTEM WORKS ────────────────────────────────────────────────────────── -# Scripts source all three files at startup: -# source master.conf ← shared config (this file) -# source host1.conf ← HOST1 credentials, shares, container lists -# source host2.conf ← HOST2 credentials, shares, container lists -# -# Sparse checkout (git) ensures each server only pulls its own host*.conf. -# HOST2 never sees HOST1 credentials. HOST1 never sees HOST2 credentials. -# -# What belongs here: thresholds, toggles, intervals, profiles, job lists -# What belongs in HOST*: hostnames, SSH keys, API keys, passwords, share paths, -# container names, failover lists, watchdog containers -# -# ── HOW THIS FILE WORKS ─────────────────────────────────────────────────────────────────────── -# Every script sources all three conf files and common.sh at startup. -# Change a value here and it affects all scripts that use it — no hunting through files. -# To disable something: comment it out with # rather than deleting it. -# To add a new rsync profile: add a key to each PROFILE_* array in the RSYNC section. -# To add or remove orchestrator jobs: edit the arrays in the ORCHESTRATORS section. -# -# ── INDEX ───────────────────────────────────────────────────────────────────────────────────── -# -# Section Description -# ─────────────────────────────────────────────────────────────────────────────────────────── -# SHARED HOST CONFIGURATION DATA_DIR and shared connection settings -# PARTNERSHIP Mirror relationship lifecycle — onboard/offboard/transfer -# LOGGING Enable or disable verbose logging -# NOTIFICATIONS unRAID native and Discord webhook settings -# GIT / REPO Gitea repository and SSH settings -# -# ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────── -# ARRAY START Scripts launched at array start (array_started.sh) -# ARRAY STOP Scripts run at planned shutdown (array_stopping.sh) -# WATCHDOG ORCHESTRATOR Per-minute watchdog runner (watchdog_orchestrator.sh) -# SYSTEM WATCHDOG Sub-scripts called by watchdog_orchestrator.sh -# CRITICAL SYNC MAINTENANCE 30-minute jobs + sync shares + partnership check (critical_sync_maintenance.sh) -# INTERMEDIATE SYNC MAINTENANCE Arr sync + optional mid-day rsync (intermediate_sync_maintenance.sh) -# DAILY SYNC MAINTENANCE Job list + media shares (daily_sync_maintenance.sh) -# WEEKLY SYNC MAINTENANCE Job list + sync shares + update toggles (weekly_sync_maintenance.sh) -# MONTHLY MAINTENANCE Uptime-triggered heavy tasks — ZFS scrub, SMART tests (monthly_maintenance.sh) -# -# ── RSYNC ────────────────────────────────────────────────────────────────────────────────── -# RSYNC ENABLE/DISABLE Two-tier toggle system — global gate + per-orchestrator -# RSYNC DEFAULTS Global fallback rsync options and limits -# REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort -# RSYNC PROFILE SYSTEM Per-profile overrides for appdata syncs -# -# ── FALLBACK ─────────────────────────────────────────────────────────────────────────────── -# FALLBACK Mutual container failover shared settings -# FALLBACK TEST Simulated outage settings for fallback_test.sh -# -# ── DOCKER ESSENTIALS ────────────────────────────────────────────────────────────────────── -# DOWNLOADERS RESET Retention and thresholds for slskd, SABnzbd, qBittorrent -# DOCKER DAILY RESTART Containers restarted daily -# DOCKER WEEKLY RESTART Containers restarted weekly -# DOCKER WATCHDOG Thresholds and toggles for container monitoring -# DOCKER NETWORK CONNECT Networks to ensure + containers to connect -# -# ── UNRAID ESSENTIALS ────────────────────────────────────────────────────────────────────── -# INOTIFY TUNING inotify limits — raised at array start by inotify_tuning.sh -# SYSTEM TUNING MONITOR Tracks inotify + php-fpm usage over time -# REBOOT User warning delay before scheduled reboot -# MOVER Mover stop timeout -# SYSLOG FILTER Docker veth noise filter file path -# PHP-FPM PHP-FPM max children config -# CLEAR LOGS System log file paths -# WEBGUI WATCHDOG WebGUI nginx + emhttp monitoring and restart -# -# ── MEDIA ────────────────────────────────────────────────────────────────────────────────── -# MEDIA PERMISSIONS Share list, mode and owner for permissions script -# MEDIA CLEANER Anime and media folder lists and file patterns -# ARR CLEANUP Lidarr, Sonarr, Radarr orphan file cleanup settings -# ARR FAILED/STALLED RECOVERY Auto blocklist + re-search settings -# -# ── TRANSCODES ───────────────────────────────────────────────────────────────────────────── -# TRANSCODE MANAGER Ramdisk and SSD fallback transcode management -# TRANSCODE SERVER ARRAY Multi-server session monitoring (Emby, Jellyfin, Plex) -# -# ── MONITORS ─────────────────────────────────────────────────────────────────────────────── -# CERTIFICATE MONITOR SSL certificate expiry monitoring -# BACKUP VERIFY Random sample checksum verification against remote -# SMART HEALTH Drive SMART attribute monitoring -# ZFS MEMORY SNAPSHOT Weekly ZFS health and memory diagnostic report -# BANDWIDTH MONITOR Daily rsync transfer logging and weekly summary -# HEALTH DIGEST Aggregated system health digest — always/smart/weekly -# EMBY SESSION REPORT Weekly Emby usage statistics via API -# -# ── SYSTEM WATCHDOG ──────────────────────────────────────────────────────────────────────── -# SYSTEM WATCHDOG Continuous system health monitoring — last line of defense -# -# ============================================================================================== - -# ============================================================================================== -# ── HOST IDENTITIES ─────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Hostnames for every server in the ecosystem — not credentials, safe for all machines. -# Must match the exact unRAID hostname AND Tailscale device name (case sensitive). -# detect_hosts() in common.sh matches the local hostname against these to set MY_ID / REMOTE_ID. -# Tailscale IP resolution uses these names — no hardcoded IPs needed. -# To add a new server: add HOST3="unRAID-NewServer" here + create host3.conf. - HOST1="unRAID-Gmer4Lfe" - HOST2="unRAID-Jayred365" - -# ============================================================================================== -# ── SHARED HOST CONFIGURATION ───────────────────────────────────────────────────────────────── -# ============================================================================================== -# Requirement: internal NVMe/SSD boot drive. Everything lives in the plugin folder on flash — -# scripts, state, and data are all available before the array mounts. -# Both directories are created automatically if they don't exist. -# -# DATA_DIR — historical logs, statistics, discovery histories, blocklists -# STATE_DIR — runtime state files for all scripts (watchdogs, fallback, transcode, etc.) -# Requirement: ALL state files MUST use $STATE_DIR. No /tmp, no /boot/config root. - DATA_DIR="/boot/config/plugins/varaverk/data" - STATE_DIR="/boot/config/plugins/varaverk/State_Files" - -# ── Version Parity ── -# Controls behaviour when local and remote unRAID versions differ. -# Major version mismatch always aborts regardless of this setting. -# "warn" — log warning and continue (safe for minor/patch differences) -# "abort" — refuse to continue (strict — ensures both sides always match) - UNRAID_VERSION_MISMATCH_ACTION="warn" - -# ── Arr Sync ── -# arr_sync.sh syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally. -# Runs before rsync — all nodes agree on tracked library before files are transferred. -# Remote API keys are read live from each node's config.xml via SSH — never stored here. - ARR_SYNC_ENABLED=true - ARR_SYNC_BLOCKLIST="${DATA_DIR}/arr_sync_blocklist.tsv" - ARR_SYNC_CONNECT_TIMEOUT=10 # seconds — SSH connect timeout per node - ARR_SYNC_API_TIMEOUT=60 # seconds — curl timeout for library fetches - DOCKER_APPDATA_BASE="/mnt/user/appdata" - ARR_SYNC_LIDARR_PORT=8686 - ARR_SYNC_SONARR_PORT=8989 - ARR_SYNC_RADARR_PORT=7878 - -# ── Remote Docker Daemon ── -# Strike system for remote Docker daemon health checks. -# Scripts that issue remote container commands check the remote daemon first. -# Below limit → skip operation this run | At limit → notify critical + exit - REMOTE_DOCKER_STRIKE_LIMIT=3 # consecutive failures before critical notify - -# ============================================================================================== -# ── PARTNERSHIP ─────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Manages the relationship lifecycle between two unRAID servers. -# HOST1 is always the owner (source of truth) — HOST2 is always the mirror. -# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a --transfer operation. -# All identity vars (hostnames, SSH keys) live in host*.conf. -# Hostnames already match Tailscale device names — IP resolution is automatic. -# -# State files in STATE_DIR — survives reboots, array must be up (scripts run after array start): -# STATE_DIR/partnership_HOST1.db ← HOST1 writes only -# STATE_DIR/partnership_HOST2.db ← HOST2 writes only -# STATE_DIR/partnership_blocklist.db ← shared blocklist -# STATE_DIR/varaverk_setup.db ← onboarding progress state -# Propagated via SSH — no rsync needed -# -# critical_sync_maintenance.sh runs --check every 30min: -# Reads both state files via SSH -# Detects offboard requests → finalises from owner side -# Increments offline counter → auto-offboards after threshold -# Silent when healthy ✅ -# -# See README-Partnership.md for full lifecycle documentation. - - PARTNERSHIP_ENABLED=false - PARTNERSHIP_OWNER_HOST="HOST1" # "HOST1" or "HOST2" — flips on --transfer - -# Auth containers reconfigured on onboard/offboard — defined per host in host*.conf. -# Format: "ContainerName|WebUIPort" -# HOST1_PARTNERSHIP_AUTH_WEBUIS / HOST2_PARTNERSHIP_AUTH_WEBUIS -# On onboard → WebUI pointed at owner's Tailscale IP -# On offboard → WebUI pointed back at localhost - -# Paths to collect during the grace window after offboard — defined per host in host*.conf. -# HOST1_PARTNERSHIP_MIRROR_BACKUPS / HOST2_PARTNERSHIP_MIRROR_BACKUPS -# Notified on offboard — no auto-deletion, manual collection. - -# Timing — single var controls both Tailscale removal and backup access expiry. -# Both expire at the same time — keeping backups accessible beyond Tailscale removal is pointless. - PARTNERSHIP_GRACE_HOURS=6 # hours after offboard before Tailscale removal - # backup access expires at the same time - PARTNERSHIP_OFFLINE_THRESHOLD=30 # days either server unreachable before auto-offboard - # works both directions independently - -# Partnership and setup state files — all in STATE_DIR per the project requirement. -# Scripts use these variables; do not hardcode /boot/config paths in scripts. - PARTNERSHIP_BLOCKLIST_FILE="$STATE_DIR/partnership_blocklist.db" - VARAVERK_SETUP_FILE="$STATE_DIR/varaverk_setup.db" - -# Tailscale removal on offboard. - PARTNERSHIP_REMOVE_TAILSCALE=true # remove mirror from Tailscale tailnet on offboard - # false = skip removal (manual or testing) - -# Tailscale API — required when PARTNERSHIP_REMOVE_TAILSCALE=true. -# Stays in shared conf — only owner uses it, and owner is always running this script. -# API key: https://login.tailscale.com/admin/settings/keys → Devices write scope - TAILSCALE_API_KEY="" # tskey-api-... - TAILSCALE_TAILNET="" # your tailnet name (e.g. yourname.github) - -# Transfer safety. - PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership" - PARTNERSHIP_TRANSFER_STRIKES=3 # consecutive health checks required - PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20 # max health check attempts before giving up - -# Onboard settings. - PARTNERSHIP_ONBOARD_VERIFY=true # verify WebUI reachable after reconfiguration - PARTNERSHIP_ONBOARD_NOTIFY=true # notify both servers on completion - PARTNERSHIP_SYNC_INTERVAL=15 # minutes — informational, actual schedule in cron - -# SSH key lifecycle — managed by Partnership/ssh_setup.sh. -# Key named after this server: hostname lowercased, unraid- prefix stripped. -# unRAID-Gmer4Lfe → /root/.ssh/gmer4lfe_rsync_automation -# Run Partnership/partnership_onboard.sh to generate, copy, and update conf automatically. - SSH_MAX_STRIKES=5 # consecutive SSH auth failures before critical notify - SSH_STRIKE_RESET_HRS=24 # hours since last failure before strike counter resets - -# FolderView3 plugin integration — chodeus/VladoPortos/scolcipitato. -# Creates a "{Partner}-Failover" folder on onboard, removes + cleans containers on offboard. -# Folder name derived from remote hostname (strip unraid- prefix case-insensitively). -# Plugin config: /boot/config/plugins/folder.view3/docker.json - PARTNERSHIP_FOLDERVIEW3=true # create/remove FolderView3 folder on onboard/offboard - PARTNERSHIP_FOLDERVIEW3_URL="" # CA plugin URL — leave empty to skip auto-install - -# ============================================================================================== -# ── LOGGING ─────────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# Controls verbose [LOG] output across all scripts. -# true = show detailed [LOG] lines — useful for debugging or first-time setup -# false = show only user-facing output — cleaner for scheduled runs - ENABLE_LOGGING=false - -# ============================================================================================== -# ── NOTIFICATIONS ───────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# unRAID native notification system — integrates with the bell icon in the WebGUI. -# normal = job completed successfully / warning = something failed or needs attention - NOTIFY_UNRAID=true - -# Discord webhook URL — defined per host in host*.conf. -# HOST1_DISCORD_WEBHOOK / HOST2_DISCORD_WEBHOOK -# Allows different webhooks per server, or only one server notifying. - -# ============================================================================================== -# ── GIT / REPO ──────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# Gitea self-hosted repository — used by git_pull_execute.sh. -# Detects Gitea container location at runtime — works through failover automatically. -# Falls back to GITEA_DOMAIN if local and Tailscale both fail. - GITEA_CONTAINER="Gitea" - GITEA_REPO_PATH="FailedProxy/Varaverk.git" - GITEA_DOMAIN="" # e.g. git.yourdomain.com — requires NPM + DNS - TARGET_DIR="/boot/config/plugins/varaverk" - GITEA_SSH_KEY="/root/.ssh/unraid_gitea" - SSH_PORT=221 # Gitea SSH port (default 22, Gitea often uses 221/222) - GITEA_HTTP_PORT=3000 # Gitea web/API port — used by gitea_ssh_setup.sh - -# ============================================================================================== -# ── ORCHESTRATORS ───────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# All orchestrator job lists live here — edit arrays to add/remove scripts. -# No changes to orchestrator scripts needed when adding or removing jobs. -# Sections ordered by run frequency: array events first, then shortest interval to longest. - -# ━━━ Array Start ━━━ -# Scripts launched by array_started.sh when the array comes online. -# Launched in order — each as a background process. -# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally. -# Continuous scripts (failover) run until array stops. -# Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via -# watchdog_orchestrator.sh — NOT launched here. - ARRAY_START_SCRIPTS=( - "Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts - "System_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill - "Plugin/unraid/System_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning - "System_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted - "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers - "Fallback/fallback.sh" # mutual failover — continuous - ) - -# ━━━ Array Stop ━━━ -# Scripts run by array_stopping.sh for a planned shutdown — stops everything cleanly in order. -# Run sequentially (foreground) — each must complete before the next starts. -# Order matters: user scripts first (prevents new ops), then data movement, then containers. - ARRAY_STOP_SCRIPTS=( - "Plugin/unraid/System_Essentials/user_scripts_stop.sh" # stop background scripts before they start new ops - "Fallback/fallback.sh --stop" # gracefully stop fallback (not caught by user_scripts_stop) - "System_Essentials/rsync_stop.sh --rsync-only" # kill rsync; skip container recovery (handled below) - "Plugin/unraid/System_Essentials/mover_stop.sh" # stop mover after rsync (they conflict on same files) - "Docker_Essentials/docker_container_stop.sh" # stop all containers last - ) - -# ━━━ Watchdog Orchestrator ━━━ -# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle. -# Schedule: * * * * * (every minute) -# NOT in ARRAY_START_SCRIPTS — has its own cron entry. -# Order matters — resource first (frees pressure), docker second (heals with freed resources), -# system third (storage + webgui component health), stability last (last line of defense). - WATCHDOG_ORCHESTRATOR_SCRIPTS=( - "Watchdogs/resource_watchdog.sh" # reduce system pressure before healing attempts - "Watchdogs/docker_watchdog.sh" # heal containers with freed resources - "Watchdogs/system_watchdog.sh" # system component health — storage + webgui - "Watchdogs/stability_watchdog.sh" # reboot if all else fails — last line of defense - ) - -# ━━━ System Watchdog ━━━ -# system_watchdog.sh runs SYSTEM_WATCHDOG_SCRIPTS sequentially each cycle. -# Called by watchdog_orchestrator.sh — not scheduled directly. - SYSTEM_WATCHDOG_SCRIPTS=( - "Watchdogs/System/storage_watchdog.sh" # pool growth + runaway log detection - "Plugin/unraid/Watchdogs/System/webgui_watchdog.sh" # WebGUI availability — nginx → php-fpm → emhttp - "Watchdogs/System/network_watchdog.sh" # internet, DDNS, Tailscale, NPM proxy - ) - -# ━━━ Critical Sync Maintenance ━━━ -# critical_sync_maintenance.sh runs every 30 minutes. -# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check -# partnership --check always runs last regardless of rsync gate. -# Comment out entries to disable without removing. - CRITICAL_MAINTENANCE_SCRIPTS=( - "Docker_Essentials/downloaders_reset.sh" # clear stuck download states every 30min - ) - -# Shares synced every 30 minutes — defined per host in host*.conf. -# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES -# Format: "/path/to/share" or "/path/to/share|profile-name" -# Order matters — Critical-Data first (auth stack), then Emby dirty sync. - -# ━━━ Intermediate Sync Maintenance ━━━ -# intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch, -# and optional mid-day rsync for any shares that need sub-daily propagation. -# Schedule: 0 */4 * * * - # INTERMEDIATE_SYNC_SHARES is host-specific — configure HOST*_INTERMEDIATE_SYNC_SHARES in host*.conf. - - INTERMEDIATE_MAINTENANCE_SCRIPTS=( - "Media/arrs_failed_stalled_recovery.sh" # blocklist + re-search failed/stalled arr queue items - ) - # arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here. - # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above). - -# ━━━ Daily Sync Maintenance ━━━ -# daily_sync_maintenance.sh runs media share sync first, then iterates -# DAILY_MAINTENANCE_SCRIPTS for all jobs. -# Schedule: 0 1 * * * (1am daily) - DAILY_MAINTENANCE_SCRIPTS=( - "git_pull_execute.sh" # pull latest scripts — always runs first - "Media/media_shares_permissions.sh" # apply permissions — runs before cleaners - "Media/media_cleaner.sh anime" # remove junk from anime shares - "Media/media_cleaner.sh media" # remove junk from media shares - #"Media/lidarr_cleanup.sh" # remove orphaned music files — enable when ready - #"Media/sonarr_cleanup.sh" # remove orphaned TV files — enable when ready - "Media/radarr_cleanup.sh" # remove orphaned movie files - "Media/lidarr_missing_art.sh" # fetch missing album/artist artwork (HOST1 only — self-guards) - "Media/radarr_tmdb_removed.sh" # remove movies dropped from TMDb - "Media/sonarr_tvdb_removed.sh" # remove series dropped from TVDB - "Docker_Essentials/docker_update.sh" # pull container image updates before restart - "Docker_Essentials/docker_daily_restart.sh" # daily container restarts — runs last - ) - -# Pull latest images for DAILY_RESTART_CONTAINERS before the daily restart. -# Containers keep running during pull — no extra downtime. -# Set false to skip updates while still running the daily restart. - DAILY_CONTAINER_UPDATES=true - -# Media shares synced daily by daily_sync_maintenance.sh. -# Defined per-host in host*.conf — HOST1_DAILY_SYNC_SHARES and HOST2_DAILY_SYNC_SHARES. -# Mesh model: every node pushes every media share. rsync has no --delete so pushes are additive. -# arr_sync (union) ensures all arr libraries converge first. arr_cleanup removes true orphans. -# Any node can download content to any share — it propagates to all nodes on the next cycle. -# Adding HOST3: list every media share in HOST3_DAILY_SYNC_SHARES. No ownership to track. -# These shares use DEFAULT_RSYNC_OPTS — no profile entry needed. -# For shares needing custom options or container stops — create a profile in RSYNC section. - -# Personal encrypted shares defined per-host in host*.conf. -# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content. -# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting. - -# ━━━ Weekly Sync Maintenance ━━━ -# weekly_sync_maintenance.sh stops containers both sides → pulls updates → -# syncs WEEKLY_SYNC_SHARES → restarts → then iterates WEEKLY_MAINTENANCE_SCRIPTS. -# Schedule: 30 2 * * 0 (Sunday 2:30am) - WEEKLY_MAINTENANCE_SCRIPTS=( - "Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync - "Docker_Essentials/docker_update_remaining.sh" # pull updates for all other containers - "System_Essentials/clear_logs.sh" # purge aged logs — Sunday only, low priority - "Media/playback_aware_lidarr_discovery.sh" # behavior-driven music discovery using weekly Emby playback history - "Media/playback_aware_radarr_discovery.sh" # behavior-driven movie discovery using TMDB recommendations - "Media/playback_aware_sonarr_discovery.sh" # behavior-driven TV discovery using TMDB recommendations - ) - -# Pull updates for all running containers NOT in daily/weekly restart lists. -# Ensures every deployed container receives at least one image pull per week. -# Set false to skip — docker_weekly_restart.sh still runs regardless. - WEEKLY_REMAINING_UPDATES=true - -# Shares synced during the weekly maintenance window — defined per host in host*.conf. -# HOST1_WEEKLY_SYNC_SHARES / HOST2_WEEKLY_SYNC_SHARES -# Containers stopped both sides before sync — full clean state guaranteed. -# Profiles drive container stops, excludes, and options — configure in RSYNC section. -# Order matters — Emby first (larger), then Critical-Data (auth stack). - -# Container update toggles for the weekly sync window. -# Containers already stopped for sync — updates pull at no extra downtime. -# Both false → sync only, no updates. - WEEKLY_SYNC_UPDATES=true # pull container updates locally during weekly window - WEEKLY_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH - -# ━━━ Monthly Maintenance ━━━ -# monthly_maintenance.sh fires only when BOTH gates pass: -# 1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days -# 2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run) -# Cron: 0 3 * * * (daily 3am check — script self-gates, calling daily is safe) -# NOT in WATCHDOG_ORCHESTRATOR_SCRIPTS — has its own cron entry. -# Add scripts that require a long-stable settled system — scrubs, extended drive tests. -# State file on /boot/config — survives reboots (interval gate independent of uptime gate). - MONTHLY_MAINTENANCE_SCRIPTS=( - #"Tools/zfs_pool_scrub.sh" # ZFS pool integrity scrub — not yet built - #"Tools/smart_long_test.sh" # SMART extended drive health test — not yet built - ) - - MONTHLY_UPTIME_THRESHOLD_DAYS=30 # minimum uptime in days before maintenance fires - MONTHLY_RUN_INTERVAL_DAYS=30 # minimum days since last run before running again - MONTHLY_LAST_RUN_FILE="$STATE_DIR/monthly_maintenance_last_run.db" - -# ━━━ Sunday Morning Coffee Report ━━━ -# Orchestrator that runs all Sunday monitor scripts in sequence. -# Schedule: 0 7 * * 0 (Sunday 7am — after weekly_sync_maintenance.sh finishes at ~3am) -# Each script runs independently and notifies on its own findings. - COFFEE_REPORT_SCRIPTS=( - "Monitors/zfs_memory_snapshot.sh" # ZFS pool health + ARC + Docker memory snapshot - "Monitors/smart_health.sh" # drive SMART attributes — reallocated, pending, temp - "Monitors/cert_monitor.sh" # SSL certificate expiry for all configured domains - "Monitors/backup_verify.sh" # rsync mirror integrity via independent MD5 checksums - "Monitors/bandwidth_monitor.sh" # weekly rsync transfer totals and per-share breakdown - "Monitors/emby_session_report.sh" # Emby usage — streams, users, library, transcode ratio - "Monitors/weekly_health_digest.sh" # aggregated digest — watchdogs, fallback, skip list - ) - -# ============================================================================================== -# ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ Rsync Enable/Disable ━━━ -# Two-tier toggle system — Tier 1 overrides Tier 2. -# -# Tier 1 — Global gate: -# RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions -# Use when: remote completely offline, major maintenance, disaster recovery -# -# Tier 2 — Per-orchestrator (only applies when Tier 1 is true): -# Fine grained control — disable specific orchestrators while keeping others -# Use when: rebuilding secondary, testing, per-window bandwidth management -# -# Example — HOST2 data rebuild: -# RSYNC_ENABLED=true ← rsync works, individual scripts run fine -# CRITICAL_RSYNC_ENABLED=true ← 30min auth stack sync still runs -# INTERMEDIATE_RSYNC_ENABLED=false ← skip 4h arr/mid-day rsync during rebuild -# DAILY_RSYNC_ENABLED=false ← skip daily HDD syncs during rebuild -# WEEKLY_RSYNC_ENABLED=true ← Emby + Critical-Data still sync (NVMe) -# FALLBACK_RSYNC_ENABLED=true ← handback writeback still works -# → Run individual: bash Rsync/rsync.sh /mnt/user/Movies -# → When ready: INTERMEDIATE_RSYNC_ENABLED=true DAILY_RSYNC_ENABLED=true - RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below - CRITICAL_RSYNC_ENABLED=false # Tier 2 — disabled during HOST2 rebuild, re-enable when ready - INTERMEDIATE_RSYNC_ENABLED=true # Tier 2 — intermediate_sync_maintenance.sh rsync section - DAILY_RSYNC_ENABLED=false # Tier 2 — HOST2 rebuild in progress, re-enable when ready - WEEKLY_RSYNC_ENABLED=true # Tier 2 — weekly_sync_maintenance.sh rsync section - FALLBACK_RSYNC_ENABLED=true # Tier 2 — fallback.sh writeback jobs on handback - -# ━━━ Rsync Defaults ━━━ -# Global fallback values used when no profile match is found. -# Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed. -# Appdata shares match profiles by directory basename (lowercased). - BW_LIMIT=12500 # KB/s — 12500 ≈ 100Mbit - RETRY_COUNT=3 # retry attempts before giving up - SLEEP=300 # seconds between retry attempts - CRITICAL_CONTAINER_NAMES=() # containers stopped on REMOTE before rsync — profiles override - DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override - CONTAINER_DELAY=5 # seconds before starting delayed containers - EXCLUDE_DIRS=() # directories excluded from transfer — profiles override - -# --inplace writes directly to destination — delta against existing file, better for large media -# --partial keep partial file on interrupted transfer so next run resumes, not re-transfers -# --timeout kill stalled transfers instead of hanging indefinitely -# --numeric-ids use UIDs/GIDs numerically — prevents ownership mismatches between servers -# --delete intentionally omitted — arr_cleanup.sh enforces media truth post-rsync. -# Media shares use this default; rsync spreads files only, never removes them. -# Note: --no-whole-file removed — redundant over SSH (delta transfer is already the default). -# Note: arr_cleanup only catches true orphans under the union model — intentional removals -# require arr_sync.sh --blocklist-add first, then manual file deletion. - DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --inplace --partial --timeout=60 --numeric-ids) - -# ━━━ Remote Health Checks ━━━ -# Pre-flight — aborts if remote rootfs (/) usage is at or above this percentage. -# When remote array is down, rsync writes land on rootfs and fill it rapidly. - ROOTFS_WARN=75 - -# ━━━ Rsync Profile System ━━━ -# Profiles allow per-share rsync behaviour without touching script logic. -# Profile key matched by basename of directory passed to rsync.sh (lowercased). -# Override with --profile=name flag. -# -# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit DEFAULT_RSYNC_OPTS. -# List ALL desired options explicitly when defining a profile. -# -# Current profiles: -# arrs_stack — arr databases — lower bandwidth, containers stopped for consistency -# critical-data — auth stack — full stop both sides, Authelia delayed start -# called by weekly_sync_maintenance.sh — full clean sync weekly -# critical-fallback — dirty sync — auth stays running both sides, WAL excluded -# called by critical_sync_maintenance.sh every 30min -# host1-appdata — HOST1 server-specific appdata — defined in host1.conf -# host2-appdata — HOST2 server-specific appdata — defined in host2.conf -# important-data — NextCloud + Postgres — NextCloud delayed start after Postgres -# emby — weekly clean sync — both Emby stopped, full mirror -# called by weekly_sync_maintenance.sh only — do NOT schedule separately -# emby-fallback — dirty sync — Emby stays running, WAL excluded -# called by critical_sync_maintenance.sh every 30min - - declare -A PROFILE_RSYNC_OPTS=( - [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" - [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete" - [critical-fallback]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" - [important-data]="-av --human-readable --bwlimit=$BW_LIMIT" - [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" - [emby-fallback]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" - ) - -# Per-profile bandwidth limits in KB/s - declare -A PROFILE_BW_LIMIT=( - [arrs_stack]=5000 # lower — runs alongside other syncs - [critical-data]=9500 # high — small dataset, sync fast - [critical-fallback]=9500 # high — small dataset, sync fast - [important-data]=9500 # high — database sync - [emby]=8000 # medium — large full mirror - [emby-fallback]=9500 # high — small critical dataset - ) - -# Retry attempts per profile - declare -A PROFILE_RETRY_COUNT=( - [arrs_stack]=3 - [critical-data]=3 - [critical-fallback]=3 - [important-data]=3 - [emby]=3 - [emby-fallback]=3 - ) - -# Seconds between retry attempts - declare -A PROFILE_SLEEP=( - [arrs_stack]=300 - [critical-data]=300 - [critical-fallback]=120 # shorter — frequent dirty sync, retry faster - [important-data]=300 - [emby]=300 - [emby-fallback]=120 # shorter — frequent dirty sync, retry faster - ) - -# Containers stopped on BOTH LOCAL and REMOTE before rsync. -# Local stops first — flushes databases cleanly. Remote stops next — prevents writes. -# Only running containers get restarted — stopped containers stay stopped. -# SPACE-SEPARATED STRINGS — converted to array at runtime - declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( - [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" - [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary" - [critical-fallback]="" # dirty sync — auth stays running both sides - [important-data]="Postgres-NextCloud NextCloud" - [emby]="Emby" - [emby-fallback]="" # dirty sync — Emby stays running both sides - ) - -# Containers needing a delay after rsync before starting. -# SPACE-SEPARATED STRINGS — converted to array at runtime - declare -A PROFILE_DELAYED_CONTAINERS=( - [arrs_stack]="" - [critical-data]="Authelia Authelia-Secondary" # wait for Mariadb + Redis - [critical-fallback]="" - [important-data]="NextCloud" # wait for Postgres - [emby]="" - [emby-fallback]="" - ) - -# Seconds before starting delayed containers - declare -A PROFILE_CONTAINER_DELAY=( - [arrs_stack]=5 - [critical-data]=15 # Mariadb + Redis need time to accept connections - [critical-fallback]=5 - [important-data]=10 # Postgres needs time before NextCloud - [emby]=5 - [emby-fallback]=5 - ) - -# Directories excluded from rsync per profile. -# SPACE-SEPARATED STRINGS — converted to array at runtime - declare -A PROFILE_EXCLUDE_DIRS=( - [arrs_stack]="logs *.tmp" - [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt" - [critical-fallback]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt *.db-wal *.db-shm" - [important-data]="logs *.tmp" - [emby]="logs transcodes cache crash*" - [emby-fallback]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root" - ) - -# Remote restart after dirty sync — restart these on remote IF they were running before sync. -# Same logic as stop/start — was stopped = stays stopped, was running = gets restarted. -# Used by dirty sync profiles (critical-fallback, emby-fallback) so remote picks up changes. -# SPACE-SEPARATED STRINGS — converted to array at runtime - declare -A PROFILE_REMOTE_RESTART_CONTAINERS=( - [critical-fallback]="NginxProxyManager Authelia Authelia-Secondary Lldap-Gmer4Lfe Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary" - [emby-fallback]="Emby" - ) - -# Note: disk check is auto-detected from disks.ini — no PROFILE_SKIP_DISK_CHECK needed. -# check_remote_disks() reads fsType per disk and handles XFS, ZFS, and cache pools automatically. - -# ============================================================================================== -# ── FALLBACK ────────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Mutual container failover between two unRAID servers. -# Each server runs Fallback/fallback.sh independently via array_started.sh. -# All decisions based on two pings: remote reachable + internet reachable. -# -# States: NORMAL | FALLBACK | NO_INTERNET | DARK -# -# DDNS rules — absolute: -# Internet loss → stop own DDNS immediately -# Failover → start remote DDNS first (Tier 1) -# Handback → stop remote DDNS → rsync → start containers → start local DDNS last -# -# Per-host container lists and tier delays live in host*.conf. -# Shared settings (intervals, state file, thresholds) live here. - - EXTERNAL_IP="8.8.8.8" - FALLBACK_CHECK_INTERVAL=30 # seconds between fallback state checks - FALLBACK_HANDBACK_STRIKES=3 # consecutive healthy checks before initiating handback (3×30s = 90s) - FALLBACK_STATE_FILE="$STATE_DIR/fallback_state.db" - FALLBACK_ENABLED=true # HOST2 back online - # false = suppresses "not running" warnings in status scripts - FALLBACK_PARTNERSHIP_REQUIRED=true # gate fallback on an active partnership - # false = standalone mode, no partnership dependency - FALLBACK_PARTNERSHIP_SUSPEND_AFTER=120 # minutes without active partnership before suspending - # 0 = suspend immediately when partnership goes inactive - -# ━━━ Failover Test ━━━ -# Controlled simulation of a failover event — run manually via fallback_test.sh. - FALLBACK_TEST_BLOCK_WAIT=150 # seconds to wait after blocking connectivity - FALLBACK_TEST_HANDBACK_WAIT=360 # seconds to wait before initiating handback - -# ============================================================================================== -# ── DOCKER ESSENTIALS ───────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ Downloaders Reset ━━━ -# Runs every 30 minutes via CRITICAL_MAINTENANCE_SCRIPTS. -# Clears stuck states, purges old history, prepares each download client for a clean cycle. -# Per-host URLs and API keys live in host*.conf. - DOWNLOADER_RETENTION_DAYS=7 # days — purge history older than this - -# qBittorrent failsafe — removes torrents older than threshold regardless of ratio -# deleteFiles=false — removes from qBit but leaves files on disk (arr manages files) - QBIT_FAILSAFE_MIN_DAYS=180 # days — minimum age before failsafe deletion - QBIT_FAILSAFE_MIN_RATIO=0 # 0 = age only, no ratio requirement - -# ━━━ Docker Daily Restart ━━━ -# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS. -# Per-host lists live in host*.conf: -# HOST1_DAILY_RESTART_CONTAINERS -# HOST2_DAILY_RESTART_CONTAINERS -# detect_hosts() sets DAILY_RESTART_CONTAINERS to the correct host array at runtime. - -# ━━━ Docker Weekly Restart ━━━ -# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am). -# Containers already stopped for weekly sync — restart adds zero extra downtime. -# Per-host lists live in host*.conf: -# HOST1_WEEKLY_RESTART_CONTAINERS -# HOST2_WEEKLY_RESTART_CONTAINERS - -# ━━━ Docker Watchdog ━━━ -# Continuous two-tier self-healing container monitoring. -# Started by array_started.sh — runs until array stops. -# Re-sources all three conf files each cycle — add/remove containers without restarting watchdog. -# -# Tier 1 — strict monitoring of explicitly configured containers: -# Memory hard limits — immediate restart if exceeded -# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT sustained strikes -# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT failed checks -# Required containers — must always be running, strike + skip list with auto-clear -# -# Tier 2 — global health scan of ALL running containers: -# Unhealthy status — Docker HEALTHCHECK unhealthy → restart -# OOM killed — kernel killed → restart + notify -# Crash loop detection — RestartCount climbing → notify, critical above limit -# Dead containers — remove and restart -# Unexpected exits — non-zero exit code → restart -# -# All per-host container lists live in host*.conf: -# HOST*_WATCHDOG_CONTAINERS — memory hard limits per container -# HOST*_WATCHDOG_CONTAINER_URLS — HTTP health check URLs -# HOST*_WATCHDOG_REQUIRED_CONTAINERS — must always be running -# HOST*_WATCHDOG_SCAN_IGNORE — skip in Tier 2 scan -# HOST*_WATCHDOG_DEPENDENCIES — dependency ordering for restart decisions - -# Strike state file — persistent in STATE_DIR - WATCHDOG_STATE_FILE="$STATE_DIR/container_watchdog_state.db" - -# CPU thresholds — normalised against total core count at runtime - SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU - HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU - CPU_FAIL_LIMIT=2 # consecutive hard CPU strikes before container restart - -# Memory soft threshold — warn when container reaches this % of its hard limit - SOFT_MEM_THRESHOLD=80 - -# HTTP responsiveness - RESP_FAIL_LIMIT=2 # consecutive failed checks before restart - CURL_TIMEOUT=5 # seconds per check before timeout - -# Tier 2 master toggle - WATCHDOG_SCAN_ALL=true # false = only WATCHDOG_CONTAINERS + required containers - -# Individual Tier 2 check toggles - WATCHDOG_RESTART_UNHEALTHY=true - WATCHDOG_RESTART_DEAD=true - WATCHDOG_RESTART_CRASHED=true - WATCHDOG_NOTIFY_OOM=true - WATCHDOG_NOTIFY_CRASHLOOP=true - -# Crash loop threshold - WATCHDOG_CRASH_LIMIT=5 # RestartCount above this = critical crash loop - -# Startup grace period — skip restarts while system is still booting - WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures - -# Restart loop protection — prevents watchdog from endlessly restarting a broken container - WATCHDOG_CONTAINER_RESTART_LIMIT=3 - WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours - WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db" - -# Notification batching — one summary per cycle instead of one ping per event - WATCHDOG_BATCH_NOTIFY=true - -# Appdata size monitoring — two-part catch-all for runaway growth and oversized log files. -# -# Part 1 — Growth rate (zero-config): -# Reads per-container dir totals each cycle via du, compares to previous cycle. -# Any container growing more than WATCHDOG_APPDATA_GROWTH_GB triggers a focused *.log scan -# inside that container. No per-container config required — new containers are covered -# automatically. Baseline built on first run after boot; growth detection starts cycle 2. -# -# Part 2 — Absolute log size: -# Finds *.log / *.log.* files over WATCHDOG_APPDATA_LOG_MAX_GB across all appdata paths. -# Catches logs that have already stabilised at a large size and are no longer actively growing. -# -# Strike system (reuses existing watchdog infrastructure): -# Strike 1 — warn + notify: condition first detected -# Strike 2 — warn + escalated notify: still present next cycle -# Strike 3 (WATCHDOG_APPDATA_STRIKE_LIMIT) — action cycle: -# If WATCHDOG_APPDATA_TRUNCATE_LOGS=true: truncate *.log files in-place, clear strikes -# If false: critical notify only, strikes held until condition resolves -# Condition resolves (growth stops / log drops below threshold) → strikes auto-clear -# -# HOST*_WATCHDOG_APPDATA_SIZES (in host*.conf) suppresses growth warnings for a -# container until its dir exceeds the configured ceiling. Only needed when a container -# legitimately has large stable data and you want to guarantee it never triggers a false alarm. - WATCHDOG_CHECK_APPDATA=true - WATCHDOG_APPDATA_PATHS=("/mnt/docker-unraid/appdata") - WATCHDOG_APPDATA_GROWTH_GB=2 # flag containers growing more than this per cycle - WATCHDOG_APPDATA_LOG_MAX_GB=2 # flag *.log files exceeding this size (absolute) - WATCHDOG_APPDATA_TRUNCATE_LOGS=false # set true to auto-truncate oversized *.log files on action cycle - WATCHDOG_APPDATA_STRIKE_LIMIT=3 # cycles before action fires (matches existing watchdog pattern) - WATCHDOG_APPDATA_GROWTH_FILE="$STATE_DIR/watchdog_appdata_growth.db" - STORAGE_WATCHDOG_STATE_FILE="$STATE_DIR/storage_watchdog_state.db" - -# ━━━ Docker Network Connect ━━━ -# Ensures custom networks exist and connects containers at array start. -# Runs once via ARRAY_START_SCRIPTS — idempotent, safe to re-run. -# Container and network lists are host-specific — defined in host*.conf: -# HOST1_NETWORK_CONNECT_CONTAINERS / HOST2_NETWORK_CONNECT_CONTAINERS -# HOST1_NETWORK_CONNECT_NETWORKS / HOST2_NETWORK_CONNECT_NETWORKS - -# ============================================================================================== -# ── UNRAID ESSENTIALS ───────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ inotify Tuning ━━━ -# Linux inotify limits — applied at every array start by inotify_tuning.sh. -# Default unRAID values are very low — with many Docker containers watching files -# you can silently exhaust the limit causing containers to miss file events. -# Symptoms: containers miss file events, downloads not detected, library not updated. -# Live TV stutter is a known symptom of inotify exhaustion. -# These settings are lost on reboot — reapplied automatically at array start. - INOTIFY_MAX_INSTANCES=1024 # default: 128 — max inotify instances per user - # 1024 handles ~20-30 containers watching files simultaneously - INOTIFY_MAX_WATCHES=1048576 # default: 8192 — SHARED budget across ALL users/containers - # 524288 (512K) was previous value — raised to 1M (1048576) - # VSCode/Code-Server alone needs ~50K-200K for large workspaces - # with node_modules. All arr containers + Emby + VSCode share - # this budget. 1M is safe on 128GB RAM (~128MB kernel memory) - # If VSCode shows "unable to watch for file changes" → too low - INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — max events queued before dropping - -# ━━━ System Tuning Monitor ━━━ -# Tracks inotify and php-fpm usage over time. -# Snapshot written every 6 hours by system_tuning_monitor.sh. -# Read by sunday_morning_coffee_report.sh for weekly peak/avg/warning summary. - INOTIFY_WARN_PCT=80 # warn if inotify instances exceed this % of limit - PHP_FPM_WARN_PCT=80 # warn if php-fpm workers exceed this % of max_children - TUNING_MONITOR_LOG="$DATA_DIR/system_tuning_history.db" - TUNING_LOG_RETENTION=30 # days before old entries are purged - -# ━━━ Reboot ━━━ -# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots. -# Gives users time to save work — 300s = 5 minutes. - REBOOT_SLEEP=300 - -# ━━━ Mover ━━━ -# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process. -# Gives mover time to finish current file transfer before being interrupted. - MOVER_STOP_TIMEOUT=300 - -# ━━━ Syslog Filter ━━━ -# Path for the rsyslog filter file that suppresses Docker veth interface noise. -# Docker creates a new veth interface for each container — generates hundreds of -# log lines per hour that have no diagnostic value. Filter removes them at source. - FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" - -# ━━━ PHP-FPM ━━━ -# Higher max_children allows more concurrent PHP requests to the unRAID WebGUI. -# Default is very low — increasing it prevents WebGUI slowdowns under load. -# 250 is safe for servers with 32GB+ RAM. - PHP_CONF="/etc/php-fpm.d/www.conf" - PHP_MAX_CHILDREN=250 - -# ━━━ Clear Logs ━━━ -# System log files cleared weekly to prevent rootfs fill over time. -# These grow continuously — without clearing they eventually consume all rootfs space. -# clear_logs.sh runs in WEEKLY_MAINTENANCE_SCRIPTS — Sunday 2:30am. -# -# Size threshold approach — only clear if log exceeds threshold. -# Avoids destroying useful recent diagnostic context when logs are small. -# LOG_MIN_SIZE_MB: skip clearing if log is under this size (not worth clearing) -# LOG_DOCKER_MAX_MB: clear a container log only if it exceeds this size -# Docker logs grow fastest on active containers (Emby, SABnzbd, Sonarr) -# 100MB per container × 30 containers = 3GB before clearing kicks in - LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) - LOG_MIN_SIZE_MB=10 # skip system log if under this size (already small) - LOG_DOCKER_MAX_MB=100 # clear Docker container log only if over this size (MB) - -# ━━━ Network Watchdog ━━━ -# Services-layer connectivity — internet reachability, DDNS sync, Tailscale, NPM proxy. -# Host-specific values (domain, container, NPM URL) live in host*.conf. - NETWORK_WATCHDOG_ENABLED=true - NETWORK_WATCHDOG_INTERNET_URL="https://1.1.1.1" - NETWORK_WATCHDOG_INTERNET_TIMEOUT=5 - NETWORK_WATCHDOG_CHECK_TAILSCALE=true - NETWORK_WATCHDOG_NPM_TIMEOUT=10 - NETWORK_WATCHDOG_NPM_STRIKE_LIMIT=2 - NETWORK_WATCHDOG_NPM_STATE_FILE="$STATE_DIR/network_watchdog_state.db" - -# ━━━ WebGUI Watchdog ━━━ -# Monitors unRAID WebGUI responsiveness — escalates through nginx restart → emhttp restart. -# Separate from docker_watchdog — this monitors the unRAID UI itself, not containers. - WEBGUI_URL="http://localhost" - WEBGUI_TIMEOUT=5 # seconds before curl gives up on WebGUI check - WEBGUI_NGINX_WAIT=15 # seconds after nginx restart before rechecking - WEBGUI_EMHTTP_WAIT=30 # seconds after emhttp restart before rechecking - -# ============================================================================================== -# ── MEDIA ───────────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ Media Permissions ━━━ -# Applied recursively by media_shares_permissions.sh. -# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership. -# -# Why split directory vs file permissions: -# Directories need execute bit to enter — 755 allows owner+group+others to traverse -# Files should NOT be executable — 664 allows owner+group read/write, others read -# This is POSIX best practice for media servers, not a blanket 777 band-aid -# -# Why nobody:users ownership: -# All ecosystem containers run as PUID=99 (nobody) PGID=100 (users) on unRAID -# Files owned by nobody:users are accessible to all containers without world-write -# If this script fixes many files each run → a container has wrong PUID/PGID set -# Fix: add PUID=99 PGID=100 to the container's environment variables -# -# This script runs daily as a failsafe — even with correct container config: -# rsync may bring files with source ownership if not run with --chown -# Manual admin copies create root:root files -# New containers may not have PUID/PGID set yet - PERMISSIONS_DIR_MODE="755" # directories — traverse + list, no world-write - PERMISSIONS_FILE_MODE="664" # files — group read/write, no execute - PERMISSIONS_OWNER="nobody:users" - -# Share list defined per host in host*.conf: -# HOST1_MEDIA_PERMISSION_SHARES / HOST2_MEDIA_PERMISSION_SHARES - -# ━━━ Media Cleaner ━━━ -# Removes junk files from media shares — two profiles: anime and media. -# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media -# Folder lists defined per host in host*.conf: -# HOST1_ANIME_CLEAN_FOLDERS / HOST2_ANIME_CLEAN_FOLDERS -# HOST1_MEDIA_CLEAN_FOLDERS / HOST2_MEDIA_CLEAN_FOLDERS - - ANIME_FILE_PATTERNS=( - # ── Checksums ──────────────────────────────────────────────────────────────────────── - '*.sfv' '*.md5' '*.sha1' # verification files — useless post-download - - # ── Scene / download metadata ───────────────────────────────────────────────────────── - '*.url' '*.lnk' # scene links - '*.info' '*.diz' # scene description files - '*.nzb' # usenet download files - '*.torrent' # torrent files left by download clients - - # ── Archives and segments ───────────────────────────────────────────────────────────── - '*.rar' '*.zip' '*.7z' '*.ace' # archives — source files not needed after extract - '*.r00' '*.r01' '*.r02' '*.r03' # multi-part rar segments - '*.r04' '*.r05' '*.r06' '*.r07' - '*.r08' '*.r09' '*.srr' # scene repair files - '*.001' '*.002' '*.003' # split archive parts - '*.gz' '*.tar' '*.bz2' # linux archives - - # ── Scene samples and proofs ────────────────────────────────────────────────────────── - '*.sample*' '*.proof*' # scene samples — never needed in library - - # ── Executables and scripts ─────────────────────────────────────────────────────────── - '*.exe' '*.scr' '*.com' # Windows executables - '*.bat' '*.cmd' '*.vbs' '*.ps1' # Windows scripts - '*.msi' '*.dll' '*.sys' # Windows installers and system files - '*.sh' # shell scripts in media folders = suspicious - - # ── Incomplete downloads ────────────────────────────────────────────────────────────── - '*.!ut' '*.!qB' # uTorrent / qBittorrent incomplete markers - '*.crdownload' '*.opdownload' # Chrome / Opera incomplete downloads - '*.part' # partial download files - - # ── Sync conflicts ──────────────────────────────────────────────────────────────────── - '*sync-conflict*' # Syncthing conflict copies - ) - - MEDIA_FILE_PATTERNS=( - # ── Checksums ──────────────────────────────────────────────────────────────────────── - '*.sfv' '*.md5' '*.sha1' # verification files — useless post-download - - # ── Scene / download metadata ───────────────────────────────────────────────────────── - '*.url' '*.lnk' # scene links - '*.info' '*.diz' # scene description files - '*.nzb' # usenet download files - '*.torrent' # torrent files left by download clients - - # ── Archives and segments ───────────────────────────────────────────────────────────── - '*.rar' '*.zip' '*.7z' '*.ace' # archives — source files not needed after extract - '*.r00' '*.r01' '*.r02' '*.r03' # multi-part rar segments - '*.r04' '*.r05' '*.r06' '*.r07' - '*.r08' '*.r09' '*.srr' # scene repair files - '*.001' '*.002' '*.003' # split archive parts - '*.gz' '*.tar' '*.bz2' # linux archives - - # ── Scene samples and proofs ────────────────────────────────────────────────────────── - '*.sample*' '*.proof*' # scene samples — never needed in library - - # ── Executables and scripts ─────────────────────────────────────────────────────────── - '*.exe' '*.scr' '*.com' # Windows executables - '*.bat' '*.cmd' '*.vbs' '*.ps1' # Windows scripts - '*.msi' '*.dll' '*.sys' # Windows installers and system files - '*.sh' # shell scripts in media folders = suspicious - - # ── Incomplete downloads ────────────────────────────────────────────────────────────── - '*.!ut' '*.!qB' # uTorrent / qBittorrent incomplete markers - '*.crdownload' '*.opdownload' # Chrome / Opera incomplete downloads - '*.part' # partial download files - - # ── Sync conflicts ──────────────────────────────────────────────────────────────────── - '*sync-conflict*' # Syncthing conflict copies - - # ── Media-specific extras ───────────────────────────────────────────────────────────── - '*.iso' # disc images — extracted content is in library - '*.lrc' # lyrics files — handled by music apps not Emby - ) - -# ━━━ Arr Cleanup ━━━ -# Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs. -# Per-host URLs, API keys, and path maps live in host*.conf. -# detect_hosts() selects correct host vars at runtime. -# -# API versions — update MAJOR version here when script is updated for a new arr version: -# Sonarr v4 → /api/v3/series → /api/v3/episodefile?seriesId=X -# Radarr v6 → /api/v3/movie → /api/v3/moviefile?movieId=X -# Lidarr v3 → /api/v1/artist → /api/v1/trackFile?artistId=X - SONARR_VERSION_MAJOR=4 - RADARR_VERSION_MAJOR=6 - LIDARR_VERSION_MAJOR=3 - -# Lidarr shared settings - LIDARR_LOCK_WARN_AGE=3600 # 1hr — large libraries take time, not stuck - LIDARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion - LIDARR_MAX_DELETE_GB=5 # require --i-know-what-im-doing if deletion exceeds this - LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run - # protects against API returning partial data on a bad day - LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count" - LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") - LIDARR_PROTECTED_PATTERNS=( - # Metadata - "*.nfo" "*.tbn" - # Images — album art, artist images, Emby artwork - "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" - "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" - "banner.*" "thumb.*" "landscape.*" - "folder.*" "cover.*" "album.*" "artist.*" "disc.*" - # Lyrics - "*.lrc" - ) - -# Lidarr artwork fetcher settings - LIDARR_ART_MIN_SIZE=10000 # bytes — reject downloads smaller than this - LIDARR_ART_MAX_PARALLEL=4 # concurrent background download jobs - LIDARR_ART_RETRIES=2 # download retry attempts per image - LIDARR_ART_SLEEP_BETWEEN=0.2 # seconds between fanart.tv API calls -# HOST*_FANART_API_KEY / HOST*_LASTFM_API_KEY — set in host*.conf - -# Lidarr discovery settings (playback_aware_lidarr_discovery.sh) - LIDARR_DISCOVERY_THRESHOLD=70 # score to accept candidate (0-100) - LIDARR_DISCOVERY_LOOKBACK_DAYS=7 # Emby play history window in days - LIDARR_DISCOVERY_MIN_PLAYS=3 # min plays in window before evaluating an artist - LIDARR_DISCOVERY_USER_CAP_PCT=35 # max % any single user can contribute to play score (prevents one listener dominating) - LIDARR_DISCOVERY_MAX_ADDS=5 # max artists to add per run — quality over bulk - LIDARR_DISCOVERY_REJECT_COOLDOWN=30 # days before re-evaluating a rejected artist - LIDARR_DISCOVERY_HISTORY="$DATA_DIR/lidarr_discovery_history.db" - -# Sonarr discovery settings (playback_aware_sonarr_discovery.sh) - SONARR_DISCOVERY_THRESHOLD=52 # score to accept candidate (0-100) - SONARR_DISCOVERY_LOOKBACK_DAYS=14 # Emby episode play history window in days (shorter than movies — TV watched more frequently) - SONARR_DISCOVERY_MAX_SEEDS=5 # max seed series from Stage 1 - SONARR_DISCOVERY_MAX_ADDS=3 # max shows to add per run — TV is a larger commitment than movies - SONARR_DISCOVERY_MIN_VOTE_COUNT=50 # min TMDB votes (TV has fewer votes than movies at same popularity) - SONARR_DISCOVERY_MIN_RATING=65 # min TMDB vote_average × 10 (65 = 6.5/10) - SONARR_DISCOVERY_REJECT_COOLDOWN=60 # days before re-evaluating a rejected show - SONARR_DISCOVERY_USER_EPISODE_CAP=8 # max episodes any one user contributes to seed volume score - SONARR_DISCOVERY_MONITOR_MODE="all" # Sonarr monitor mode on add: all | future | first | latest | none - SONARR_DISCOVERY_HISTORY="$DATA_DIR/sonarr_discovery_history.db" - -# Radarr discovery shared settings - RADARR_DISCOVERY_THRESHOLD=52 # score to accept candidate (0-100) — lower than Lidarr since diverse seeds rarely overlap - RADARR_DISCOVERY_LOOKBACK_DAYS=30 # Emby watch history window in days (movies rewatched less often) - RADARR_DISCOVERY_MAX_SEEDS=5 # max seed movies from Stage 1 - RADARR_DISCOVERY_MAX_ADDS=5 # max movies to add per run - RADARR_DISCOVERY_MIN_VOTE_COUNT=100 # min TMDB votes to be considered a candidate - RADARR_DISCOVERY_MIN_RATING=60 # min TMDB vote_average × 10 (60 = 6.0/10) - RADARR_DISCOVERY_REJECT_COOLDOWN=60 # days before re-evaluating a rejected movie - RADARR_DISCOVERY_SEED_LIBRARIES=("Movies") # Emby libraries to draw seed movies from - RADARR_DISCOVERY_HISTORY="$DATA_DIR/radarr_discovery_history.db" - -# Emby → arr sync library allowlists -# Only these Emby library names will be considered by the sync tools. -# Names must match exactly as shown in Emby > Dashboard > Libraries. - SONARR_EMBY_LIBRARIES=("Anime" "Kids Shows" "Stand-Up Comedy" "TV shows") - RADARR_EMBY_LIBRARIES=("Movies" "Kid's Movies") - -# Sonarr shared settings - SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion - SONARR_MAX_DELETE_GB=10 # require --i-know-what-im-doing if deletion exceeds this - SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") - SONARR_PROTECTED_PATTERNS=( - # Subtitles - "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" - # Metadata - "*.nfo" "*.tbn" - # Images - "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" - "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" - "banner.*" "thumb.*" "landscape.*" - # Kodi/Emby extras — not tracked by Sonarr API - "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" - "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" - "*-clip.*" "*-other.*" - # Theme songs — stored in show folder, not tracked by arr - "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" - ) - -# Radarr shared settings - RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion - RADARR_MAX_DELETE_GB=15 # require --i-know-what-im-doing if deletion exceeds this - RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") - RADARR_PROTECTED_PATTERNS=( - # Subtitles - "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" - # Metadata - "*.nfo" "*.tbn" - # Images - "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" - "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" - "banner.*" "thumb.*" "landscape.*" - # Kodi/Emby extras — not tracked by Radarr API - "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" - "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" - "*-clip.*" "*-other.*" - # Theme songs — stored in movie folder, not tracked by arr - "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" - ) - -# Radarr/Sonarr TMDb/TVDB removed entry cleanup - RADARR_DROPPED_ADD_EXCLUSION=true # add removed movies to import exclusion list - SONARR_DROPPED_ADD_EXCLUSION=true # add removed series to import exclusion list - -# ━━━ Arr Failed/Stalled Recovery ━━━ -# Auto blocklist + re-search failed imports and stalled downloads. -# Runs every 6 hours — schedule: 0 */6 * * * -# -# Targets four problem types: -# importFailed — downloaded but arr couldn't import -# importPending — downloaded, stuck waiting to import (won't self-resolve) -# error status — serious failure not covered above -# stalled — download stuck with no connections or progress -# -# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first. -# Per-host recovery toggles (HOST1_SONARR_RECOVERY etc.) live in host*.conf. - ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this - # matches cron interval — items eligible after one missed cycle - -# ============================================================================================== -# ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Session-based storage allocator using filesystem symlink indirection. -# ffmpeg resolves the symlink ONCE at session start — existing sessions never affected. -# Symlink flips between ramdisk and SSD are transparent to active streams. -# -# ⚠️ Docker mount — must use shared propagation so symlink flips work inside container: -# --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared -# Standard path mappings use rprivate — symlink changes are NOT visible inside container. - -# ━━━ Transcode Manager ━━━ -# tmpfs mount point — created at array start by ramdisk_setup.sh. -# Must exist before Emby starts so the symlink resolves correctly. - RAMDISK_PATH="/mnt/ramdisk_transcodes" - -# Ramdisk size and flip thresholds — defined per host in host*.conf. -# All three are coupled — if size changes, thresholds must change with it. -# HOST1_RAMDISK_SIZE / HOST2_RAMDISK_SIZE -# HOST1_RAMDISK_WARN_GB / HOST2_RAMDISK_WARN_GB ← flip to SSD at this usage -# HOST1_RAMDISK_LOW_GB / HOST2_RAMDISK_LOW_GB ← flip back to ramdisk at this usage - -# Symlink that Emby points at — this path NEVER changes regardless of ramdisk/SSD state. -# Emby resolves the symlink once per session at start — symlink flips are transparent. -# Must match the container path configured in Emby's Extra Parameters. - TRANSCODE_LINK="/mnt/ram-transcode" - -# SSD fallback location — defined per host in host*.conf (cache path differs per server): -# HOST1_TRANSCODE_SSD / HOST2_TRANSCODE_SSD - -# Minimum free GB on SSD before allowing flip from ramdisk to SSD. - RAMDISK_SSD_MIN_GB=20 - - TRANSCODE_MAX_AGE=20 # minutes — HLS segment age before cleanup eligibility - TRANSCODE_ORPHAN_AGE=30 # minutes — files with no matching active session - TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in one hour - - TRANSCODE_OWNER="nobody:users" - TRANSCODE_CHMOD="755" - -# Operating mode — controls symlink direction behaviour. -# smart — auto-flips between ramdisk and SSD based on thresholds (default) -# ramdisk — always uses ramdisk, warns if RAMDISK_WARN_GB exceeded but holds -# ssd — always uses SSD, never flips to ramdisk - TRANSCODE_MANAGER_MODE="smart" - -# Daily statistics log — read by weekly_health_digest.sh for transcode summary. - TRANSCODE_STATE_FILE="$STATE_DIR/transcode_state.db" - TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db" - TRANSCODE_LOG_RETENTION=90 # days before old entries purged - - TRANSCODE_CHECK_EMBY=true - -# ━━━ Transcode Server Array ━━━ -# All media servers sharing the ramdisk transcode space. -# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex -# Entries with placeholder API keys are skipped automatically. -# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk. -# Defined per host in host*.conf — Emby container names and keys differ per server: -# HOST1_TRANSCODE_SERVERS / HOST2_TRANSCODE_SERVERS - -# ============================================================================================== -# ── MONITORS ────────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ Certificate Monitor ━━━ -# Checks SSL certificate expiry via direct openssl connection — no NPM dependency. -# Checks the actual certificate served by each domain, not what NPM thinks it has. -# Domains defined per host in host*.conf — each server monitors its own domains: -# HOST1_CERT_MONITOR_DOMAINS / HOST2_CERT_MONITOR_DOMAINS - CERT_WARN_DAYS=30 # warn when cert expires within this many days - CERT_CRIT_DAYS=7 # critical alert within this many days - CERT_TIMEOUT=10 # seconds per domain before giving up - -# ━━━ Backup Verify ━━━ -# Verifies rsync mirror health by comparing random file checksums between servers. -# Catches silent corruption or incomplete syncs that rsync itself wouldn't detect. -# Defined per host in host*.conf — leave empty to use HOST*_DAILY_SYNC_SHARES automatically: -# HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES - BACKUP_VERIFY_SAMPLE=10 # random files to check per share - BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample - -# ━━━ SMART Health ━━━ -# Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*. -# Thresholds read from /boot/config/plugins/dynamix/dynamix.cfg at runtime -# (hot/max/hotssd/maxssd) — these vars are fallback only if dynamix.cfg not found. - SMART_TEMP_WARN=45 # fallback — Celsius warn threshold - SMART_TEMP_CRIT=55 # fallback — Celsius critical threshold -# Drives to ignore defined per host in host*.conf — hardware is server-specific: -# HOST1_SMART_IGNORE_DRIVES / HOST2_SMART_IGNORE_DRIVES - -# ━━━ ZFS Memory Snapshot ━━━ -# Weekly ZFS pool health and memory diagnostic report — informational only. - ZFS_REPORT_LOG="$DATA_DIR/zfs-weekly-health.log" - ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC using more than this % of its max - ZFS_REPORT_FREE_WARN_GB=10 # warn if less than this GB free RAM - ZFS_REPORT_AVAIL_WARN_GB=20 # warn if less than this GB available on ZFS pool - ZFS_REPORT_DOCKER_TOP=10 # how many top Docker containers to show by memory -# Pool ignore list defined per host in host*.conf — pool names are server-specific: -# HOST1_ZFS_REPORT_IGNORE_POOLS / HOST2_ZFS_REPORT_IGNORE_POOLS - -# ━━━ Bandwidth Monitor ━━━ -# Called automatically by rsync.sh after each sync — one bounded write per run. -# Tracks transfer size, duration and profile per sync for weekly summary reporting. - BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db" - BANDWIDTH_LOG_RETENTION=90 # days before old entries purged - BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report - -# Stats files — written by cleanup and recovery scripts, read by coffee report. -# All in DATA_DIR — array always running when these are written. - ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats - ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats - -# ━━━ Health Digest ━━━ -# Aggregated system health summary — reads existing state files, no new writes. -# Three profiles control when the digest is sent: -# always — sends every run regardless of findings -# smart — sends only when DIGEST_SMART_ON_* conditions are found -# weekly — sends once per week on DIGEST_DAY only - DIGEST_PROFILE="weekly" # always | smart | weekly - DIGEST_DAY="Sunday" - DIGEST_SMART_ON_WATCHDOG=true # send if any watchdog strikes are active - DIGEST_SMART_ON_FALLBACK=true # send if fallback state is not NORMAL - DIGEST_SMART_ON_CERT_WARN=true # send if any cert is under CERT_WARN_DAYS - DIGEST_SMART_ON_BANDWIDTH=true # send if any transfer exceeded BANDWIDTH_WARN_GB - -# ━━━ Emby Session Report ━━━ -# Weekly Emby usage statistics via API — no persistent writes, queries fresh each run. -# URL and API key pulled from HOST*_EMBY_URL and HOST*_EMBY_API_KEY in host*.conf. - EMBY_REPORT_DAYS=7 # days to include in the report period - EMBY_REPORT_TOP_N=10 # number of top content items to show - -# ============================================================================================== -# ── RESOURCE MANAGER ────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Pressure reduction layer — keeps the system comfortable before things break. -# Called by watchdog_orchestrator.sh. Single-pass, not a continuous loop. -# -# ── PRESSURE LEVELS ─────────────────────────────────────────────────────────────────────────── -# Level 1 (soft) — throttle SABnzbd + qBit download speeds -# Level 2 (medium) — further throttle + docker pause background containers -# Level 3 (hard) — docker stop optional containers, signal docker_watchdog to defer -# -# ── PER-HOST CONTAINER LISTS ────────────────────────────────────────────────────────────────── -# HOST*_RW_PAUSE_CONTAINERS — docker pause at medium pressure (in host*.conf) -# HOST*_RW_STOP_CONTAINERS — docker stop at hard pressure (in host*.conf) - - RW_ENABLED=true - RW_STATE_FILE="$STATE_DIR/resource_watchdog_state.db" - -# ━━━ Pressure Thresholds ━━━ -# Graduated RAM response — resource_watchdog acts before system_watchdog reboots. -# RW_RAM_SOFT_GB > RW_RAM_MEDIUM_GB > RW_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always - RW_RAM_SOFT_GB=12 # throttle start — reduce background load - RW_RAM_MEDIUM_GB=8 # pause background containers - RW_RAM_HARD_GB=6 # stop optional containers (was SYS_WATCHDOG_MEM_SHUTDOWN_GB) - RW_RAM_RECOVER_GB=20 # RAM must reach this before restoring hard-stopped containers - -# Load average thresholds — multiplier × core count - RW_LOAD_SOFT_MULTIPLIER=2.0 # soft pressure: 2× cores sustained - RW_LOAD_MEDIUM_MULTIPLIER=3.0 # medium pressure: 3× cores sustained - -# Consecutive runs at lower pressure before de-escalating - RW_RECOVER_CYCLES=3 - -# ━━━ SABnzbd Throttle ━━━ -# Speed values: "50M" = 50 MB/s, "0" = unlimited - RW_SABNZBD_ENABLED=true - RW_SABNZBD_SPEED_SOFT="50M" - RW_SABNZBD_SPEED_MEDIUM="10M" - -# ━━━ qBittorrent Throttle ━━━ -# KB/s — 0 = unlimited - RW_QBIT_ENABLED=true - RW_QBIT_DL_SOFT=51200 # 50 MB/s - RW_QBIT_DL_MEDIUM=10240 # 10 MB/s - -# ━━━ Critical Containers ━━━ -# Never paused or stopped regardless of pressure level. -# Keep DNS, auth, media serving, and live TV always running. - RW_CRITICAL_CONTAINERS=( - "NginxProxyManager" # reverse proxy — internet access - "Authelia" # auth — nothing accessible without it - "Authelia-Secondary" - "Mariadb-Authelia" # Authelia dependency - "Mariadb-Authelia-Secondary" - "Redis-Authelia" # Authelia dependency - "Redis-Authelia-Secondary" - "AdGuard-Home" # DNS — all LAN resolution - "Emby" # media server — Live TV buffering - "Dispatcharr" # Live TV scheduler — loses state if stopped - "Dispatcharr-Basic" - "Dispatcharr-Iptv-Users" - ) - -# ============================================================================================== -# ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Single-pass system health check — last line of defense before a crash. -# Called by watchdog_orchestrator.sh every minute. NOT started by array_started.sh. -# Re-sources config at each orchestrator run — config changes take effect immediately. -# -# ── THREE-TIER RESPONSE SYSTEM ──────────────────────────────────────────────────────────────── -# CRITICAL — bypass ALL strikes, reboot immediately -# Docker daemon down, rootfs 100%, kernel oops, FD exhaustion, /boot read-only -# -# URGENT — bypass strikes only when OOM confirms active crisis -# RAM < MEM_GB AND OOM kills >= OOM_LIMIT in this cycle → reboot NOW -# Without OOM confirmation → normal strike system -# -# STANDARD — strike system (N consecutive failures → reboot) -# RAM tiers, high load, CPU temp, zombies, /var/log, /tmp, containers -# -# ── RAM ─────────────────────────────────────────────────────────────────────────────────────── -# SYS_WATCHDOG_MEM_GB — strike system → reboot (or OOM bypass) -# Warn/shutdown/recover RAM tiers are handled by resource_watchdog.sh - -# ━━━ State Files ━━━ - SYS_WATCHDOG_STATE_FILE="$STATE_DIR/system_watchdog_state.db" - DOCKER_WATCHDOG_FAILED_FILE="$STATE_DIR/docker_watchdog_failed.db" - SYS_WATCHDOG_REBOOT_LOG="$STATE_DIR/system_watchdog_reboots.db" - SYS_WATCHDOG_OOM_FILE="$STATE_DIR/system_watchdog_oom.db" - -# ━━━ Strike and Reboot Loop Settings ━━━ -# Strike system: a check must fail this many consecutive cycles before action is taken. -# Single spikes (one bad reading) are ignored — sustained problems trigger reboot. - SYS_WATCHDOG_STRIKE_LIMIT=2 # consecutive failures before reboot trigger - -# Reboot loop protection — if system keeps rebooting something is seriously wrong. -# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS → shutdown instead of reboot. - SYS_WATCHDOG_REBOOT_LIMIT=3 - SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 - -# ━━━ RAM Reboot Threshold ━━━ -# Reboot trigger only — warn/shutdown/recover handled by resource_watchdog.sh -# RW_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always (RM acts before watchdog reboots) - SYS_WATCHDOG_MEM_GB=4 # strike system → reboot - -# ━━━ OOM Bypass Settings ━━━ -# OOM bypass: if RAM is critically low AND kernel OOM kills exceed this threshold -# in a single cycle → bypass strike system and reboot immediately. -# Rate-based: kills per 5-minute cycle, not absolute count. -# Rationale: 1-2 kills = docker_watchdog handles it ✅ -# 3+ kills while RAM critical = system dying faster than watchdogs can heal ✅ - SYS_WATCHDOG_OOM_LIMIT=3 # OOM kills in one cycle to trigger bypass - -# ━━━ Thresholds ━━━ -# Set at "about to become unstable" — not "things are a bit high". - -# rootfs (/) usage — two levels: strike at 95%, critical bypass at 99% - SYS_WATCHDOG_ROOTFS_PCT=95 # standard strike threshold - SYS_WATCHDOG_ROOTFS_CRITICAL_PCT=99 # bypass strikes — truly full, writes failing - -# /var/log usage — log spam indicates something broken - SYS_WATCHDOG_LOG_PCT=95 - -# /tmp usage — tmpfs fills from downloads, lock files fail at 100% - SYS_WATCHDOG_TMP_PCT=90 # warn + attempt clear - SYS_WATCHDOG_TMP_CRITICAL_PCT=98 # bypass strikes if clear failed - -# ZFS ARC pinned percentage — ARC not releasing after reclaim = memory stuck - SYS_WATCHDOG_ARC_PINNED_PCT=98 - SYS_WATCHDOG_ARC_RELEASE_PCT=95 - -# Load average multiplier — threshold = MULTIPLIER × CPU core count -# MULTIPLIER=3 on 16-core = load of 48 before triggering - SYS_WATCHDOG_LOAD_MULTIPLIER=3 - -# Zombie process count — large numbers = serious process management failure - SYS_WATCHDOG_ZOMBIE_LIMIT=50 - -# CPU temperature — sustained high temp causes throttling or kernel panic - SYS_WATCHDOG_CPU_TEMP_MAX=95 - -# File descriptor exhaustion — system-wide FD limit near exhaustion -# New connections fail silently, Docker can't spawn processes, SSH fails - SYS_WATCHDOG_FD_CRITICAL_PCT=95 # bypass strikes — critical tier - -# Runaway process — single non-container process consuming excessive CPU -# Multiple strikes before action — single spikes are normal - SYS_WATCHDOG_RUNAWAY_CPU_PCT=90 # % single process must sustain - SYS_WATCHDOG_RUNAWAY_STRIKES=3 # consecutive cycles before warning - -# Array disk errors — accumulating mdstat errors = disk failing NOW - SYS_WATCHDOG_MDSTAT_ERROR_LIMIT=5 # new errors in one cycle before acting - -# ━━━ Check Toggles ━━━ -# Per-host — moved to host*.conf -# Different servers may have different hardware, NICs, and check requirements -# See HOST*_SYS_WATCHDOG_CHECK_* in host*.conf - -# ━━━ Abort Toggles ━━━ -# Conditions that prevent reboot even when a threshold is hit. -# CRITICAL tier bypasses these — a truly critical condition reboots regardless. -# true = abort standard reboot if this condition is active -# false = reboot anyway - SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # rebooting with bad pool risks data loss - SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity better than crashing mid-check - SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting move better than crashing mid-move - -# ============================================================================================== -# ──────────────────────── End Of User Variables ─────────────────────────────────────────────── -# ============================================================================================== \ No newline at end of file diff --git a/Deployment/conf_upgrade.sh b/Deployment/conf_upgrade.sh index a7d6a6b..2a5af14 100755 --- a/Deployment/conf_upgrade.sh +++ b/Deployment/conf_upgrade.sh @@ -53,13 +53,13 @@ # RUNTIME MODES # ============================================================================================== # -# conf_upgrade.sh --template master.conf.template --target Configurations/master.conf --dry-run +# conf_upgrade.sh --template Configurations/master.conf.template --target Configurations/master.conf --dry-run # Preview what would be added, removed, and kept — no changes written. # -# conf_upgrade.sh --template master.conf.template --target Configurations/master.conf --backup +# conf_upgrade.sh --template Configurations/master.conf.template --target Configurations/master.conf --backup # Apply the upgrade, writing a .bak first. # -# conf_upgrade.sh --template master.conf.template --target Configurations/master.conf +# conf_upgrade.sh --template Configurations/master.conf.template --target Configurations/master.conf # Apply the upgrade in-place with no backup. # # ============================================================================================== diff --git a/Plugin/varaverk.plg b/Plugin/varaverk.plg index a6c87eb..0e506c3 100644 --- a/Plugin/varaverk.plg +++ b/Plugin/varaverk.plg @@ -195,8 +195,8 @@ fi # Seed master.conf from template if absent. mkdir -p "$CONF_DIR" -if [[ ! -f "$CONF_DIR/master.conf" && -f "$SCRIPTS_DIR/Deployment/conf_templates/master.conf" ]]; then - cp "$SCRIPTS_DIR/Deployment/conf_templates/master.conf" "$CONF_DIR/master.conf" +if [[ ! -f "$CONF_DIR/master.conf" && -f "$CONF_DIR/master.conf.template" ]]; then + cp "$CONF_DIR/master.conf.template" "$CONF_DIR/master.conf" log "seeded master.conf from template" fi log "install step complete" diff --git a/git_pull_execute.sh b/git_pull_execute.sh index 32d1d26..bf7d2b3 100755 --- a/git_pull_execute.sh +++ b/git_pull_execute.sh @@ -281,7 +281,6 @@ echo "" echo "━━━ $ICON_GEAR Conf Upgrade ━━━" UPGRADE_SCRIPT="$TARGET_DIR/Deployment/conf_upgrade.sh" -TMPL_DIR="$TARGET_DIR/Deployment/conf_templates" CONF_DIR="$TARGET_DIR/Configurations" if [[ ! -f "$UPGRADE_SCRIPT" ]]; then @@ -292,20 +291,20 @@ elif [[ "$SYNC_SUCCESS" == true ]]; then _DRY="" # master.conf - if [[ -f "$TMPL_DIR/master.conf" && -f "$CONF_DIR/master.conf" ]]; then + if [[ -f "$CONF_DIR/master.conf.template" && -f "$CONF_DIR/master.conf" ]]; then bash "$UPGRADE_SCRIPT" \ - --template "$TMPL_DIR/master.conf" \ + --template "$CONF_DIR/master.conf.template" \ --target "$CONF_DIR/master.conf" \ --backup $_DRY else - warn "master.conf template or target not found — skipping" + warn "master.conf.template or master.conf not found — skipping" fi # This server's host conf only — sparse checkout ensures we have it HOST_CONF="$CONF_DIR/${MY_ID,,}.conf" - if [[ -f "$TMPL_DIR/host.conf.template" && -f "$HOST_CONF" ]]; then + if [[ -f "$CONF_DIR/host.conf.template" && -f "$HOST_CONF" ]]; then bash "$UPGRADE_SCRIPT" \ - --template "$TMPL_DIR/host.conf.template" \ + --template "$CONF_DIR/host.conf.template" \ --target "$HOST_CONF" \ --backup $_DRY else