2784 lines
157 KiB
Plaintext
2784 lines
157 KiB
Plaintext
On branch main
|
||
Your branch is ahead of 'origin/main' by 6 commits.
|
||
(use "git push" to publish your local commits)
|
||
|
||
You are in a sparse checkout with 100% of tracked files present.
|
||
|
||
Changes not staged for commit:
|
||
(use "git add/rm <file>..." to update what will be committed)
|
||
(use "git restore <file>..." to discard changes in working directory)
|
||
deleted: Deployment/conf_templates/host.conf.template
|
||
deleted: Deployment/conf_templates/master.conf
|
||
modified: Deployment/conf_upgrade.sh
|
||
modified: Fallback/fallback.sh
|
||
modified: Media/radarr_cleanup.sh
|
||
modified: Media/sonarr_cleanup.sh
|
||
modified: Orchestrators/array_started.sh
|
||
modified: Plugin/unraid/System_Essentials/unraid_api_key_renew.sh
|
||
modified: Plugin/unraid/api/stop.php
|
||
modified: Plugin/unraid/js/varaverk.js
|
||
modified: Plugin/unraid/pages/arrs.php
|
||
modified: Plugin/unraid/pages/docker.php
|
||
modified: Plugin/unraid/pages/fallback.php
|
||
modified: Plugin/unraid/pages/partnership.php
|
||
modified: Plugin/unraid/pages/setup.php
|
||
modified: Plugin/varaverk.plg
|
||
modified: common.sh
|
||
modified: git_pull_execute.sh
|
||
|
||
no changes added to commit (use "git add" and/or "git commit -a")
|
||
diff --git a/Deployment/conf_templates/host.conf.template b/Deployment/conf_templates/host.conf.template
|
||
deleted file mode 100644
|
||
index a763285..0000000
|
||
--- a/Deployment/conf_templates/host.conf.template
|
||
+++ /dev/null
|
||
@@ -1,769 +0,0 @@
|
||
-#!/bin/bash
|
||
-# ==============================================================================================
|
||
-# ========================== HOST1 CONFIGURATION — unRAID-Gmer4Lfe ============================
|
||
-# ==============================================================================================
|
||
-# HOST1-specific variables — credentials, container names, share paths, failover lists.
|
||
-# Sourced after master.conf — values here extend shared profile arrays and add HOST1-specific
|
||
-# identity, credentials, and container configuration.
|
||
-#
|
||
-# Sparse checkout (git) ensures HOST2 never receives this file.
|
||
-# HOST2 never sees HOST1 credentials — clean separation at the file level.
|
||
-#
|
||
-# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf.
|
||
-# DO NOT put HOST2 variables here — they belong in host2.conf.
|
||
-#
|
||
-# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||
-#
|
||
-# ── IDENTITY & CONNECTIVITY ────────────────────────────────────────────────────────────────
|
||
-# IDENTITY hostname, SSH key
|
||
-# EMBY container name, URL, API key
|
||
-# NOTIFICATIONS Discord webhook
|
||
-# PARTNERSHIP auth containers, backup paths
|
||
-#
|
||
-# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||
-# DAILY SYNC SHARES media shares HOST1 owns and pushes to HOST2
|
||
-# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am)
|
||
-# CRITICAL SYNC SHARES appdata shares synced every 30 minutes
|
||
-# BACKUP VERIFY shares for checksum verification against remote
|
||
-# HOST1 RSYNC PROFILE host1-appdata profile for HOST1-specific appdata syncs
|
||
-#
|
||
-# ── DOCKER ─────────────────────────────────────────────────────────────────────────────────
|
||
-# DOCKER DAILY RESTART containers restarted daily
|
||
-# DOCKER WEEKLY RESTART containers restarted weekly
|
||
-# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list
|
||
-# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh
|
||
-#
|
||
-# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||
-# DDNS DDNS containers managed by HOST1
|
||
-# INTERNET LOSS containers stopped when internet is lost
|
||
-# FALLBACK TIERS what HOST1 runs for HOST2 per tier
|
||
-# TIER DELAYS how long HOST1 must be down before each tier activates on HOST2
|
||
-# RSYNC WRITEBACK HOST1 appdata synced back on handback
|
||
-#
|
||
-# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||
-# MEDIA PERMISSIONS share list for media_shares_permissions.sh
|
||
-# MEDIA CLEANER folder lists for media_cleaner.sh
|
||
-#
|
||
-# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
|
||
-# CERTIFICATE MONITOR domains checked for SSL expiry
|
||
-# SMART HEALTH drives to skip in SMART monitoring
|
||
-# ZFS REPORT pools to exclude from ZFS health report
|
||
-#
|
||
-# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||
-# TRANSCODES ramdisk size, thresholds, SSD path, server array
|
||
-#
|
||
-# ── ARR STACK ──────────────────────────────────────────────────────────────────────────────
|
||
-# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs
|
||
-# LIDARR URL, API key, path map
|
||
-# SONARR URL, API key, path map
|
||
-# RADARR URL, API key, path map
|
||
-# ARR RECOVERY per-arr recovery toggles
|
||
-#
|
||
-# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||
-# SYSTEM WATCHDOG per-host check toggles and NIC configuration
|
||
-#
|
||
-# ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
|
||
-# RESOURCE MANAGER containers paused/stopped under memory pressure
|
||
-#
|
||
-# ==============================================================================================
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-
|
||
-# ━━━ Identity ━━━
|
||
-# HOST1 hostname lives in master.conf (not a credential — safe for all servers).
|
||
-# SSH key used for all server-to-server operations — rsync, failover container commands.
|
||
-# Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys.
|
||
- HOST1_SSH_KEY="" # /root/.ssh/your_key_name
|
||
- HOST1_OWNER="" # your Gitea/GitHub username
|
||
- HOST1_OWNER_EMAIL="" # your email
|
||
-
|
||
-# ━━━ Emby ━━━
|
||
-# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh,
|
||
-# weekly_sync_maintenance.sh, and HOST1_TRANSCODE_SERVERS below.
|
||
-# API key: Emby Dashboard → API Keys → + New Key
|
||
- HOST1_EMBY_CONTAINER="Emby"
|
||
- HOST1_EMBY_URL="http://localhost:8096"
|
||
- HOST1_EMBY_API_KEY="" # Emby Dashboard → API Keys → + New Key
|
||
-
|
||
-# ━━━ Jellyfin ━━━
|
||
-# API key: Jellyfin Dashboard → Administration → API Keys → + New Key
|
||
- HOST1_JELLYFIN_CONTAINER="Jellyfin"
|
||
- HOST1_JELLYFIN_URL="http://localhost:8096"
|
||
- HOST1_JELLYFIN_API_KEY="" # Jellyfin Dashboard → API Keys → + New Key
|
||
-
|
||
-# ━━━ Gitea ━━━
|
||
-# Personal access token for gitea_ssh_setup.sh — registers this server's SSH public key
|
||
-# with Gitea so git operations use key auth instead of passwords.
|
||
-# Create in Gitea: Settings → Applications → Generate Token → scope: write:user
|
||
- HOST1_GITEA_API_TOKEN="" # Gitea Settings → Applications → Generate Token
|
||
-
|
||
-# ━━━ Notifications ━━━
|
||
-# Discord webhook — leave blank to disable.
|
||
-# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies.
|
||
- HOST1_DISCORD_WEBHOOK="" # Discord channel → Integrations → Webhooks
|
||
-
|
||
-# ━━━ Partnership ━━━
|
||
-# HOST1 is always the owner (source of truth) unless --transfer has been run.
|
||
-# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs.
|
||
-
|
||
-# Auth containers reconfigured on onboard/offboard.
|
||
-# Format: "ContainerName|WebUIPort"
|
||
-# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth)
|
||
-# On offboard → WebUI pointed back at localhost
|
||
- HOST1_PARTNERSHIP_AUTH_WEBUIS=(
|
||
- "NginxProxyManager|81"
|
||
- "Lldap-Gmer4Lfe|17170"
|
||
- "Authelia|9091"
|
||
- "Authelia-Secondary|9092"
|
||
- )
|
||
-
|
||
-# XML templates (from this server's templates-user/) pushed to mirror during onboard.
|
||
-# These become the mirror's active auth stack, backed by the rsync-synced appdata.
|
||
-# Update filename if Lldap is renamed to drop the host suffix.
|
||
- HOST1_PARTNERSHIP_AUTH_STACK=(
|
||
- # Dependencies first — Mariadb/Redis must be healthy before Authelia starts
|
||
- "my-Mariadb-Authelia.xml"
|
||
- "my-Mariadb-Authelia-Secondary.xml"
|
||
- "my-Redis-Authelia.xml"
|
||
- "my-Redis-Authelia-Secondary.xml"
|
||
- # Auth apps — deployed after their deps are confirmed healthy
|
||
- "my-Authelia.xml"
|
||
- "my-Authelia-Secondary.xml"
|
||
- "my-NginxProxyManager.xml"
|
||
- "my-Lldap-Gmer4Lfe.xml"
|
||
- # Source of truth — must be available on HOST2 independently of the auth stack
|
||
- "my-Gitea.xml"
|
||
- )
|
||
-
|
||
-# XML templates pushed to mirror for the arr stack during onboard.
|
||
-# Deps (e.g. databases) first if any — same ordering rule as auth stack.
|
||
- HOST1_PARTNERSHIP_ARR_STACK=(
|
||
- # "my-Sonarr.xml"
|
||
- # "my-Radarr.xml"
|
||
- # "my-Lidarr.xml"
|
||
- # "my-Prowlarr.xml"
|
||
- # "my-Bazarr.xml"
|
||
- )
|
||
-
|
||
-# Paths HOST2 should collect during the grace window after offboard.
|
||
-# Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS.
|
||
- HOST1_PARTNERSHIP_MIRROR_BACKUPS=(
|
||
- # "/mnt/user/appdata-Fallback/Jayred365-Emby"
|
||
- )
|
||
-
|
||
-# Containers parked on this server when partnership is active.
|
||
-# Stopped on onboard (owner deploys its stack instead), restarted on offboard.
|
||
- HOST1_PARTNERSHIP_OWN_CONTAINERS=(
|
||
- # "Emby"
|
||
- # "NginxProxyManager"
|
||
- )
|
||
-
|
||
-# Emby admin provisioning — toggle is owner-only, credentials are per-host.
|
||
-# Owner enables/disables the feature. Each host sets the account they want on the shared Emby.
|
||
-# On onboard: owner reads mirror's HOST*_PARTNERSHIP_EMBY_ADMIN_* and creates that account.
|
||
-# On offboard: account is deleted. Username collision → onboard exits with error.
|
||
- HOST1_PARTNERSHIP_PROVISION_EMBY_ADMIN=false # owner controls whether Emby is shared
|
||
- HOST1_PARTNERSHIP_EMBY_PORT=8096
|
||
- HOST1_PARTNERSHIP_EMBY_ADMIN_USER="" # this server's desired Emby username
|
||
- HOST1_PARTNERSHIP_EMBY_ADMIN_PASS="" # this server's desired Emby password
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-
|
||
-# ━━━ Daily Sync Shares ━━━
|
||
-# Shares HOST1 pushes to all other nodes every night (1am via daily_sync_maintenance.sh).
|
||
-# Mesh model: every node pushes every media share — no ownership, no mirrors.
|
||
-# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete).
|
||
-# arr_cleanup removes true orphans based on local arr state.
|
||
-# Any node can download content to any share — it propagates to all nodes on the next cycle.
|
||
-# Nextcloud is intentionally one-directional (HOST1→HOST2 offsite backup — not arr-managed).
|
||
-# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
|
||
-# For shares needing container stops or custom options — add a profile in master.conf.
|
||
- HOST1_DAILY_SYNC_SHARES=(
|
||
- /mnt/user/Books
|
||
- /mnt/user/Intros
|
||
- /mnt/user/Kids_Movies
|
||
- /mnt/user/Kids_Tv_Shows
|
||
- /mnt/user/Movies
|
||
- /mnt/user/Music
|
||
- /mnt/user/Music_Videos
|
||
- /mnt/user/Nextcloud
|
||
- /mnt/user/stand-up_comedy
|
||
- /mnt/user/Sports
|
||
- /mnt/user/Tv_Shows
|
||
- /mnt/user/Anime_Shows-Old
|
||
- /mnt/user/Anime_Movies-Old
|
||
- /mnt/user/Anime_Movies
|
||
- /mnt/user/Anime_Shows
|
||
- )
|
||
-
|
||
-# Personal encrypted shares — synced for offsite backup, independent of media shares.
|
||
-# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
|
||
-# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
|
||
- HOST1_PERSONAL_SHARES=(
|
||
- # /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset
|
||
- )
|
||
-
|
||
-# ━━━ Weekly Sync Shares ━━━
|
||
-# Appdata shares synced during the weekly maintenance window (Sunday 2:30am).
|
||
-# Containers stopped both sides before sync — full clean state guaranteed.
|
||
-# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section.
|
||
-# Order matters — Emby first (larger transfer), then Critical-Data (auth stack).
|
||
- HOST1_WEEKLY_SYNC_SHARES=(
|
||
- "/mnt/user/Media_Server/Emby" # emby profile — full clean mirror
|
||
- "/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — auth stack
|
||
- )
|
||
-
|
||
-# ━━━ Intermediate Sync Shares ━━━
|
||
-# Shares synced every 4 hours by intermediate_sync_maintenance.sh.
|
||
-# Uses DEFAULT_RSYNC_OPTS (no --delete) — for sub-daily propagation of metadata or watch state.
|
||
-# Full media share sync stays in the daily window. Leave empty to skip mid-day rsync entirely.
|
||
- HOST1_INTERMEDIATE_SYNC_SHARES=(
|
||
- # Add shares here to enable mid-day rsync
|
||
- # Example: "/mnt/user/Emby_Metadata"
|
||
- )
|
||
-
|
||
-# ━━━ Critical Sync Shares ━━━
|
||
-# Appdata shares synced every 30 minutes by critical_sync_maintenance.sh.
|
||
-# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||
-# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
|
||
- HOST1_CRITICAL_SYNC_SHARES=(
|
||
- "/mnt/user/appdata-Fallback/Critical-Data|critical-fallback" # auth dirty sync — stays running
|
||
- "/mnt/user/Media_Server/Emby|emby-fallback" # Emby dirty sync — stays running
|
||
- )
|
||
-
|
||
-# ━━━ Backup Verify ━━━
|
||
-# Shares verified by backup_verify.sh — random file checksum comparison against remote.
|
||
-# Leave empty to use HOST1_DAILY_SYNC_SHARES automatically.
|
||
-# Sample size and minimum file size defined in master.conf.
|
||
- HOST1_BACKUP_VERIFY_SHARES=(
|
||
- # leave empty to use HOST1_DAILY_SYNC_SHARES automatically
|
||
- )
|
||
-
|
||
-# ━━━ HOST1 Rsync Profile — host1-appdata ━━━
|
||
-# HOST1-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf.
|
||
-# Use for appdata unique to HOST1 (Organizrv2, VaultWarden, UptimeKuma etc.)
|
||
-# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf.
|
||
-# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Fallback/HOST1-Appdata --profile=host1-appdata
|
||
- PROFILE_RSYNC_OPTS[host1-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host1-appdata]:-8000}"
|
||
- PROFILE_BW_LIMIT[host1-appdata]=8000
|
||
- PROFILE_RETRY_COUNT[host1-appdata]=3
|
||
- PROFILE_SLEEP[host1-appdata]=300
|
||
- PROFILE_CRITICAL_CONTAINER_NAMES[host1-appdata]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
|
||
- PROFILE_DELAYED_CONTAINERS[host1-appdata]=""
|
||
- PROFILE_CONTAINER_DELAY[host1-appdata]=5
|
||
- PROFILE_EXCLUDE_DIRS[host1-appdata]="logs *.tmp"
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-
|
||
-# ━━━ Docker Daily Restart ━━━
|
||
-# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
|
||
-# Dispatcharr degrades over time without restart — daily is intentional, not just housekeeping.
|
||
-# Order matters — auth stack first, then media services.
|
||
- HOST1_DAILY_RESTART_CONTAINERS=(
|
||
- "NginxProxyManager"
|
||
- "Lldap-Gmer4Lfe"
|
||
- "Authelia"
|
||
- "Authelia-Secondary"
|
||
- "Dispatcharr-Iptv-Users"
|
||
- "Dispatcharr" # Live TV scheduler — degrades without daily restart
|
||
- "Dispatcharr-Basic"
|
||
- "ErsatzTV-Emby"
|
||
- "Slskd" # Soulseek connection drops after extended uptime; restart refreshes share index
|
||
- )
|
||
-
|
||
-# ━━━ Docker Weekly Restart ━━━
|
||
-# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
|
||
-# Containers already stopped for weekly sync — restart adds zero extra downtime.
|
||
- HOST1_WEEKLY_RESTART_CONTAINERS=(
|
||
- "NextCloud"
|
||
- "Organizrv2-Gmer4Lfe"
|
||
- "AdGuard-Home"
|
||
- "Immich-Gmer4Lfe"
|
||
- )
|
||
-
|
||
-# ━━━ Docker Watchdog ━━━
|
||
-# Per-HOST1 container configuration for docker_watchdog.sh.
|
||
-# Shared thresholds and toggles live in master.conf.
|
||
-
|
||
-# Memory hard limits in MB — immediate restart if exceeded.
|
||
-# Set at "container is clearly broken" not "container is busy".
|
||
-# 20GB=20480 18GB=18432 16GB=16384 12GB=12288 8GB=8192 4GB=4096 2GB=2048 1GB=1024
|
||
- declare -A HOST1_WATCHDOG_CONTAINERS=(
|
||
- ["Emby"]=18432 # 18GB — large library + active transcodes
|
||
- ["LidaTube"]=6144 # 6GB — memory leak over time
|
||
- ["Tdarr"]=6144 # 6GB — encoding is memory intensive
|
||
- ["Code-Server"]=1024 # 1GB — should never need more
|
||
- )
|
||
-
|
||
-# HTTP health check URLs — checked every cycle, strike system before restart.
|
||
-# Only add containers with a meaningful web interface to check.
|
||
- declare -A HOST1_WATCHDOG_CONTAINER_URLS=(
|
||
- ["Emby"]="http://localhost:8096"
|
||
- )
|
||
-
|
||
-# Required containers — must always be running on HOST1.
|
||
-# Strike system before restart — repeated failures go on skip list, auto-clears on recovery.
|
||
-# Listed in dependency order — dependencies before dependents.
|
||
- HOST1_WATCHDOG_REQUIRED_CONTAINERS=(
|
||
- "NginxProxyManager"
|
||
- "Lldap-Gmer4Lfe"
|
||
- "Mariadb-Authelia"
|
||
- "Mariadb-Authelia-Secondary"
|
||
- "Redis-Authelia"
|
||
- "Redis-Authelia-Secondary"
|
||
- "Authelia"
|
||
- "Authelia-Secondary"
|
||
- )
|
||
-
|
||
-# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting.
|
||
-# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking.
|
||
- HOST1_WATCHDOG_SCAN_IGNORE=(
|
||
- "DashGate"
|
||
- "PIA-WG-Config-Generator"
|
||
- "Aperture"
|
||
- "Aperture-Kids"
|
||
- "pgvector-18-Apeture-Kids"
|
||
- "Pgvector18-Aperture"
|
||
- )
|
||
-
|
||
-# Dependency ordering — skip restarting a container if its dependency is also down.
|
||
-# Prevents watchdog from restarting Authelia before Mariadb is back up.
|
||
-# SPACE-SEPARATED STRINGS — converted to array at runtime.
|
||
- declare -A HOST1_WATCHDOG_DEPENDENCIES=(
|
||
- ["Authelia"]="Mariadb-Authelia Redis-Authelia"
|
||
- ["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
|
||
- ["NextCloud"]="Postgres-NextCloud"
|
||
- )
|
||
-
|
||
-# Per-container appdata growth suppress ceilings in MB.
|
||
-# ONLY needed in specific cases — growth rate detection covers all containers automatically.
|
||
-# Use this when a container legitimately has large stable data and you want to guarantee
|
||
-# it never triggers a false-positive growth alert. Growth warnings are suppressed while the
|
||
-# container's dir stays below this ceiling; above it, warnings resume as normal.
|
||
-# 50GB=51200 25GB=25600 20GB=20480 15GB=15360 10GB=10240 5GB=5120
|
||
- declare -A HOST1_WATCHDOG_APPDATA_SIZES=(
|
||
- ["Tdarr"]="25600" # 25GB — transcode cache grows legitimately during active jobs
|
||
- ["7dtd"]="20480" # 20GB — game server world data, expected to be large
|
||
- )
|
||
-
|
||
-# ━━━ Network Watchdog ━━━
|
||
-# Host-specific connectivity config for Watchdogs/System/network_watchdog.sh.
|
||
- HOST1_NETWORK_WATCHDOG_DDNS_DOMAIN="gmer4lfe.com"
|
||
- HOST1_NETWORK_WATCHDOG_DDNS_CONTAINER="Gmer4Lfe.com"
|
||
- HOST1_NETWORK_WATCHDOG_NPM_URL="https://gmer4lfe.com"
|
||
-
|
||
-# ━━━ Docker Network Connect ━━━
|
||
-# Containers connected to custom networks at array start by docker_network_connect.sh.
|
||
-# Networks created if they don't exist — idempotent, safe to re-run.
|
||
- HOST1_NETWORK_CONNECT_CONTAINERS=(
|
||
- "memcached"
|
||
- "Npm-CrowdSec"
|
||
- )
|
||
-
|
||
- HOST1_NETWORK_CONNECT_NETWORKS=(
|
||
- "high-availability"
|
||
- )
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-
|
||
-# ━━━ DDNS ━━━
|
||
-# DDNS containers HOST1 manages — started/stopped by fallback.sh per DDNS absolute rules:
|
||
-# Internet loss → stop immediately
|
||
-# Failover → HOST2 starts HOST1's DDNS as Tier 1 (before any other containers)
|
||
-# Handback → stop HOST1's DDNS on HOST2 → rsync → start containers → start local DDNS last
|
||
- HOST1_DDNS_CONTAINERS=(
|
||
- "Gmer4Lfe.com"
|
||
- )
|
||
-
|
||
-# ━━━ Internet Loss ━━━
|
||
-# Containers stopped immediately on HOST1 when internet connection is lost.
|
||
-# Prevents external-facing services from operating without connectivity.
|
||
- FALLBACK_HOST1_STOP_ON_NO_NET=(
|
||
- "Gmer4Lfe.com"
|
||
- )
|
||
-
|
||
-# ━━━ Fallback Tiers — HOST1 Runs for HOST2 ━━━
|
||
-# Containers HOST1 starts when HOST2 goes down.
|
||
-# Tier 1 is always immediate — vital services cannot wait.
|
||
-# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in host2.conf).
|
||
- FALLBACK_HOST1_COVERS_HOST2_TIER1=(
|
||
- "Gmer4Lfe.us"
|
||
- "VaultWarden-Jayred365"
|
||
- )
|
||
-
|
||
- FALLBACK_HOST1_COVERS_HOST2_TIER2=(
|
||
- # "container-placeholder"
|
||
- )
|
||
-
|
||
- FALLBACK_HOST1_COVERS_HOST2_TIER3=(
|
||
- # "container-placeholder"
|
||
- )
|
||
-
|
||
- FALLBACK_HOST1_COVERS_HOST2_TIER4=(
|
||
- # "container-placeholder"
|
||
- )
|
||
-
|
||
-# ━━━ Tier Delays — HOST1's Containers on HOST2 ━━━
|
||
-# How long HOST1 must be down before each tier activates on HOST2 — in minutes.
|
||
-# Tier 1 is always immediate — no delay var needed.
|
||
- HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich
|
||
- HOST1_TIER3_DELAY=720 # 12 hours — secondary services
|
||
- HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders
|
||
-
|
||
-# ━━━ Rsync Writeback — HOST1 Appdata Back on Handback ━━━
|
||
-# Syncs HOST1 appdata BACK to HOST1 when it comes back online after a failover.
|
||
-# Containers stopped before writeback — clean source, no competing writes.
|
||
-#
|
||
-# HOST1_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state
|
||
-# is more reliable than dirty sync data for brief outages.
|
||
- HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr
|
||
-
|
||
-# Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES — only list paths NOT in that array.
|
||
- FALLBACK_HOST1_WRITEBACK_TIER1=(
|
||
- "/mnt/user/Media_Server/Emby" # watch states built up during outage
|
||
- )
|
||
-
|
||
- FALLBACK_HOST1_WRITEBACK_TIER2=(
|
||
- "/mnt/user/appdata-Fallback/Important-Data" # NextCloud + Postgres — files added during outage
|
||
- )
|
||
-
|
||
- FALLBACK_HOST1_WRITEBACK_TIER3=(
|
||
- # "location-placeholder"
|
||
- )
|
||
-
|
||
- FALLBACK_HOST1_WRITEBACK_TIER4=(
|
||
- "/mnt/user/appdata-Fallback/Arrs_Stack" # arr databases — downloads queued during outage
|
||
- )
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-
|
||
-# ━━━ Media Permissions ━━━
|
||
-# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to.
|
||
-# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
|
||
- HOST1_MEDIA_PERMISSION_SHARES=(
|
||
- /mnt/user/Anime_Movies
|
||
- /mnt/user/Anime_Movies-Old
|
||
- /mnt/user/Anime_Shows
|
||
- /mnt/user/Anime_Shows-Old
|
||
- /mnt/user/appcache
|
||
- /mnt/user/Books
|
||
- /mnt/user/Downloads
|
||
- /mnt/user/Games
|
||
- /mnt/user/Intros
|
||
- /mnt/user/Kids_Movies
|
||
- /mnt/user/Kids_Tv_Shows
|
||
- /mnt/user/Movie_Recordings
|
||
- /mnt/user/Movies
|
||
- /mnt/user/Music
|
||
- /mnt/user/Music_Videos
|
||
- /mnt/user/Photo
|
||
- /mnt/user/Sports
|
||
- /mnt/user/stand-up_comedy
|
||
- /mnt/user/Temp_Storage
|
||
- /mnt/user/Tv_Recordings
|
||
- /mnt/user/Tv_Shows
|
||
- /mnt/user/YouTube
|
||
- )
|
||
-
|
||
-# ━━━ Media Cleaner ━━━
|
||
-# Folder lists for media_cleaner.sh — two profiles: anime and media.
|
||
-# File patterns shared across all servers — defined in master.conf.
|
||
-# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
|
||
- HOST1_ANIME_CLEAN_FOLDERS=(
|
||
- /mnt/user/Anime_Movies
|
||
- /mnt/user/Anime_Movies-Old
|
||
- /mnt/user/Anime_Shows
|
||
- /mnt/user/Anime_Shows-Old
|
||
- )
|
||
-
|
||
- HOST1_MEDIA_CLEAN_FOLDERS=(
|
||
- /mnt/user/Kids_Movies
|
||
- /mnt/user/Kids_Tv_Shows
|
||
- /mnt/user/Movies
|
||
- /mnt/user/Music
|
||
- /mnt/user/Sports
|
||
- /mnt/user/stand-up_comedy
|
||
- /mnt/user/Tv_Shows
|
||
- )
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-
|
||
-# ━━━ Certificate Monitor ━━━
|
||
-# Domains checked via direct openssl connection — not relying on NPM's certificate state.
|
||
-# Checks the actual certificate served, not what NPM thinks it has.
|
||
-# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf.
|
||
- HOST1_CERT_MONITOR_DOMAINS=(
|
||
- "Gmer4Lfe.com"
|
||
- "Gmer4Lfe.us"
|
||
- )
|
||
-
|
||
-# ━━━ SMART Health ━━━
|
||
-# Drives skipped in SMART attribute monitoring — hardware is server-specific.
|
||
-# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf.
|
||
- HOST1_SMART_IGNORE_DRIVES=(
|
||
- "sda" # boot USB — SMART not meaningful on flash drives
|
||
- )
|
||
-
|
||
-# ━━━ ZFS Report ━━━
|
||
-# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools.
|
||
-# These are individual array disks formatted as ZFS — converting to XFS over time via unBalance.
|
||
-# Pool health thresholds defined in master.conf.
|
||
- HOST1_ZFS_REPORT_IGNORE_POOLS=(
|
||
- "disk5"
|
||
- "disk6"
|
||
- "disk8"
|
||
- "disk9"
|
||
- "disk10"
|
||
- )
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-
|
||
-# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront.
|
||
-# Real-world: 9 streams peaked at ~5.5GB — 10G gives generous headroom on 128GB RAM.
|
||
- HOST1_RAMDISK_SIZE="10G"
|
||
-
|
||
-# Usage thresholds — coupled to HOST1_RAMDISK_SIZE, adjust all three together if size changes.
|
||
-# Hysteresis gap (8.5 - 7 = 1.5GB) prevents flip-flop between ramdisk and SSD.
|
||
- HOST1_RAMDISK_WARN_GB=8.5 # flip to SSD when ramdisk usage reaches this
|
||
- HOST1_RAMDISK_LOW_GB=7 # flip back to ramdisk when usage drops to this
|
||
-
|
||
-# SSD fallback path — where transcodes land when ramdisk exceeds HOST1_RAMDISK_WARN_GB.
|
||
-# Must be on cache pool — array disks too slow for active transcode writes.
|
||
- HOST1_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
|
||
-
|
||
-# Media servers sharing the ramdisk transcode space on HOST1.
|
||
-# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
|
||
-# Entries with placeholder API keys are skipped automatically.
|
||
-# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
|
||
- HOST1_TRANSCODE_SERVERS=(
|
||
- "${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby"
|
||
- "${HOST1_JELLYFIN_CONTAINER}|${HOST1_JELLYFIN_URL}|${HOST1_JELLYFIN_API_KEY}|jellyfin"
|
||
- )
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── ARR STACK ─────────────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh.
|
||
-# detect_hosts() selects HOST1 vars when running on HOST1.
|
||
-#
|
||
-# PATH MAPS — container path → host path translation.
|
||
-# Arr stores file paths using container-internal paths — scripts need host paths to scan.
|
||
-# Add one entry per root folder in arr Settings → Media Management → Root Folders.
|
||
-
|
||
-# ━━━ Downloaders ━━━
|
||
-# Used by downloaders_reset.sh — runs every 30min via CRITICAL_MAINTENANCE_SCRIPTS.
|
||
-# Clears stuck states, purges old history, prepares each client for a clean cycle.
|
||
-
|
||
-# slskd — clears stuck searches, dead transfers, purges expired failed imports.
|
||
-# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected.
|
||
- HOST1_SLSKD_URL="http://localhost:8980"
|
||
- HOST1_SLSKD_API_KEY="" # slskd Settings → API key
|
||
- HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports"
|
||
-
|
||
-# SABnzbd
|
||
- HOST1_SABNZBD_URL="http://localhost:8180"
|
||
- HOST1_SABNZBD_API_KEY="" # SABnzbd Config → General → API Key
|
||
-
|
||
-# qBittorrent — deleteFiles=false removes torrent from qBit but leaves files on disk.
|
||
-# Radarr/Sonarr manage actual files independently.
|
||
- HOST1_QBIT_URL="http://localhost:8080"
|
||
- HOST1_QBIT_USERNAME="root"
|
||
- HOST1_QBIT_PASSWORD="" # qBittorrent WebUI password
|
||
-
|
||
-# ━━━ Lidarr — HOST1 only ━━━
|
||
-# HOST2 does not run Lidarr — HOST1_LIDARR_RECOVERY flag handles the exit cleanly.
|
||
- HOST1_LIDARR_URL="http://localhost:8686"
|
||
- HOST1_LIDARR_API_KEY="" # Lidarr Settings → General → API Key
|
||
- HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New"
|
||
- HOST1_FANART_API_KEY="" # fanart.tv account → API Key
|
||
- HOST1_LASTFM_API_KEY="" # last.fm API → Create API Account
|
||
-
|
||
- declare -A HOST1_LIDARR_PATH_MAP=(
|
||
- ["/ext-music"]="/mnt/user/Music-New"
|
||
- )
|
||
-
|
||
-# ━━━ Sonarr ━━━
|
||
- HOST1_SONARR_URL="http://localhost:8989"
|
||
- HOST1_SONARR_API_KEY="" # Sonarr Settings → General → API Key
|
||
- HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows"
|
||
-
|
||
-# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder
|
||
- declare -A HOST1_SONARR_PATH_MAP=(
|
||
- ["/tv"]="/mnt/user/Tv_Shows"
|
||
- ["/ext-standup-comedy"]="/mnt/user/stand-up_comedy"
|
||
- ["/kids tv"]="/mnt/user/Kids_Tv_Shows"
|
||
- ["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old"
|
||
- )
|
||
-
|
||
-# ━━━ Radarr ━━━
|
||
- HOST1_RADARR_URL="http://localhost:7878"
|
||
- HOST1_RADARR_API_KEY="" # Radarr Settings → General → API Key
|
||
- HOST1_TMDB_API_KEY="" # themoviedb.org → Settings → API
|
||
- HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies"
|
||
-
|
||
-# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder
|
||
- declare -A HOST1_RADARR_PATH_MAP=(
|
||
- ["/movies"]="/mnt/user/Movies"
|
||
- ["/kids movies"]="/mnt/user/Kids_Movies"
|
||
- ["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy"
|
||
- ["/anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||
- )
|
||
-
|
||
-# ━━━ Arr Recovery Toggles ━━━
|
||
-# false = skip that arr on this host — exits cleanly without error
|
||
- HOST1_SONARR_RECOVERY=true
|
||
- HOST1_RADARR_RECOVERY=true
|
||
- HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-# Per-host check toggles and NIC config for system_watchdog.sh.
|
||
-# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names.
|
||
-# HOST1: TR1950X 128GB — full media server, active transcoding, ZFS cache pools.
|
||
-#
|
||
-# Three-tier response — all critical checks enabled by default on HOST1:
|
||
-# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot
|
||
-# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle
|
||
-# Tier 3 (standard strike system): everything else
|
||
-#
|
||
-# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section.
|
||
-
|
||
-# ━━━ Primary NIC ━━━
|
||
-# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]"
|
||
-# Common values: eth0, bond0, br0, eno1
|
||
- HOST1_SYS_WATCHDOG_NIC="eth0"
|
||
-
|
||
-# ━━━ Tier 1 — Critical Checks ━━━
|
||
-# These bypass the strike system — a single hit triggers immediate reboot.
|
||
-# Disabling any of these is not recommended — they protect against acute system failure.
|
||
-
|
||
-# Docker daemon unresponsive → try restart, reboot if restart fails.
|
||
-# Without a working daemon docker_watchdog.sh is blind and containers cannot be managed.
|
||
- HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
|
||
-
|
||
-# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately.
|
||
-# At 99% rootfs writes fail silently — logs stop, Docker errors out, SSH may stop working.
|
||
-# Standard 95% threshold still uses strike system — only 99%+ is critical tier.
|
||
- HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true
|
||
-
|
||
-# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately.
|
||
-# A kernel oops means the kernel ran with a corrupted state — stability is not guaranteed.
|
||
- HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true
|
||
-
|
||
-# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately.
|
||
-# At 95% FD: new connections fail, Docker can't spawn processes, SSH drops.
|
||
- HOST1_SYS_WATCHDOG_CHECK_FD=true
|
||
-
|
||
-# /boot read-only detected → reboot immediately.
|
||
-# Unexpected read-only /boot means state files and config writes are silently failing.
|
||
-# Fallback state, watchdog reboot log, and lock files all go stale silently.
|
||
- HOST1_SYS_WATCHDOG_CHECK_BOOT=true
|
||
-
|
||
-# ━━━ Tier 2 — Urgent OOM Check ━━━
|
||
-# Bypass strikes when RAM is critically low AND OOM kill rate confirms active crisis.
|
||
-# Both must be enabled for Tier 2 bypass to function — disable either to always use strikes.
|
||
-
|
||
-# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta.
|
||
-# Also provides diagnostic context in reboot messages (which processes were killed).
|
||
- HOST1_SYS_WATCHDOG_CHECK_OOM=true
|
||
-
|
||
-# Free RAM check — required for both Tier 2 bypass and RAM tier logic.
|
||
-# Tiers: MEM_WARN_GB(10) → notify | MEM_SHUTDOWN_GB(6) → stop containers | MEM_GB(4) → strikes
|
||
- HOST1_SYS_WATCHDOG_CHECK_RAM=true
|
||
-
|
||
-# ━━━ Tier 3 — Standard Checks (strike system) ━━━
|
||
-# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action is taken.
|
||
-# Single spikes are ignored — sustained problems trigger reboot.
|
||
-
|
||
-# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT.
|
||
-# Log spam (Docker log storms, syslog loops) fills rootfs — indicates something broken.
|
||
- HOST1_SYS_WATCHDOG_CHECK_LOG=true
|
||
-
|
||
-# ZFS ARC memory pinned above SYS_WATCHDOG_ARC_PINNED_PCT after cache drop.
|
||
-# Enabled on HOST1 — ZFS cache pools actively used. Disable on hosts without ZFS.
|
||
- HOST1_SYS_WATCHDOG_CHECK_ARC=true
|
||
-
|
||
-# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C).
|
||
-# Sustained high temp causes kernel throttling or panic. Requires lm-sensors.
|
||
- HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
||
-
|
||
-# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count.
|
||
-# DISABLED on HOST1 — Tdarr and Emby cause legitimate sustained load spikes during encoding.
|
||
-# Enable on idle servers or adjust SYS_WATCHDOG_LOAD_MULTIPLIER if load is always high.
|
||
- HOST1_SYS_WATCHDOG_CHECK_LOAD=false
|
||
-
|
||
-# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50).
|
||
-# Large zombie counts indicate serious process management failure — something is stuck.
|
||
- HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||
-
|
||
-# Check docker_watchdog.sh persistent skip list — required containers on skip list.
|
||
-# Cross-watchdog coordination: if docker_watchdog gave up, system_watchdog escalates.
|
||
-# ENABLED — HOST1 fully built and operational, skip list is meaningful.
|
||
- HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true
|
||
-
|
||
-# /tmp filesystem usage above SYS_WATCHDOG_TMP_PCT with auto-clear attempt.
|
||
-# Script tries to clear aged /tmp files first — only strikes if clear fails.
|
||
-# Lock files, rsync temp files, and Docker ops use /tmp — 100% means lock failures.
|
||
- HOST1_SYS_WATCHDOG_CHECK_TMP=true
|
||
-
|
||
-# Array disk error count delta in /proc/mdstat — accumulating errors = disk failing now.
|
||
-# Triggers on SYS_WATCHDOG_MDSTAT_ERROR_LIMIT new errors in one cycle.
|
||
- HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true
|
||
-
|
||
-# Primary NIC operstate — detects NIC going down (physical or driver failure).
|
||
-# Uses HOST1_SYS_WATCHDOG_NIC above. Strike system — brief flaps don't trigger reboot.
|
||
- HOST1_SYS_WATCHDOG_CHECK_NETWORK=true
|
||
-
|
||
-# sshd running check — attempts restart before escalating.
|
||
-# sshd down = no remote access. Script tries rc.sshd start, notifies, strikes on failure.
|
||
- HOST1_SYS_WATCHDOG_CHECK_SSHD=true
|
||
-
|
||
-# Runaway process detection — single process above SYS_WATCHDOG_RUNAWAY_CPU_PCT sustained.
|
||
-# DISABLED — Tdarr encoding and Emby transcoding legitimately peg CPU for extended periods.
|
||
-# Enable only if HOST1 has no CPU-intensive workloads.
|
||
- HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-# Containers to manage under pressure — see master.conf RW_CRITICAL_CONTAINERS for exclusions.
|
||
-
|
||
-# docker pause at medium pressure (RAM < RW_RAM_MEDIUM_GB or load > medium threshold)
|
||
-# Suspended in-place — instant to pause/unpause, no state lost, no restart delay.
|
||
- HOST1_RW_PAUSE_CONTAINERS=(
|
||
- "Huntarr" # arr search automation — safe to suspend
|
||
- "Cleanuparr" # download cleanup — safe to suspend
|
||
- "Healarr" # arr health checks — safe to suspend
|
||
- "Soularr" # Slskd automation — background only
|
||
- "ChannelTube" # YouTube archiver — background only
|
||
- "Pinchflat" # YouTube archiver — background only
|
||
- )
|
||
-
|
||
-# docker stop at hard pressure (RAM < RW_RAM_HARD_GB)
|
||
-# Full stop — these are optional/heavy services that free significant RAM when stopped.
|
||
-# resource_watchdog.sh restarts them when pressure fully clears (RAM >= RW_RAM_RECOVER_GB).
|
||
- HOST1_RW_STOP_CONTAINERS=(
|
||
- "LocalAI" # GPU/CPU heavy — largest RAM consumer when idle
|
||
- "7DaysToDie" # game server — optional
|
||
- "V-Rising" # game server — optional
|
||
- "Code-Server" # IDE — not needed during pressure events
|
||
- )
|
||
-
|
||
-# ==============================================================================================
|
||
-# ──────────────────────── End Of HOST1 Variables ──────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
\ No newline at end of file
|
||
diff --git a/Deployment/conf_templates/master.conf b/Deployment/conf_templates/master.conf
|
||
deleted file mode 100644
|
||
index ba51bf7..0000000
|
||
--- a/Deployment/conf_templates/master.conf
|
||
+++ /dev/null
|
||
@@ -1,1433 +0,0 @@
|
||
-#!/bin/bash
|
||
-# ==============================================================================================
|
||
-# ================================= MASTER CONFIGURATION =======================================
|
||
-# ==============================================================================================
|
||
-# Shared configuration for the unRAID script ecosystem.
|
||
-# Contains all settings that apply to every server — thresholds, toggles, profiles, job lists.
|
||
-#
|
||
-# ── HOW THE THREE-FILE SYSTEM WORKS ──────────────────────────────────────────────────────────
|
||
-# Scripts source all three files at startup:
|
||
-# source master.conf ← shared config (this file)
|
||
-# source host1.conf ← HOST1 credentials, shares, container lists
|
||
-# source host2.conf ← HOST2 credentials, shares, container lists
|
||
-#
|
||
-# Sparse checkout (git) ensures each server only pulls its own host*.conf.
|
||
-# HOST2 never sees HOST1 credentials. HOST1 never sees HOST2 credentials.
|
||
-#
|
||
-# What belongs here: thresholds, toggles, intervals, profiles, job lists
|
||
-# What belongs in HOST*: hostnames, SSH keys, API keys, passwords, share paths,
|
||
-# container names, failover lists, watchdog containers
|
||
-#
|
||
-# ── HOW THIS FILE WORKS ───────────────────────────────────────────────────────────────────────
|
||
-# Every script sources all three conf files and common.sh at startup.
|
||
-# Change a value here and it affects all scripts that use it — no hunting through files.
|
||
-# To disable something: comment it out with # rather than deleting it.
|
||
-# To add a new rsync profile: add a key to each PROFILE_* array in the RSYNC section.
|
||
-# To add or remove orchestrator jobs: edit the arrays in the ORCHESTRATORS section.
|
||
-#
|
||
-# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||
-#
|
||
-# Section Description
|
||
-# ───────────────────────────────────────────────────────────────────────────────────────────
|
||
-# SHARED HOST CONFIGURATION DATA_DIR and shared connection settings
|
||
-# PARTNERSHIP Mirror relationship lifecycle — onboard/offboard/transfer
|
||
-# LOGGING Enable or disable verbose logging
|
||
-# NOTIFICATIONS unRAID native and Discord webhook settings
|
||
-# GIT / REPO Gitea repository and SSH settings
|
||
-#
|
||
-# ── ORCHESTRATORS ──────────────────────────────────────────────────────────────────────────
|
||
-# ARRAY START Scripts launched at array start (array_started.sh)
|
||
-# ARRAY STOP Scripts run at planned shutdown (array_stopping.sh)
|
||
-# WATCHDOG ORCHESTRATOR Per-minute watchdog runner (watchdog_orchestrator.sh)
|
||
-# SYSTEM WATCHDOG Sub-scripts called by watchdog_orchestrator.sh
|
||
-# CRITICAL SYNC MAINTENANCE 30-minute jobs + sync shares + partnership check (critical_sync_maintenance.sh)
|
||
-# INTERMEDIATE SYNC MAINTENANCE Arr sync + optional mid-day rsync (intermediate_sync_maintenance.sh)
|
||
-# DAILY SYNC MAINTENANCE Job list + media shares (daily_sync_maintenance.sh)
|
||
-# WEEKLY SYNC MAINTENANCE Job list + sync shares + update toggles (weekly_sync_maintenance.sh)
|
||
-# MONTHLY MAINTENANCE Uptime-triggered heavy tasks — ZFS scrub, SMART tests (monthly_maintenance.sh)
|
||
-#
|
||
-# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||
-# RSYNC ENABLE/DISABLE Two-tier toggle system — global gate + per-orchestrator
|
||
-# RSYNC DEFAULTS Global fallback rsync options and limits
|
||
-# REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort
|
||
-# RSYNC PROFILE SYSTEM Per-profile overrides for appdata syncs
|
||
-#
|
||
-# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||
-# FALLBACK Mutual container failover shared settings
|
||
-# FALLBACK TEST Simulated outage settings for fallback_test.sh
|
||
-#
|
||
-# ── DOCKER ESSENTIALS ──────────────────────────────────────────────────────────────────────
|
||
-# DOWNLOADERS RESET Retention and thresholds for slskd, SABnzbd, qBittorrent
|
||
-# DOCKER DAILY RESTART Containers restarted daily
|
||
-# DOCKER WEEKLY RESTART Containers restarted weekly
|
||
-# DOCKER WATCHDOG Thresholds and toggles for container monitoring
|
||
-# DOCKER NETWORK CONNECT Networks to ensure + containers to connect
|
||
-#
|
||
-# ── UNRAID ESSENTIALS ──────────────────────────────────────────────────────────────────────
|
||
-# INOTIFY TUNING inotify limits — raised at array start by inotify_tuning.sh
|
||
-# SYSTEM TUNING MONITOR Tracks inotify + php-fpm usage over time
|
||
-# REBOOT User warning delay before scheduled reboot
|
||
-# MOVER Mover stop timeout
|
||
-# SYSLOG FILTER Docker veth noise filter file path
|
||
-# PHP-FPM PHP-FPM max children config
|
||
-# CLEAR LOGS System log file paths
|
||
-# WEBGUI WATCHDOG WebGUI nginx + emhttp monitoring and restart
|
||
-#
|
||
-# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||
-# MEDIA PERMISSIONS Share list, mode and owner for permissions script
|
||
-# MEDIA CLEANER Anime and media folder lists and file patterns
|
||
-# ARR CLEANUP Lidarr, Sonarr, Radarr orphan file cleanup settings
|
||
-# ARR FAILED/STALLED RECOVERY Auto blocklist + re-search settings
|
||
-#
|
||
-# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||
-# TRANSCODE MANAGER Ramdisk and SSD fallback transcode management
|
||
-# TRANSCODE SERVER ARRAY Multi-server session monitoring (Emby, Jellyfin, Plex)
|
||
-#
|
||
-# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
|
||
-# CERTIFICATE MONITOR SSL certificate expiry monitoring
|
||
-# BACKUP VERIFY Random sample checksum verification against remote
|
||
-# SMART HEALTH Drive SMART attribute monitoring
|
||
-# ZFS MEMORY SNAPSHOT Weekly ZFS health and memory diagnostic report
|
||
-# BANDWIDTH MONITOR Daily rsync transfer logging and weekly summary
|
||
-# HEALTH DIGEST Aggregated system health digest — always/smart/weekly
|
||
-# EMBY SESSION REPORT Weekly Emby usage statistics via API
|
||
-#
|
||
-# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||
-# SYSTEM WATCHDOG Continuous system health monitoring — last line of defense
|
||
-#
|
||
-# ==============================================================================================
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── HOST IDENTITIES ───────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-# Hostnames for every server in the ecosystem — not credentials, safe for all machines.
|
||
-# Must match the exact unRAID hostname AND Tailscale device name (case sensitive).
|
||
-# detect_hosts() in common.sh matches the local hostname against these to set MY_ID / REMOTE_ID.
|
||
-# Tailscale IP resolution uses these names — no hardcoded IPs needed.
|
||
-# To add a new server: add HOST3="unRAID-NewServer" here + create host3.conf.
|
||
- HOST1="unRAID-Gmer4Lfe"
|
||
- HOST2="unRAID-Jayred365"
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── SHARED HOST CONFIGURATION ─────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-# Requirement: internal NVMe/SSD boot drive. Everything lives in the plugin folder on flash —
|
||
-# scripts, state, and data are all available before the array mounts.
|
||
-# Both directories are created automatically if they don't exist.
|
||
-#
|
||
-# DATA_DIR — historical logs, statistics, discovery histories, blocklists
|
||
-# STATE_DIR — runtime state files for all scripts (watchdogs, fallback, transcode, etc.)
|
||
-# Requirement: ALL state files MUST use $STATE_DIR. No /tmp, no /boot/config root.
|
||
- DATA_DIR="/boot/config/plugins/varaverk/data"
|
||
- STATE_DIR="/boot/config/plugins/varaverk/State_Files"
|
||
-
|
||
-# ── Version Parity ──
|
||
-# Controls behaviour when local and remote unRAID versions differ.
|
||
-# Major version mismatch always aborts regardless of this setting.
|
||
-# "warn" — log warning and continue (safe for minor/patch differences)
|
||
-# "abort" — refuse to continue (strict — ensures both sides always match)
|
||
- UNRAID_VERSION_MISMATCH_ACTION="warn"
|
||
-
|
||
-# ── Arr Sync ──
|
||
-# arr_sync.sh syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally.
|
||
-# Runs before rsync — all nodes agree on tracked library before files are transferred.
|
||
-# Remote API keys are read live from each node's config.xml via SSH — never stored here.
|
||
- ARR_SYNC_ENABLED=true
|
||
- ARR_SYNC_BLOCKLIST="${DATA_DIR}/arr_sync_blocklist.tsv"
|
||
- ARR_SYNC_CONNECT_TIMEOUT=10 # seconds — SSH connect timeout per node
|
||
- ARR_SYNC_API_TIMEOUT=60 # seconds — curl timeout for library fetches
|
||
- DOCKER_APPDATA_BASE="/mnt/user/appdata"
|
||
- ARR_SYNC_LIDARR_PORT=8686
|
||
- ARR_SYNC_SONARR_PORT=8989
|
||
- ARR_SYNC_RADARR_PORT=7878
|
||
-
|
||
-# ── Remote Docker Daemon ──
|
||
-# Strike system for remote Docker daemon health checks.
|
||
-# Scripts that issue remote container commands check the remote daemon first.
|
||
-# Below limit → skip operation this run | At limit → notify critical + exit
|
||
- REMOTE_DOCKER_STRIKE_LIMIT=3 # consecutive failures before critical notify
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── PARTNERSHIP ───────────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-# Manages the relationship lifecycle between two unRAID servers.
|
||
-# HOST1 is always the owner (source of truth) — HOST2 is always the mirror.
|
||
-# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a --transfer operation.
|
||
-# All identity vars (hostnames, SSH keys) live in host*.conf.
|
||
-# Hostnames already match Tailscale device names — IP resolution is automatic.
|
||
-#
|
||
-# State files in STATE_DIR — survives reboots, array must be up (scripts run after array start):
|
||
-# STATE_DIR/partnership_HOST1.db ← HOST1 writes only
|
||
-# STATE_DIR/partnership_HOST2.db ← HOST2 writes only
|
||
-# STATE_DIR/partnership_blocklist.db ← shared blocklist
|
||
-# STATE_DIR/varaverk_setup.db ← onboarding progress state
|
||
-# Propagated via SSH — no rsync needed
|
||
-#
|
||
-# critical_sync_maintenance.sh runs --check every 30min:
|
||
-# Reads both state files via SSH
|
||
-# Detects offboard requests → finalises from owner side
|
||
-# Increments offline counter → auto-offboards after threshold
|
||
-# Silent when healthy ✅
|
||
-#
|
||
-# See README-Partnership.md for full lifecycle documentation.
|
||
-
|
||
- PARTNERSHIP_ENABLED=false
|
||
- PARTNERSHIP_OWNER_HOST="HOST1" # "HOST1" or "HOST2" — flips on --transfer
|
||
-
|
||
-# Auth containers reconfigured on onboard/offboard — defined per host in host*.conf.
|
||
-# Format: "ContainerName|WebUIPort"
|
||
-# HOST1_PARTNERSHIP_AUTH_WEBUIS / HOST2_PARTNERSHIP_AUTH_WEBUIS
|
||
-# On onboard → WebUI pointed at owner's Tailscale IP
|
||
-# On offboard → WebUI pointed back at localhost
|
||
-
|
||
-# Paths to collect during the grace window after offboard — defined per host in host*.conf.
|
||
-# HOST1_PARTNERSHIP_MIRROR_BACKUPS / HOST2_PARTNERSHIP_MIRROR_BACKUPS
|
||
-# Notified on offboard — no auto-deletion, manual collection.
|
||
-
|
||
-# Timing — single var controls both Tailscale removal and backup access expiry.
|
||
-# Both expire at the same time — keeping backups accessible beyond Tailscale removal is pointless.
|
||
- PARTNERSHIP_GRACE_HOURS=6 # hours after offboard before Tailscale removal
|
||
- # backup access expires at the same time
|
||
- PARTNERSHIP_OFFLINE_THRESHOLD=30 # days either server unreachable before auto-offboard
|
||
- # works both directions independently
|
||
-
|
||
-# Partnership and setup state files — all in STATE_DIR per the project requirement.
|
||
-# Scripts use these variables; do not hardcode /boot/config paths in scripts.
|
||
- PARTNERSHIP_BLOCKLIST_FILE="$STATE_DIR/partnership_blocklist.db"
|
||
- VARAVERK_SETUP_FILE="$STATE_DIR/varaverk_setup.db"
|
||
-
|
||
-# Tailscale removal on offboard.
|
||
- PARTNERSHIP_REMOVE_TAILSCALE=true # remove mirror from Tailscale tailnet on offboard
|
||
- # false = skip removal (manual or testing)
|
||
-
|
||
-# Tailscale API — required when PARTNERSHIP_REMOVE_TAILSCALE=true.
|
||
-# Stays in shared conf — only owner uses it, and owner is always running this script.
|
||
-# API key: https://login.tailscale.com/admin/settings/keys → Devices write scope
|
||
- TAILSCALE_API_KEY="" # tskey-api-...
|
||
- TAILSCALE_TAILNET="" # your tailnet name (e.g. yourname.github)
|
||
-
|
||
-# Transfer safety.
|
||
- PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership"
|
||
- PARTNERSHIP_TRANSFER_STRIKES=3 # consecutive health checks required
|
||
- PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20 # max health check attempts before giving up
|
||
-
|
||
-# Onboard settings.
|
||
- PARTNERSHIP_ONBOARD_VERIFY=true # verify WebUI reachable after reconfiguration
|
||
- PARTNERSHIP_ONBOARD_NOTIFY=true # notify both servers on completion
|
||
- PARTNERSHIP_SYNC_INTERVAL=15 # minutes — informational, actual schedule in cron
|
||
-
|
||
-# SSH key lifecycle — managed by Partnership/ssh_setup.sh.
|
||
-# Key named after this server: hostname lowercased, unraid- prefix stripped.
|
||
-# unRAID-Gmer4Lfe → /root/.ssh/gmer4lfe_rsync_automation
|
||
-# Run Partnership/partnership_onboard.sh to generate, copy, and update conf automatically.
|
||
- SSH_MAX_STRIKES=5 # consecutive SSH auth failures before critical notify
|
||
- SSH_STRIKE_RESET_HRS=24 # hours since last failure before strike counter resets
|
||
-
|
||
-# FolderView3 plugin integration — chodeus/VladoPortos/scolcipitato.
|
||
-# Creates a "{Partner}-Failover" folder on onboard, removes + cleans containers on offboard.
|
||
-# Folder name derived from remote hostname (strip unraid- prefix case-insensitively).
|
||
-# Plugin config: /boot/config/plugins/folder.view3/docker.json
|
||
- PARTNERSHIP_FOLDERVIEW3=true # create/remove FolderView3 folder on onboard/offboard
|
||
- PARTNERSHIP_FOLDERVIEW3_URL="" # CA plugin URL — leave empty to skip auto-install
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── LOGGING ───────────────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-
|
||
-# Controls verbose [LOG] output across all scripts.
|
||
-# true = show detailed [LOG] lines — useful for debugging or first-time setup
|
||
-# false = show only user-facing output — cleaner for scheduled runs
|
||
- ENABLE_LOGGING=false
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── NOTIFICATIONS ─────────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-
|
||
-# unRAID native notification system — integrates with the bell icon in the WebGUI.
|
||
-# normal = job completed successfully / warning = something failed or needs attention
|
||
- NOTIFY_UNRAID=true
|
||
-
|
||
-# Discord webhook URL — defined per host in host*.conf.
|
||
-# HOST1_DISCORD_WEBHOOK / HOST2_DISCORD_WEBHOOK
|
||
-# Allows different webhooks per server, or only one server notifying.
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── GIT / REPO ────────────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-
|
||
-# Gitea self-hosted repository — used by git_pull_execute.sh.
|
||
-# Detects Gitea container location at runtime — works through failover automatically.
|
||
-# Falls back to GITEA_DOMAIN if local and Tailscale both fail.
|
||
- GITEA_CONTAINER="Gitea"
|
||
- GITEA_REPO_PATH="FailedProxy/Varaverk.git"
|
||
- GITEA_DOMAIN="" # e.g. git.yourdomain.com — requires NPM + DNS
|
||
- TARGET_DIR="/boot/config/plugins/varaverk"
|
||
- GITEA_SSH_KEY="/root/.ssh/unraid_gitea"
|
||
- SSH_PORT=221 # Gitea SSH port (default 22, Gitea often uses 221/222)
|
||
- GITEA_HTTP_PORT=3000 # Gitea web/API port — used by gitea_ssh_setup.sh
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── ORCHESTRATORS ─────────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-# All orchestrator job lists live here — edit arrays to add/remove scripts.
|
||
-# No changes to orchestrator scripts needed when adding or removing jobs.
|
||
-# Sections ordered by run frequency: array events first, then shortest interval to longest.
|
||
-
|
||
-# ━━━ Array Start ━━━
|
||
-# Scripts launched by array_started.sh when the array comes online.
|
||
-# Launched in order — each as a background process.
|
||
-# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally.
|
||
-# Continuous scripts (failover) run until array stops.
|
||
-# Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via
|
||
-# watchdog_orchestrator.sh — NOT launched here.
|
||
- ARRAY_START_SCRIPTS=(
|
||
- "Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts
|
||
- "System_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill
|
||
- "Plugin/unraid/System_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning
|
||
- "System_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted
|
||
- "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers
|
||
- "Fallback/fallback.sh" # mutual failover — continuous
|
||
- )
|
||
-
|
||
-# ━━━ Array Stop ━━━
|
||
-# Scripts run by array_stopping.sh for a planned shutdown — stops everything cleanly in order.
|
||
-# Run sequentially (foreground) — each must complete before the next starts.
|
||
-# Order matters: user scripts first (prevents new ops), then data movement, then containers.
|
||
- ARRAY_STOP_SCRIPTS=(
|
||
- "Plugin/unraid/System_Essentials/user_scripts_stop.sh" # stop background scripts before they start new ops
|
||
- "Fallback/fallback.sh --stop" # gracefully stop fallback (not caught by user_scripts_stop)
|
||
- "System_Essentials/rsync_stop.sh --rsync-only" # kill rsync; skip container recovery (handled below)
|
||
- "Plugin/unraid/System_Essentials/mover_stop.sh" # stop mover after rsync (they conflict on same files)
|
||
- "Docker_Essentials/docker_container_stop.sh" # stop all containers last
|
||
- )
|
||
-
|
||
-# ━━━ Watchdog Orchestrator ━━━
|
||
-# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
|
||
-# Schedule: * * * * * (every minute)
|
||
-# NOT in ARRAY_START_SCRIPTS — has its own cron entry.
|
||
-# Order matters — resource first (frees pressure), docker second (heals with freed resources),
|
||
-# system third (storage + webgui component health), stability last (last line of defense).
|
||
- WATCHDOG_ORCHESTRATOR_SCRIPTS=(
|
||
- "Watchdogs/resource_watchdog.sh" # reduce system pressure before healing attempts
|
||
- "Watchdogs/docker_watchdog.sh" # heal containers with freed resources
|
||
- "Watchdogs/system_watchdog.sh" # system component health — storage + webgui
|
||
- "Watchdogs/stability_watchdog.sh" # reboot if all else fails — last line of defense
|
||
- )
|
||
-
|
||
-# ━━━ System Watchdog ━━━
|
||
-# system_watchdog.sh runs SYSTEM_WATCHDOG_SCRIPTS sequentially each cycle.
|
||
-# Called by watchdog_orchestrator.sh — not scheduled directly.
|
||
- SYSTEM_WATCHDOG_SCRIPTS=(
|
||
- "Watchdogs/System/storage_watchdog.sh" # pool growth + runaway log detection
|
||
- "Plugin/unraid/Watchdogs/System/webgui_watchdog.sh" # WebGUI availability — nginx → php-fpm → emhttp
|
||
- "Watchdogs/System/network_watchdog.sh" # internet, DDNS, Tailscale, NPM proxy
|
||
- )
|
||
-
|
||
-# ━━━ Critical Sync Maintenance ━━━
|
||
-# critical_sync_maintenance.sh runs every 30 minutes.
|
||
-# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check
|
||
-# partnership --check always runs last regardless of rsync gate.
|
||
-# Comment out entries to disable without removing.
|
||
- CRITICAL_MAINTENANCE_SCRIPTS=(
|
||
- "Docker_Essentials/downloaders_reset.sh" # clear stuck download states every 30min
|
||
- )
|
||
-
|
||
-# Shares synced every 30 minutes — defined per host in host*.conf.
|
||
-# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES
|
||
-# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||
-# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
|
||
-
|
||
-# ━━━ Intermediate Sync Maintenance ━━━
|
||
-# intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch,
|
||
-# and optional mid-day rsync for any shares that need sub-daily propagation.
|
||
-# Schedule: 0 */4 * * *
|
||
- # INTERMEDIATE_SYNC_SHARES is host-specific — configure HOST*_INTERMEDIATE_SYNC_SHARES in host*.conf.
|
||
-
|
||
- INTERMEDIATE_MAINTENANCE_SCRIPTS=(
|
||
- "Media/arrs_failed_stalled_recovery.sh" # blocklist + re-search failed/stalled arr queue items
|
||
- )
|
||
- # arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here.
|
||
- # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).
|
||
-
|
||
-# ━━━ Daily Sync Maintenance ━━━
|
||
-# daily_sync_maintenance.sh runs media share sync first, then iterates
|
||
-# DAILY_MAINTENANCE_SCRIPTS for all jobs.
|
||
-# Schedule: 0 1 * * * (1am daily)
|
||
- DAILY_MAINTENANCE_SCRIPTS=(
|
||
- "git_pull_execute.sh" # pull latest scripts — always runs first
|
||
- "Media/media_shares_permissions.sh" # apply permissions — runs before cleaners
|
||
- "Media/media_cleaner.sh anime" # remove junk from anime shares
|
||
- "Media/media_cleaner.sh media" # remove junk from media shares
|
||
- #"Media/lidarr_cleanup.sh" # remove orphaned music files — enable when ready
|
||
- #"Media/sonarr_cleanup.sh" # remove orphaned TV files — enable when ready
|
||
- "Media/radarr_cleanup.sh" # remove orphaned movie files
|
||
- "Media/lidarr_missing_art.sh" # fetch missing album/artist artwork (HOST1 only — self-guards)
|
||
- "Media/radarr_tmdb_removed.sh" # remove movies dropped from TMDb
|
||
- "Media/sonarr_tvdb_removed.sh" # remove series dropped from TVDB
|
||
- "Docker_Essentials/docker_update.sh" # pull container image updates before restart
|
||
- "Docker_Essentials/docker_daily_restart.sh" # daily container restarts — runs last
|
||
- )
|
||
-
|
||
-# Pull latest images for DAILY_RESTART_CONTAINERS before the daily restart.
|
||
-# Containers keep running during pull — no extra downtime.
|
||
-# Set false to skip updates while still running the daily restart.
|
||
- DAILY_CONTAINER_UPDATES=true
|
||
-
|
||
-# Media shares synced daily by daily_sync_maintenance.sh.
|
||
-# Defined per-host in host*.conf — HOST1_DAILY_SYNC_SHARES and HOST2_DAILY_SYNC_SHARES.
|
||
-# Mesh model: every node pushes every media share. rsync has no --delete so pushes are additive.
|
||
-# arr_sync (union) ensures all arr libraries converge first. arr_cleanup removes true orphans.
|
||
-# Any node can download content to any share — it propagates to all nodes on the next cycle.
|
||
-# Adding HOST3: list every media share in HOST3_DAILY_SYNC_SHARES. No ownership to track.
|
||
-# These shares use DEFAULT_RSYNC_OPTS — no profile entry needed.
|
||
-# For shares needing custom options or container stops — create a profile in RSYNC section.
|
||
-
|
||
-# Personal encrypted shares defined per-host in host*.conf.
|
||
-# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
|
||
-# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
|
||
-
|
||
-# ━━━ Weekly Sync Maintenance ━━━
|
||
-# weekly_sync_maintenance.sh stops containers both sides → pulls updates →
|
||
-# syncs WEEKLY_SYNC_SHARES → restarts → then iterates WEEKLY_MAINTENANCE_SCRIPTS.
|
||
-# Schedule: 30 2 * * 0 (Sunday 2:30am)
|
||
- WEEKLY_MAINTENANCE_SCRIPTS=(
|
||
- "Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync
|
||
- "Docker_Essentials/docker_update_remaining.sh" # pull updates for all other containers
|
||
- "System_Essentials/clear_logs.sh" # purge aged logs — Sunday only, low priority
|
||
- "Media/playback_aware_lidarr_discovery.sh" # behavior-driven music discovery using weekly Emby playback history
|
||
- "Media/playback_aware_radarr_discovery.sh" # behavior-driven movie discovery using TMDB recommendations
|
||
- "Media/playback_aware_sonarr_discovery.sh" # behavior-driven TV discovery using TMDB recommendations
|
||
- )
|
||
-
|
||
-# Pull updates for all running containers NOT in daily/weekly restart lists.
|
||
-# Ensures every deployed container receives at least one image pull per week.
|
||
-# Set false to skip — docker_weekly_restart.sh still runs regardless.
|
||
- WEEKLY_REMAINING_UPDATES=true
|
||
-
|
||
-# Shares synced during the weekly maintenance window — defined per host in host*.conf.
|
||
-# HOST1_WEEKLY_SYNC_SHARES / HOST2_WEEKLY_SYNC_SHARES
|
||
-# Containers stopped both sides before sync — full clean state guaranteed.
|
||
-# Profiles drive container stops, excludes, and options — configure in RSYNC section.
|
||
-# Order matters — Emby first (larger), then Critical-Data (auth stack).
|
||
-
|
||
-# Container update toggles for the weekly sync window.
|
||
-# Containers already stopped for sync — updates pull at no extra downtime.
|
||
-# Both false → sync only, no updates.
|
||
- WEEKLY_SYNC_UPDATES=true # pull container updates locally during weekly window
|
||
- WEEKLY_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH
|
||
-
|
||
-# ━━━ Monthly Maintenance ━━━
|
||
-# monthly_maintenance.sh fires only when BOTH gates pass:
|
||
-# 1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days
|
||
-# 2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run)
|
||
-# Cron: 0 3 * * * (daily 3am check — script self-gates, calling daily is safe)
|
||
-# NOT in WATCHDOG_ORCHESTRATOR_SCRIPTS — has its own cron entry.
|
||
-# Add scripts that require a long-stable settled system — scrubs, extended drive tests.
|
||
-# State file on /boot/config — survives reboots (interval gate independent of uptime gate).
|
||
- MONTHLY_MAINTENANCE_SCRIPTS=(
|
||
- #"Tools/zfs_pool_scrub.sh" # ZFS pool integrity scrub — not yet built
|
||
- #"Tools/smart_long_test.sh" # SMART extended drive health test — not yet built
|
||
- )
|
||
-
|
||
- MONTHLY_UPTIME_THRESHOLD_DAYS=30 # minimum uptime in days before maintenance fires
|
||
- MONTHLY_RUN_INTERVAL_DAYS=30 # minimum days since last run before running again
|
||
- MONTHLY_LAST_RUN_FILE="$STATE_DIR/monthly_maintenance_last_run.db"
|
||
-
|
||
-# ━━━ Sunday Morning Coffee Report ━━━
|
||
-# Orchestrator that runs all Sunday monitor scripts in sequence.
|
||
-# Schedule: 0 7 * * 0 (Sunday 7am — after weekly_sync_maintenance.sh finishes at ~3am)
|
||
-# Each script runs independently and notifies on its own findings.
|
||
- COFFEE_REPORT_SCRIPTS=(
|
||
- "Monitors/zfs_memory_snapshot.sh" # ZFS pool health + ARC + Docker memory snapshot
|
||
- "Monitors/smart_health.sh" # drive SMART attributes — reallocated, pending, temp
|
||
- "Monitors/cert_monitor.sh" # SSL certificate expiry for all configured domains
|
||
- "Monitors/backup_verify.sh" # rsync mirror integrity via independent MD5 checksums
|
||
- "Monitors/bandwidth_monitor.sh" # weekly rsync transfer totals and per-share breakdown
|
||
- "Monitors/emby_session_report.sh" # Emby usage — streams, users, library, transcode ratio
|
||
- "Monitors/weekly_health_digest.sh" # aggregated digest — watchdogs, fallback, skip list
|
||
- )
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-
|
||
-# ━━━ Rsync Enable/Disable ━━━
|
||
-# Two-tier toggle system — Tier 1 overrides Tier 2.
|
||
-#
|
||
-# Tier 1 — Global gate:
|
||
-# RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions
|
||
-# Use when: remote completely offline, major maintenance, disaster recovery
|
||
-#
|
||
-# Tier 2 — Per-orchestrator (only applies when Tier 1 is true):
|
||
-# Fine grained control — disable specific orchestrators while keeping others
|
||
-# Use when: rebuilding secondary, testing, per-window bandwidth management
|
||
-#
|
||
-# Example — HOST2 data rebuild:
|
||
-# RSYNC_ENABLED=true ← rsync works, individual scripts run fine
|
||
-# CRITICAL_RSYNC_ENABLED=true ← 30min auth stack sync still runs
|
||
-# INTERMEDIATE_RSYNC_ENABLED=false ← skip 4h arr/mid-day rsync during rebuild
|
||
-# DAILY_RSYNC_ENABLED=false ← skip daily HDD syncs during rebuild
|
||
-# WEEKLY_RSYNC_ENABLED=true ← Emby + Critical-Data still sync (NVMe)
|
||
-# FALLBACK_RSYNC_ENABLED=true ← handback writeback still works
|
||
-# → Run individual: bash Rsync/rsync.sh /mnt/user/Movies
|
||
-# → When ready: INTERMEDIATE_RSYNC_ENABLED=true DAILY_RSYNC_ENABLED=true
|
||
- RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below
|
||
- CRITICAL_RSYNC_ENABLED=false # Tier 2 — disabled during HOST2 rebuild, re-enable when ready
|
||
- INTERMEDIATE_RSYNC_ENABLED=true # Tier 2 — intermediate_sync_maintenance.sh rsync section
|
||
- DAILY_RSYNC_ENABLED=false # Tier 2 — HOST2 rebuild in progress, re-enable when ready
|
||
- WEEKLY_RSYNC_ENABLED=true # Tier 2 — weekly_sync_maintenance.sh rsync section
|
||
- FALLBACK_RSYNC_ENABLED=true # Tier 2 — fallback.sh writeback jobs on handback
|
||
-
|
||
-# ━━━ Rsync Defaults ━━━
|
||
-# Global fallback values used when no profile match is found.
|
||
-# Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed.
|
||
-# Appdata shares match profiles by directory basename (lowercased).
|
||
- BW_LIMIT=12500 # KB/s — 12500 ≈ 100Mbit
|
||
- RETRY_COUNT=3 # retry attempts before giving up
|
||
- SLEEP=300 # seconds between retry attempts
|
||
- CRITICAL_CONTAINER_NAMES=() # containers stopped on REMOTE before rsync — profiles override
|
||
- DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override
|
||
- CONTAINER_DELAY=5 # seconds before starting delayed containers
|
||
- EXCLUDE_DIRS=() # directories excluded from transfer — profiles override
|
||
-
|
||
-# --inplace writes directly to destination — delta against existing file, better for large media
|
||
-# --partial keep partial file on interrupted transfer so next run resumes, not re-transfers
|
||
-# --timeout kill stalled transfers instead of hanging indefinitely
|
||
-# --numeric-ids use UIDs/GIDs numerically — prevents ownership mismatches between servers
|
||
-# --delete intentionally omitted — arr_cleanup.sh enforces media truth post-rsync.
|
||
-# Media shares use this default; rsync spreads files only, never removes them.
|
||
-# Note: --no-whole-file removed — redundant over SSH (delta transfer is already the default).
|
||
-# Note: arr_cleanup only catches true orphans under the union model — intentional removals
|
||
-# require arr_sync.sh --blocklist-add first, then manual file deletion.
|
||
- DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --inplace --partial --timeout=60 --numeric-ids)
|
||
-
|
||
-# ━━━ Remote Health Checks ━━━
|
||
-# Pre-flight — aborts if remote rootfs (/) usage is at or above this percentage.
|
||
-# When remote array is down, rsync writes land on rootfs and fill it rapidly.
|
||
- ROOTFS_WARN=75
|
||
-
|
||
-# ━━━ Rsync Profile System ━━━
|
||
-# Profiles allow per-share rsync behaviour without touching script logic.
|
||
-# Profile key matched by basename of directory passed to rsync.sh (lowercased).
|
||
-# Override with --profile=name flag.
|
||
-#
|
||
-# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit DEFAULT_RSYNC_OPTS.
|
||
-# List ALL desired options explicitly when defining a profile.
|
||
-#
|
||
-# Current profiles:
|
||
-# arrs_stack — arr databases — lower bandwidth, containers stopped for consistency
|
||
-# critical-data — auth stack — full stop both sides, Authelia delayed start
|
||
-# called by weekly_sync_maintenance.sh — full clean sync weekly
|
||
-# critical-fallback — dirty sync — auth stays running both sides, WAL excluded
|
||
-# called by critical_sync_maintenance.sh every 30min
|
||
-# host1-appdata — HOST1 server-specific appdata — defined in host1.conf
|
||
-# host2-appdata — HOST2 server-specific appdata — defined in host2.conf
|
||
-# important-data — NextCloud + Postgres — NextCloud delayed start after Postgres
|
||
-# emby — weekly clean sync — both Emby stopped, full mirror
|
||
-# called by weekly_sync_maintenance.sh only — do NOT schedule separately
|
||
-# emby-fallback — dirty sync — Emby stays running, WAL excluded
|
||
-# called by critical_sync_maintenance.sh every 30min
|
||
-
|
||
- declare -A PROFILE_RSYNC_OPTS=(
|
||
- [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace"
|
||
- [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete"
|
||
- [critical-fallback]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
|
||
- [important-data]="-av --human-readable --bwlimit=$BW_LIMIT"
|
||
- [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
|
||
- [emby-fallback]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
|
||
- )
|
||
-
|
||
-# Per-profile bandwidth limits in KB/s
|
||
- declare -A PROFILE_BW_LIMIT=(
|
||
- [arrs_stack]=5000 # lower — runs alongside other syncs
|
||
- [critical-data]=9500 # high — small dataset, sync fast
|
||
- [critical-fallback]=9500 # high — small dataset, sync fast
|
||
- [important-data]=9500 # high — database sync
|
||
- [emby]=8000 # medium — large full mirror
|
||
- [emby-fallback]=9500 # high — small critical dataset
|
||
- )
|
||
-
|
||
-# Retry attempts per profile
|
||
- declare -A PROFILE_RETRY_COUNT=(
|
||
- [arrs_stack]=3
|
||
- [critical-data]=3
|
||
- [critical-fallback]=3
|
||
- [important-data]=3
|
||
- [emby]=3
|
||
- [emby-fallback]=3
|
||
- )
|
||
-
|
||
-# Seconds between retry attempts
|
||
- declare -A PROFILE_SLEEP=(
|
||
- [arrs_stack]=300
|
||
- [critical-data]=300
|
||
- [critical-fallback]=120 # shorter — frequent dirty sync, retry faster
|
||
- [important-data]=300
|
||
- [emby]=300
|
||
- [emby-fallback]=120 # shorter — frequent dirty sync, retry faster
|
||
- )
|
||
-
|
||
-# Containers stopped on BOTH LOCAL and REMOTE before rsync.
|
||
-# Local stops first — flushes databases cleanly. Remote stops next — prevents writes.
|
||
-# Only running containers get restarted — stopped containers stay stopped.
|
||
-# SPACE-SEPARATED STRINGS — converted to array at runtime
|
||
- declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
|
||
- [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
|
||
- [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary"
|
||
- [critical-fallback]="" # dirty sync — auth stays running both sides
|
||
- [important-data]="Postgres-NextCloud NextCloud"
|
||
- [emby]="Emby"
|
||
- [emby-fallback]="" # dirty sync — Emby stays running both sides
|
||
- )
|
||
-
|
||
-# Containers needing a delay after rsync before starting.
|
||
-# SPACE-SEPARATED STRINGS — converted to array at runtime
|
||
- declare -A PROFILE_DELAYED_CONTAINERS=(
|
||
- [arrs_stack]=""
|
||
- [critical-data]="Authelia Authelia-Secondary" # wait for Mariadb + Redis
|
||
- [critical-fallback]=""
|
||
- [important-data]="NextCloud" # wait for Postgres
|
||
- [emby]=""
|
||
- [emby-fallback]=""
|
||
- )
|
||
-
|
||
-# Seconds before starting delayed containers
|
||
- declare -A PROFILE_CONTAINER_DELAY=(
|
||
- [arrs_stack]=5
|
||
- [critical-data]=15 # Mariadb + Redis need time to accept connections
|
||
- [critical-fallback]=5
|
||
- [important-data]=10 # Postgres needs time before NextCloud
|
||
- [emby]=5
|
||
- [emby-fallback]=5
|
||
- )
|
||
-
|
||
-# Directories excluded from rsync per profile.
|
||
-# SPACE-SEPARATED STRINGS — converted to array at runtime
|
||
- declare -A PROFILE_EXCLUDE_DIRS=(
|
||
- [arrs_stack]="logs *.tmp"
|
||
- [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt"
|
||
- [critical-fallback]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt *.db-wal *.db-shm"
|
||
- [important-data]="logs *.tmp"
|
||
- [emby]="logs transcodes cache crash*"
|
||
- [emby-fallback]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root"
|
||
- )
|
||
-
|
||
-# Remote restart after dirty sync — restart these on remote IF they were running before sync.
|
||
-# Same logic as stop/start — was stopped = stays stopped, was running = gets restarted.
|
||
-# Used by dirty sync profiles (critical-fallback, emby-fallback) so remote picks up changes.
|
||
-# SPACE-SEPARATED STRINGS — converted to array at runtime
|
||
- declare -A PROFILE_REMOTE_RESTART_CONTAINERS=(
|
||
- [critical-fallback]="NginxProxyManager Authelia Authelia-Secondary Lldap-Gmer4Lfe Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary"
|
||
- [emby-fallback]="Emby"
|
||
- )
|
||
-
|
||
-# Note: disk check is auto-detected from disks.ini — no PROFILE_SKIP_DISK_CHECK needed.
|
||
-# check_remote_disks() reads fsType per disk and handles XFS, ZFS, and cache pools automatically.
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-# Mutual container failover between two unRAID servers.
|
||
-# Each server runs Fallback/fallback.sh independently via array_started.sh.
|
||
-# All decisions based on two pings: remote reachable + internet reachable.
|
||
-#
|
||
-# States: NORMAL | FALLBACK | NO_INTERNET | DARK
|
||
-#
|
||
-# DDNS rules — absolute:
|
||
-# Internet loss → stop own DDNS immediately
|
||
-# Failover → start remote DDNS first (Tier 1)
|
||
-# Handback → stop remote DDNS → rsync → start containers → start local DDNS last
|
||
-#
|
||
-# Per-host container lists and tier delays live in host*.conf.
|
||
-# Shared settings (intervals, state file, thresholds) live here.
|
||
-
|
||
- EXTERNAL_IP="8.8.8.8"
|
||
- FALLBACK_CHECK_INTERVAL=30 # seconds between fallback state checks
|
||
- FALLBACK_HANDBACK_STRIKES=3 # consecutive healthy checks before initiating handback (3×30s = 90s)
|
||
- FALLBACK_STATE_FILE="$STATE_DIR/fallback_state.db"
|
||
- FALLBACK_ENABLED=true # HOST2 back online
|
||
- # false = suppresses "not running" warnings in status scripts
|
||
- FALLBACK_PARTNERSHIP_REQUIRED=true # gate fallback on an active partnership
|
||
- # false = standalone mode, no partnership dependency
|
||
- FALLBACK_PARTNERSHIP_SUSPEND_AFTER=120 # minutes without active partnership before suspending
|
||
- # 0 = suspend immediately when partnership goes inactive
|
||
-
|
||
-# ━━━ Failover Test ━━━
|
||
-# Controlled simulation of a failover event — run manually via fallback_test.sh.
|
||
- FALLBACK_TEST_BLOCK_WAIT=150 # seconds to wait after blocking connectivity
|
||
- FALLBACK_TEST_HANDBACK_WAIT=360 # seconds to wait before initiating handback
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── DOCKER ESSENTIALS ─────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-
|
||
-# ━━━ Downloaders Reset ━━━
|
||
-# Runs every 30 minutes via CRITICAL_MAINTENANCE_SCRIPTS.
|
||
-# Clears stuck states, purges old history, prepares each download client for a clean cycle.
|
||
-# Per-host URLs and API keys live in host*.conf.
|
||
- DOWNLOADER_RETENTION_DAYS=7 # days — purge history older than this
|
||
-
|
||
-# qBittorrent failsafe — removes torrents older than threshold regardless of ratio
|
||
-# deleteFiles=false — removes from qBit but leaves files on disk (arr manages files)
|
||
- QBIT_FAILSAFE_MIN_DAYS=180 # days — minimum age before failsafe deletion
|
||
- QBIT_FAILSAFE_MIN_RATIO=0 # 0 = age only, no ratio requirement
|
||
-
|
||
-# ━━━ Docker Daily Restart ━━━
|
||
-# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
|
||
-# Per-host lists live in host*.conf:
|
||
-# HOST1_DAILY_RESTART_CONTAINERS
|
||
-# HOST2_DAILY_RESTART_CONTAINERS
|
||
-# detect_hosts() sets DAILY_RESTART_CONTAINERS to the correct host array at runtime.
|
||
-
|
||
-# ━━━ Docker Weekly Restart ━━━
|
||
-# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
|
||
-# Containers already stopped for weekly sync — restart adds zero extra downtime.
|
||
-# Per-host lists live in host*.conf:
|
||
-# HOST1_WEEKLY_RESTART_CONTAINERS
|
||
-# HOST2_WEEKLY_RESTART_CONTAINERS
|
||
-
|
||
-# ━━━ Docker Watchdog ━━━
|
||
-# Continuous two-tier self-healing container monitoring.
|
||
-# Started by array_started.sh — runs until array stops.
|
||
-# Re-sources all three conf files each cycle — add/remove containers without restarting watchdog.
|
||
-#
|
||
-# Tier 1 — strict monitoring of explicitly configured containers:
|
||
-# Memory hard limits — immediate restart if exceeded
|
||
-# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT sustained strikes
|
||
-# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT failed checks
|
||
-# Required containers — must always be running, strike + skip list with auto-clear
|
||
-#
|
||
-# Tier 2 — global health scan of ALL running containers:
|
||
-# Unhealthy status — Docker HEALTHCHECK unhealthy → restart
|
||
-# OOM killed — kernel killed → restart + notify
|
||
-# Crash loop detection — RestartCount climbing → notify, critical above limit
|
||
-# Dead containers — remove and restart
|
||
-# Unexpected exits — non-zero exit code → restart
|
||
-#
|
||
-# All per-host container lists live in host*.conf:
|
||
-# HOST*_WATCHDOG_CONTAINERS — memory hard limits per container
|
||
-# HOST*_WATCHDOG_CONTAINER_URLS — HTTP health check URLs
|
||
-# HOST*_WATCHDOG_REQUIRED_CONTAINERS — must always be running
|
||
-# HOST*_WATCHDOG_SCAN_IGNORE — skip in Tier 2 scan
|
||
-# HOST*_WATCHDOG_DEPENDENCIES — dependency ordering for restart decisions
|
||
-
|
||
-# Strike state file — persistent in STATE_DIR
|
||
- WATCHDOG_STATE_FILE="$STATE_DIR/container_watchdog_state.db"
|
||
-
|
||
-# CPU thresholds — normalised against total core count at runtime
|
||
- SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU
|
||
- HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU
|
||
- CPU_FAIL_LIMIT=2 # consecutive hard CPU strikes before container restart
|
||
-
|
||
-# Memory soft threshold — warn when container reaches this % of its hard limit
|
||
- SOFT_MEM_THRESHOLD=80
|
||
-
|
||
-# HTTP responsiveness
|
||
- RESP_FAIL_LIMIT=2 # consecutive failed checks before restart
|
||
- CURL_TIMEOUT=5 # seconds per check before timeout
|
||
-
|
||
-# Tier 2 master toggle
|
||
- WATCHDOG_SCAN_ALL=true # false = only WATCHDOG_CONTAINERS + required containers
|
||
-
|
||
-# Individual Tier 2 check toggles
|
||
- WATCHDOG_RESTART_UNHEALTHY=true
|
||
- WATCHDOG_RESTART_DEAD=true
|
||
- WATCHDOG_RESTART_CRASHED=true
|
||
- WATCHDOG_NOTIFY_OOM=true
|
||
- WATCHDOG_NOTIFY_CRASHLOOP=true
|
||
-
|
||
-# Crash loop threshold
|
||
- WATCHDOG_CRASH_LIMIT=5 # RestartCount above this = critical crash loop
|
||
-
|
||
-# Startup grace period — skip restarts while system is still booting
|
||
- WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures
|
||
-
|
||
-# Restart loop protection — prevents watchdog from endlessly restarting a broken container
|
||
- WATCHDOG_CONTAINER_RESTART_LIMIT=3
|
||
- WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours
|
||
- WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db"
|
||
-
|
||
-# Notification batching — one summary per cycle instead of one ping per event
|
||
- WATCHDOG_BATCH_NOTIFY=true
|
||
-
|
||
-# Appdata size monitoring — two-part catch-all for runaway growth and oversized log files.
|
||
-#
|
||
-# Part 1 — Growth rate (zero-config):
|
||
-# Reads per-container dir totals each cycle via du, compares to previous cycle.
|
||
-# Any container growing more than WATCHDOG_APPDATA_GROWTH_GB triggers a focused *.log scan
|
||
-# inside that container. No per-container config required — new containers are covered
|
||
-# automatically. Baseline built on first run after boot; growth detection starts cycle 2.
|
||
-#
|
||
-# Part 2 — Absolute log size:
|
||
-# Finds *.log / *.log.* files over WATCHDOG_APPDATA_LOG_MAX_GB across all appdata paths.
|
||
-# Catches logs that have already stabilised at a large size and are no longer actively growing.
|
||
-#
|
||
-# Strike system (reuses existing watchdog infrastructure):
|
||
-# Strike 1 — warn + notify: condition first detected
|
||
-# Strike 2 — warn + escalated notify: still present next cycle
|
||
-# Strike 3 (WATCHDOG_APPDATA_STRIKE_LIMIT) — action cycle:
|
||
-# If WATCHDOG_APPDATA_TRUNCATE_LOGS=true: truncate *.log files in-place, clear strikes
|
||
-# If false: critical notify only, strikes held until condition resolves
|
||
-# Condition resolves (growth stops / log drops below threshold) → strikes auto-clear
|
||
-#
|
||
-# HOST*_WATCHDOG_APPDATA_SIZES (in host*.conf) suppresses growth warnings for a
|
||
-# container until its dir exceeds the configured ceiling. Only needed when a container
|
||
-# legitimately has large stable data and you want to guarantee it never triggers a false alarm.
|
||
- WATCHDOG_CHECK_APPDATA=true
|
||
- WATCHDOG_APPDATA_PATHS=("/mnt/docker-unraid/appdata")
|
||
- WATCHDOG_APPDATA_GROWTH_GB=2 # flag containers growing more than this per cycle
|
||
- WATCHDOG_APPDATA_LOG_MAX_GB=2 # flag *.log files exceeding this size (absolute)
|
||
- WATCHDOG_APPDATA_TRUNCATE_LOGS=false # set true to auto-truncate oversized *.log files on action cycle
|
||
- WATCHDOG_APPDATA_STRIKE_LIMIT=3 # cycles before action fires (matches existing watchdog pattern)
|
||
- WATCHDOG_APPDATA_GROWTH_FILE="$STATE_DIR/watchdog_appdata_growth.db"
|
||
- STORAGE_WATCHDOG_STATE_FILE="$STATE_DIR/storage_watchdog_state.db"
|
||
-
|
||
-# ━━━ Docker Network Connect ━━━
|
||
-# Ensures custom networks exist and connects containers at array start.
|
||
-# Runs once via ARRAY_START_SCRIPTS — idempotent, safe to re-run.
|
||
-# Container and network lists are host-specific — defined in host*.conf:
|
||
-# HOST1_NETWORK_CONNECT_CONTAINERS / HOST2_NETWORK_CONNECT_CONTAINERS
|
||
-# HOST1_NETWORK_CONNECT_NETWORKS / HOST2_NETWORK_CONNECT_NETWORKS
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── UNRAID ESSENTIALS ─────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-
|
||
-# ━━━ inotify Tuning ━━━
|
||
-# Linux inotify limits — applied at every array start by inotify_tuning.sh.
|
||
-# Default unRAID values are very low — with many Docker containers watching files
|
||
-# you can silently exhaust the limit causing containers to miss file events.
|
||
-# Symptoms: containers miss file events, downloads not detected, library not updated.
|
||
-# Live TV stutter is a known symptom of inotify exhaustion.
|
||
-# These settings are lost on reboot — reapplied automatically at array start.
|
||
- INOTIFY_MAX_INSTANCES=1024 # default: 128 — max inotify instances per user
|
||
- # 1024 handles ~20-30 containers watching files simultaneously
|
||
- INOTIFY_MAX_WATCHES=1048576 # default: 8192 — SHARED budget across ALL users/containers
|
||
- # 524288 (512K) was previous value — raised to 1M (1048576)
|
||
- # VSCode/Code-Server alone needs ~50K-200K for large workspaces
|
||
- # with node_modules. All arr containers + Emby + VSCode share
|
||
- # this budget. 1M is safe on 128GB RAM (~128MB kernel memory)
|
||
- # If VSCode shows "unable to watch for file changes" → too low
|
||
- INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — max events queued before dropping
|
||
-
|
||
-# ━━━ System Tuning Monitor ━━━
|
||
-# Tracks inotify and php-fpm usage over time.
|
||
-# Snapshot written every 6 hours by system_tuning_monitor.sh.
|
||
-# Read by sunday_morning_coffee_report.sh for weekly peak/avg/warning summary.
|
||
- INOTIFY_WARN_PCT=80 # warn if inotify instances exceed this % of limit
|
||
- PHP_FPM_WARN_PCT=80 # warn if php-fpm workers exceed this % of max_children
|
||
- TUNING_MONITOR_LOG="$DATA_DIR/system_tuning_history.db"
|
||
- TUNING_LOG_RETENTION=30 # days before old entries are purged
|
||
-
|
||
-# ━━━ Reboot ━━━
|
||
-# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots.
|
||
-# Gives users time to save work — 300s = 5 minutes.
|
||
- REBOOT_SLEEP=300
|
||
-
|
||
-# ━━━ Mover ━━━
|
||
-# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process.
|
||
-# Gives mover time to finish current file transfer before being interrupted.
|
||
- MOVER_STOP_TIMEOUT=300
|
||
-
|
||
-# ━━━ Syslog Filter ━━━
|
||
-# Path for the rsyslog filter file that suppresses Docker veth interface noise.
|
||
-# Docker creates a new veth interface for each container — generates hundreds of
|
||
-# log lines per hour that have no diagnostic value. Filter removes them at source.
|
||
- FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"
|
||
-
|
||
-# ━━━ PHP-FPM ━━━
|
||
-# Higher max_children allows more concurrent PHP requests to the unRAID WebGUI.
|
||
-# Default is very low — increasing it prevents WebGUI slowdowns under load.
|
||
-# 250 is safe for servers with 32GB+ RAM.
|
||
- PHP_CONF="/etc/php-fpm.d/www.conf"
|
||
- PHP_MAX_CHILDREN=250
|
||
-
|
||
-# ━━━ Clear Logs ━━━
|
||
-# System log files cleared weekly to prevent rootfs fill over time.
|
||
-# These grow continuously — without clearing they eventually consume all rootfs space.
|
||
-# clear_logs.sh runs in WEEKLY_MAINTENANCE_SCRIPTS — Sunday 2:30am.
|
||
-#
|
||
-# Size threshold approach — only clear if log exceeds threshold.
|
||
-# Avoids destroying useful recent diagnostic context when logs are small.
|
||
-# LOG_MIN_SIZE_MB: skip clearing if log is under this size (not worth clearing)
|
||
-# LOG_DOCKER_MAX_MB: clear a container log only if it exceeds this size
|
||
-# Docker logs grow fastest on active containers (Emby, SABnzbd, Sonarr)
|
||
-# 100MB per container × 30 containers = 3GB before clearing kicks in
|
||
- LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg)
|
||
- LOG_MIN_SIZE_MB=10 # skip system log if under this size (already small)
|
||
- LOG_DOCKER_MAX_MB=100 # clear Docker container log only if over this size (MB)
|
||
-
|
||
-# ━━━ Network Watchdog ━━━
|
||
-# Services-layer connectivity — internet reachability, DDNS sync, Tailscale, NPM proxy.
|
||
-# Host-specific values (domain, container, NPM URL) live in host*.conf.
|
||
- NETWORK_WATCHDOG_ENABLED=true
|
||
- NETWORK_WATCHDOG_INTERNET_URL="https://1.1.1.1"
|
||
- NETWORK_WATCHDOG_INTERNET_TIMEOUT=5
|
||
- NETWORK_WATCHDOG_CHECK_TAILSCALE=true
|
||
- NETWORK_WATCHDOG_NPM_TIMEOUT=10
|
||
- NETWORK_WATCHDOG_NPM_STRIKE_LIMIT=2
|
||
- NETWORK_WATCHDOG_NPM_STATE_FILE="$STATE_DIR/network_watchdog_state.db"
|
||
-
|
||
-# ━━━ WebGUI Watchdog ━━━
|
||
-# Monitors unRAID WebGUI responsiveness — escalates through nginx restart → emhttp restart.
|
||
-# Separate from docker_watchdog — this monitors the unRAID UI itself, not containers.
|
||
- WEBGUI_URL="http://localhost"
|
||
- WEBGUI_TIMEOUT=5 # seconds before curl gives up on WebGUI check
|
||
- WEBGUI_NGINX_WAIT=15 # seconds after nginx restart before rechecking
|
||
- WEBGUI_EMHTTP_WAIT=30 # seconds after emhttp restart before rechecking
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-
|
||
-# ━━━ Media Permissions ━━━
|
||
-# Applied recursively by media_shares_permissions.sh.
|
||
-# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
|
||
-#
|
||
-# Why split directory vs file permissions:
|
||
-# Directories need execute bit to enter — 755 allows owner+group+others to traverse
|
||
-# Files should NOT be executable — 664 allows owner+group read/write, others read
|
||
-# This is POSIX best practice for media servers, not a blanket 777 band-aid
|
||
-#
|
||
-# Why nobody:users ownership:
|
||
-# All ecosystem containers run as PUID=99 (nobody) PGID=100 (users) on unRAID
|
||
-# Files owned by nobody:users are accessible to all containers without world-write
|
||
-# If this script fixes many files each run → a container has wrong PUID/PGID set
|
||
-# Fix: add PUID=99 PGID=100 to the container's environment variables
|
||
-#
|
||
-# This script runs daily as a failsafe — even with correct container config:
|
||
-# rsync may bring files with source ownership if not run with --chown
|
||
-# Manual admin copies create root:root files
|
||
-# New containers may not have PUID/PGID set yet
|
||
- PERMISSIONS_DIR_MODE="755" # directories — traverse + list, no world-write
|
||
- PERMISSIONS_FILE_MODE="664" # files — group read/write, no execute
|
||
- PERMISSIONS_OWNER="nobody:users"
|
||
-
|
||
-# Share list defined per host in host*.conf:
|
||
-# HOST1_MEDIA_PERMISSION_SHARES / HOST2_MEDIA_PERMISSION_SHARES
|
||
-
|
||
-# ━━━ Media Cleaner ━━━
|
||
-# Removes junk files from media shares — two profiles: anime and media.
|
||
-# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
|
||
-# Folder lists defined per host in host*.conf:
|
||
-# HOST1_ANIME_CLEAN_FOLDERS / HOST2_ANIME_CLEAN_FOLDERS
|
||
-# HOST1_MEDIA_CLEAN_FOLDERS / HOST2_MEDIA_CLEAN_FOLDERS
|
||
-
|
||
- ANIME_FILE_PATTERNS=(
|
||
- # ── Checksums ────────────────────────────────────────────────────────────────────────
|
||
- '*.sfv' '*.md5' '*.sha1' # verification files — useless post-download
|
||
-
|
||
- # ── Scene / download metadata ─────────────────────────────────────────────────────────
|
||
- '*.url' '*.lnk' # scene links
|
||
- '*.info' '*.diz' # scene description files
|
||
- '*.nzb' # usenet download files
|
||
- '*.torrent' # torrent files left by download clients
|
||
-
|
||
- # ── Archives and segments ─────────────────────────────────────────────────────────────
|
||
- '*.rar' '*.zip' '*.7z' '*.ace' # archives — source files not needed after extract
|
||
- '*.r00' '*.r01' '*.r02' '*.r03' # multi-part rar segments
|
||
- '*.r04' '*.r05' '*.r06' '*.r07'
|
||
- '*.r08' '*.r09' '*.srr' # scene repair files
|
||
- '*.001' '*.002' '*.003' # split archive parts
|
||
- '*.gz' '*.tar' '*.bz2' # linux archives
|
||
-
|
||
- # ── Scene samples and proofs ──────────────────────────────────────────────────────────
|
||
- '*.sample*' '*.proof*' # scene samples — never needed in library
|
||
-
|
||
- # ── Executables and scripts ───────────────────────────────────────────────────────────
|
||
- '*.exe' '*.scr' '*.com' # Windows executables
|
||
- '*.bat' '*.cmd' '*.vbs' '*.ps1' # Windows scripts
|
||
- '*.msi' '*.dll' '*.sys' # Windows installers and system files
|
||
- '*.sh' # shell scripts in media folders = suspicious
|
||
-
|
||
- # ── Incomplete downloads ──────────────────────────────────────────────────────────────
|
||
- '*.!ut' '*.!qB' # uTorrent / qBittorrent incomplete markers
|
||
- '*.crdownload' '*.opdownload' # Chrome / Opera incomplete downloads
|
||
- '*.part' # partial download files
|
||
-
|
||
- # ── Sync conflicts ────────────────────────────────────────────────────────────────────
|
||
- '*sync-conflict*' # Syncthing conflict copies
|
||
- )
|
||
-
|
||
- MEDIA_FILE_PATTERNS=(
|
||
- # ── Checksums ────────────────────────────────────────────────────────────────────────
|
||
- '*.sfv' '*.md5' '*.sha1' # verification files — useless post-download
|
||
-
|
||
- # ── Scene / download metadata ─────────────────────────────────────────────────────────
|
||
- '*.url' '*.lnk' # scene links
|
||
- '*.info' '*.diz' # scene description files
|
||
- '*.nzb' # usenet download files
|
||
- '*.torrent' # torrent files left by download clients
|
||
-
|
||
- # ── Archives and segments ─────────────────────────────────────────────────────────────
|
||
- '*.rar' '*.zip' '*.7z' '*.ace' # archives — source files not needed after extract
|
||
- '*.r00' '*.r01' '*.r02' '*.r03' # multi-part rar segments
|
||
- '*.r04' '*.r05' '*.r06' '*.r07'
|
||
- '*.r08' '*.r09' '*.srr' # scene repair files
|
||
- '*.001' '*.002' '*.003' # split archive parts
|
||
- '*.gz' '*.tar' '*.bz2' # linux archives
|
||
-
|
||
- # ── Scene samples and proofs ──────────────────────────────────────────────────────────
|
||
- '*.sample*' '*.proof*' # scene samples — never needed in library
|
||
-
|
||
- # ── Executables and scripts ───────────────────────────────────────────────────────────
|
||
- '*.exe' '*.scr' '*.com' # Windows executables
|
||
- '*.bat' '*.cmd' '*.vbs' '*.ps1' # Windows scripts
|
||
- '*.msi' '*.dll' '*.sys' # Windows installers and system files
|
||
- '*.sh' # shell scripts in media folders = suspicious
|
||
-
|
||
- # ── Incomplete downloads ──────────────────────────────────────────────────────────────
|
||
- '*.!ut' '*.!qB' # uTorrent / qBittorrent incomplete markers
|
||
- '*.crdownload' '*.opdownload' # Chrome / Opera incomplete downloads
|
||
- '*.part' # partial download files
|
||
-
|
||
- # ── Sync conflicts ────────────────────────────────────────────────────────────────────
|
||
- '*sync-conflict*' # Syncthing conflict copies
|
||
-
|
||
- # ── Media-specific extras ─────────────────────────────────────────────────────────────
|
||
- '*.iso' # disc images — extracted content is in library
|
||
- '*.lrc' # lyrics files — handled by music apps not Emby
|
||
- )
|
||
-
|
||
-# ━━━ Arr Cleanup ━━━
|
||
-# Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs.
|
||
-# Per-host URLs, API keys, and path maps live in host*.conf.
|
||
-# detect_hosts() selects correct host vars at runtime.
|
||
-#
|
||
-# API versions — update MAJOR version here when script is updated for a new arr version:
|
||
-# Sonarr v4 → /api/v3/series → /api/v3/episodefile?seriesId=X
|
||
-# Radarr v6 → /api/v3/movie → /api/v3/moviefile?movieId=X
|
||
-# Lidarr v3 → /api/v1/artist → /api/v1/trackFile?artistId=X
|
||
- SONARR_VERSION_MAJOR=4
|
||
- RADARR_VERSION_MAJOR=6
|
||
- LIDARR_VERSION_MAJOR=3
|
||
-
|
||
-# Lidarr shared settings
|
||
- LIDARR_LOCK_WARN_AGE=3600 # 1hr — large libraries take time, not stuck
|
||
- LIDARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion
|
||
- LIDARR_MAX_DELETE_GB=5 # require --i-know-what-im-doing if deletion exceeds this
|
||
- LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||
- # protects against API returning partial data on a bad day
|
||
- LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
|
||
- LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
|
||
- LIDARR_PROTECTED_PATTERNS=(
|
||
- # Metadata
|
||
- "*.nfo" "*.tbn"
|
||
- # Images — album art, artist images, Emby artwork
|
||
- "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
|
||
- "poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
|
||
- "banner.*" "thumb.*" "landscape.*"
|
||
- "folder.*" "cover.*" "album.*" "artist.*" "disc.*"
|
||
- # Lyrics
|
||
- "*.lrc"
|
||
- )
|
||
-
|
||
-# Lidarr artwork fetcher settings
|
||
- LIDARR_ART_MIN_SIZE=10000 # bytes — reject downloads smaller than this
|
||
- LIDARR_ART_MAX_PARALLEL=4 # concurrent background download jobs
|
||
- LIDARR_ART_RETRIES=2 # download retry attempts per image
|
||
- LIDARR_ART_SLEEP_BETWEEN=0.2 # seconds between fanart.tv API calls
|
||
-# HOST*_FANART_API_KEY / HOST*_LASTFM_API_KEY — set in host*.conf
|
||
-
|
||
-# Lidarr discovery settings (playback_aware_lidarr_discovery.sh)
|
||
- LIDARR_DISCOVERY_THRESHOLD=70 # score to accept candidate (0-100)
|
||
- LIDARR_DISCOVERY_LOOKBACK_DAYS=7 # Emby play history window in days
|
||
- LIDARR_DISCOVERY_MIN_PLAYS=3 # min plays in window before evaluating an artist
|
||
- LIDARR_DISCOVERY_USER_CAP_PCT=35 # max % any single user can contribute to play score (prevents one listener dominating)
|
||
- LIDARR_DISCOVERY_MAX_ADDS=5 # max artists to add per run — quality over bulk
|
||
- LIDARR_DISCOVERY_REJECT_COOLDOWN=30 # days before re-evaluating a rejected artist
|
||
- LIDARR_DISCOVERY_HISTORY="$DATA_DIR/lidarr_discovery_history.db"
|
||
-
|
||
-# Sonarr discovery settings (playback_aware_sonarr_discovery.sh)
|
||
- SONARR_DISCOVERY_THRESHOLD=52 # score to accept candidate (0-100)
|
||
- SONARR_DISCOVERY_LOOKBACK_DAYS=14 # Emby episode play history window in days (shorter than movies — TV watched more frequently)
|
||
- SONARR_DISCOVERY_MAX_SEEDS=5 # max seed series from Stage 1
|
||
- SONARR_DISCOVERY_MAX_ADDS=3 # max shows to add per run — TV is a larger commitment than movies
|
||
- SONARR_DISCOVERY_MIN_VOTE_COUNT=50 # min TMDB votes (TV has fewer votes than movies at same popularity)
|
||
- SONARR_DISCOVERY_MIN_RATING=65 # min TMDB vote_average × 10 (65 = 6.5/10)
|
||
- SONARR_DISCOVERY_REJECT_COOLDOWN=60 # days before re-evaluating a rejected show
|
||
- SONARR_DISCOVERY_USER_EPISODE_CAP=8 # max episodes any one user contributes to seed volume score
|
||
- SONARR_DISCOVERY_MONITOR_MODE="all" # Sonarr monitor mode on add: all | future | first | latest | none
|
||
- SONARR_DISCOVERY_HISTORY="$DATA_DIR/sonarr_discovery_history.db"
|
||
-
|
||
-# Radarr discovery shared settings
|
||
- RADARR_DISCOVERY_THRESHOLD=52 # score to accept candidate (0-100) — lower than Lidarr since diverse seeds rarely overlap
|
||
- RADARR_DISCOVERY_LOOKBACK_DAYS=30 # Emby watch history window in days (movies rewatched less often)
|
||
- RADARR_DISCOVERY_MAX_SEEDS=5 # max seed movies from Stage 1
|
||
- RADARR_DISCOVERY_MAX_ADDS=5 # max movies to add per run
|
||
- RADARR_DISCOVERY_MIN_VOTE_COUNT=100 # min TMDB votes to be considered a candidate
|
||
- RADARR_DISCOVERY_MIN_RATING=60 # min TMDB vote_average × 10 (60 = 6.0/10)
|
||
- RADARR_DISCOVERY_REJECT_COOLDOWN=60 # days before re-evaluating a rejected movie
|
||
- RADARR_DISCOVERY_SEED_LIBRARIES=("Movies") # Emby libraries to draw seed movies from
|
||
- RADARR_DISCOVERY_HISTORY="$DATA_DIR/radarr_discovery_history.db"
|
||
-
|
||
-# Emby → arr sync library allowlists
|
||
-# Only these Emby library names will be considered by the sync tools.
|
||
-# Names must match exactly as shown in Emby > Dashboard > Libraries.
|
||
- SONARR_EMBY_LIBRARIES=("Anime" "Kids Shows" "Stand-Up Comedy" "TV shows")
|
||
- RADARR_EMBY_LIBRARIES=("Movies" "Kid's Movies")
|
||
-
|
||
-# Sonarr shared settings
|
||
- SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion
|
||
- SONARR_MAX_DELETE_GB=10 # require --i-know-what-im-doing if deletion exceeds this
|
||
- SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
|
||
- SONARR_PROTECTED_PATTERNS=(
|
||
- # Subtitles
|
||
- "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt"
|
||
- # Metadata
|
||
- "*.nfo" "*.tbn"
|
||
- # Images
|
||
- "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
|
||
- "poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
|
||
- "banner.*" "thumb.*" "landscape.*"
|
||
- # Kodi/Emby extras — not tracked by Sonarr API
|
||
- "*-trailer.*" "*-featurette.*" "*-behindthescenes.*"
|
||
- "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*"
|
||
- "*-clip.*" "*-other.*"
|
||
- # Theme songs — stored in show folder, not tracked by arr
|
||
- "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka"
|
||
- )
|
||
-
|
||
-# Radarr shared settings
|
||
- RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion
|
||
- RADARR_MAX_DELETE_GB=15 # require --i-know-what-im-doing if deletion exceeds this
|
||
- RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov")
|
||
- RADARR_PROTECTED_PATTERNS=(
|
||
- # Subtitles
|
||
- "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt"
|
||
- # Metadata
|
||
- "*.nfo" "*.tbn"
|
||
- # Images
|
||
- "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
|
||
- "poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
|
||
- "banner.*" "thumb.*" "landscape.*"
|
||
- # Kodi/Emby extras — not tracked by Radarr API
|
||
- "*-trailer.*" "*-featurette.*" "*-behindthescenes.*"
|
||
- "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*"
|
||
- "*-clip.*" "*-other.*"
|
||
- # Theme songs — stored in movie folder, not tracked by arr
|
||
- "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka"
|
||
- )
|
||
-
|
||
-# Radarr/Sonarr TMDb/TVDB removed entry cleanup
|
||
- RADARR_DROPPED_ADD_EXCLUSION=true # add removed movies to import exclusion list
|
||
- SONARR_DROPPED_ADD_EXCLUSION=true # add removed series to import exclusion list
|
||
-
|
||
-# ━━━ Arr Failed/Stalled Recovery ━━━
|
||
-# Auto blocklist + re-search failed imports and stalled downloads.
|
||
-# Runs every 6 hours — schedule: 0 */6 * * *
|
||
-#
|
||
-# Targets four problem types:
|
||
-# importFailed — downloaded but arr couldn't import
|
||
-# importPending — downloaded, stuck waiting to import (won't self-resolve)
|
||
-# error status — serious failure not covered above
|
||
-# stalled — download stuck with no connections or progress
|
||
-#
|
||
-# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first.
|
||
-# Per-host recovery toggles (HOST1_SONARR_RECOVERY etc.) live in host*.conf.
|
||
- ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this
|
||
- # matches cron interval — items eligible after one missed cycle
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-# Session-based storage allocator using filesystem symlink indirection.
|
||
-# ffmpeg resolves the symlink ONCE at session start — existing sessions never affected.
|
||
-# Symlink flips between ramdisk and SSD are transparent to active streams.
|
||
-#
|
||
-# ⚠️ Docker mount — must use shared propagation so symlink flips work inside container:
|
||
-# --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared
|
||
-# Standard path mappings use rprivate — symlink changes are NOT visible inside container.
|
||
-
|
||
-# ━━━ Transcode Manager ━━━
|
||
-# tmpfs mount point — created at array start by ramdisk_setup.sh.
|
||
-# Must exist before Emby starts so the symlink resolves correctly.
|
||
- RAMDISK_PATH="/mnt/ramdisk_transcodes"
|
||
-
|
||
-# Ramdisk size and flip thresholds — defined per host in host*.conf.
|
||
-# All three are coupled — if size changes, thresholds must change with it.
|
||
-# HOST1_RAMDISK_SIZE / HOST2_RAMDISK_SIZE
|
||
-# HOST1_RAMDISK_WARN_GB / HOST2_RAMDISK_WARN_GB ← flip to SSD at this usage
|
||
-# HOST1_RAMDISK_LOW_GB / HOST2_RAMDISK_LOW_GB ← flip back to ramdisk at this usage
|
||
-
|
||
-# Symlink that Emby points at — this path NEVER changes regardless of ramdisk/SSD state.
|
||
-# Emby resolves the symlink once per session at start — symlink flips are transparent.
|
||
-# Must match the container path configured in Emby's Extra Parameters.
|
||
- TRANSCODE_LINK="/mnt/ram-transcode"
|
||
-
|
||
-# SSD fallback location — defined per host in host*.conf (cache path differs per server):
|
||
-# HOST1_TRANSCODE_SSD / HOST2_TRANSCODE_SSD
|
||
-
|
||
-# Minimum free GB on SSD before allowing flip from ramdisk to SSD.
|
||
- RAMDISK_SSD_MIN_GB=20
|
||
-
|
||
- TRANSCODE_MAX_AGE=20 # minutes — HLS segment age before cleanup eligibility
|
||
- TRANSCODE_ORPHAN_AGE=30 # minutes — files with no matching active session
|
||
- TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in one hour
|
||
-
|
||
- TRANSCODE_OWNER="nobody:users"
|
||
- TRANSCODE_CHMOD="755"
|
||
-
|
||
-# Operating mode — controls symlink direction behaviour.
|
||
-# smart — auto-flips between ramdisk and SSD based on thresholds (default)
|
||
-# ramdisk — always uses ramdisk, warns if RAMDISK_WARN_GB exceeded but holds
|
||
-# ssd — always uses SSD, never flips to ramdisk
|
||
- TRANSCODE_MANAGER_MODE="smart"
|
||
-
|
||
-# Daily statistics log — read by weekly_health_digest.sh for transcode summary.
|
||
- TRANSCODE_STATE_FILE="$STATE_DIR/transcode_state.db"
|
||
- TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db"
|
||
- TRANSCODE_LOG_RETENTION=90 # days before old entries purged
|
||
-
|
||
- TRANSCODE_CHECK_EMBY=true
|
||
-
|
||
-# ━━━ Transcode Server Array ━━━
|
||
-# All media servers sharing the ramdisk transcode space.
|
||
-# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
|
||
-# Entries with placeholder API keys are skipped automatically.
|
||
-# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
|
||
-# Defined per host in host*.conf — Emby container names and keys differ per server:
|
||
-# HOST1_TRANSCODE_SERVERS / HOST2_TRANSCODE_SERVERS
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-
|
||
-# ━━━ Certificate Monitor ━━━
|
||
-# Checks SSL certificate expiry via direct openssl connection — no NPM dependency.
|
||
-# Checks the actual certificate served by each domain, not what NPM thinks it has.
|
||
-# Domains defined per host in host*.conf — each server monitors its own domains:
|
||
-# HOST1_CERT_MONITOR_DOMAINS / HOST2_CERT_MONITOR_DOMAINS
|
||
- CERT_WARN_DAYS=30 # warn when cert expires within this many days
|
||
- CERT_CRIT_DAYS=7 # critical alert within this many days
|
||
- CERT_TIMEOUT=10 # seconds per domain before giving up
|
||
-
|
||
-# ━━━ Backup Verify ━━━
|
||
-# Verifies rsync mirror health by comparing random file checksums between servers.
|
||
-# Catches silent corruption or incomplete syncs that rsync itself wouldn't detect.
|
||
-# Defined per host in host*.conf — leave empty to use HOST*_DAILY_SYNC_SHARES automatically:
|
||
-# HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES
|
||
- BACKUP_VERIFY_SAMPLE=10 # random files to check per share
|
||
- BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample
|
||
-
|
||
-# ━━━ SMART Health ━━━
|
||
-# Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*.
|
||
-# Thresholds read from /boot/config/plugins/dynamix/dynamix.cfg at runtime
|
||
-# (hot/max/hotssd/maxssd) — these vars are fallback only if dynamix.cfg not found.
|
||
- SMART_TEMP_WARN=45 # fallback — Celsius warn threshold
|
||
- SMART_TEMP_CRIT=55 # fallback — Celsius critical threshold
|
||
-# Drives to ignore defined per host in host*.conf — hardware is server-specific:
|
||
-# HOST1_SMART_IGNORE_DRIVES / HOST2_SMART_IGNORE_DRIVES
|
||
-
|
||
-# ━━━ ZFS Memory Snapshot ━━━
|
||
-# Weekly ZFS pool health and memory diagnostic report — informational only.
|
||
- ZFS_REPORT_LOG="$DATA_DIR/zfs-weekly-health.log"
|
||
- ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC using more than this % of its max
|
||
- ZFS_REPORT_FREE_WARN_GB=10 # warn if less than this GB free RAM
|
||
- ZFS_REPORT_AVAIL_WARN_GB=20 # warn if less than this GB available on ZFS pool
|
||
- ZFS_REPORT_DOCKER_TOP=10 # how many top Docker containers to show by memory
|
||
-# Pool ignore list defined per host in host*.conf — pool names are server-specific:
|
||
-# HOST1_ZFS_REPORT_IGNORE_POOLS / HOST2_ZFS_REPORT_IGNORE_POOLS
|
||
-
|
||
-# ━━━ Bandwidth Monitor ━━━
|
||
-# Called automatically by rsync.sh after each sync — one bounded write per run.
|
||
-# Tracks transfer size, duration and profile per sync for weekly summary reporting.
|
||
- BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db"
|
||
- BANDWIDTH_LOG_RETENTION=90 # days before old entries purged
|
||
- BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report
|
||
-
|
||
-# Stats files — written by cleanup and recovery scripts, read by coffee report.
|
||
-# All in DATA_DIR — array always running when these are written.
|
||
- ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats
|
||
- ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats
|
||
-
|
||
-# ━━━ Health Digest ━━━
|
||
-# Aggregated system health summary — reads existing state files, no new writes.
|
||
-# Three profiles control when the digest is sent:
|
||
-# always — sends every run regardless of findings
|
||
-# smart — sends only when DIGEST_SMART_ON_* conditions are found
|
||
-# weekly — sends once per week on DIGEST_DAY only
|
||
- DIGEST_PROFILE="weekly" # always | smart | weekly
|
||
- DIGEST_DAY="Sunday"
|
||
- DIGEST_SMART_ON_WATCHDOG=true # send if any watchdog strikes are active
|
||
- DIGEST_SMART_ON_FALLBACK=true # send if fallback state is not NORMAL
|
||
- DIGEST_SMART_ON_CERT_WARN=true # send if any cert is under CERT_WARN_DAYS
|
||
- DIGEST_SMART_ON_BANDWIDTH=true # send if any transfer exceeded BANDWIDTH_WARN_GB
|
||
-
|
||
-# ━━━ Emby Session Report ━━━
|
||
-# Weekly Emby usage statistics via API — no persistent writes, queries fresh each run.
|
||
-# URL and API key pulled from HOST*_EMBY_URL and HOST*_EMBY_API_KEY in host*.conf.
|
||
- EMBY_REPORT_DAYS=7 # days to include in the report period
|
||
- EMBY_REPORT_TOP_N=10 # number of top content items to show
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-# Pressure reduction layer — keeps the system comfortable before things break.
|
||
-# Called by watchdog_orchestrator.sh. Single-pass, not a continuous loop.
|
||
-#
|
||
-# ── PRESSURE LEVELS ───────────────────────────────────────────────────────────────────────────
|
||
-# Level 1 (soft) — throttle SABnzbd + qBit download speeds
|
||
-# Level 2 (medium) — further throttle + docker pause background containers
|
||
-# Level 3 (hard) — docker stop optional containers, signal docker_watchdog to defer
|
||
-#
|
||
-# ── PER-HOST CONTAINER LISTS ──────────────────────────────────────────────────────────────────
|
||
-# HOST*_RW_PAUSE_CONTAINERS — docker pause at medium pressure (in host*.conf)
|
||
-# HOST*_RW_STOP_CONTAINERS — docker stop at hard pressure (in host*.conf)
|
||
-
|
||
- RW_ENABLED=true
|
||
- RW_STATE_FILE="$STATE_DIR/resource_watchdog_state.db"
|
||
-
|
||
-# ━━━ Pressure Thresholds ━━━
|
||
-# Graduated RAM response — resource_watchdog acts before system_watchdog reboots.
|
||
-# RW_RAM_SOFT_GB > RW_RAM_MEDIUM_GB > RW_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always
|
||
- RW_RAM_SOFT_GB=12 # throttle start — reduce background load
|
||
- RW_RAM_MEDIUM_GB=8 # pause background containers
|
||
- RW_RAM_HARD_GB=6 # stop optional containers (was SYS_WATCHDOG_MEM_SHUTDOWN_GB)
|
||
- RW_RAM_RECOVER_GB=20 # RAM must reach this before restoring hard-stopped containers
|
||
-
|
||
-# Load average thresholds — multiplier × core count
|
||
- RW_LOAD_SOFT_MULTIPLIER=2.0 # soft pressure: 2× cores sustained
|
||
- RW_LOAD_MEDIUM_MULTIPLIER=3.0 # medium pressure: 3× cores sustained
|
||
-
|
||
-# Consecutive runs at lower pressure before de-escalating
|
||
- RW_RECOVER_CYCLES=3
|
||
-
|
||
-# ━━━ SABnzbd Throttle ━━━
|
||
-# Speed values: "50M" = 50 MB/s, "0" = unlimited
|
||
- RW_SABNZBD_ENABLED=true
|
||
- RW_SABNZBD_SPEED_SOFT="50M"
|
||
- RW_SABNZBD_SPEED_MEDIUM="10M"
|
||
-
|
||
-# ━━━ qBittorrent Throttle ━━━
|
||
-# KB/s — 0 = unlimited
|
||
- RW_QBIT_ENABLED=true
|
||
- RW_QBIT_DL_SOFT=51200 # 50 MB/s
|
||
- RW_QBIT_DL_MEDIUM=10240 # 10 MB/s
|
||
-
|
||
-# ━━━ Critical Containers ━━━
|
||
-# Never paused or stopped regardless of pressure level.
|
||
-# Keep DNS, auth, media serving, and live TV always running.
|
||
- RW_CRITICAL_CONTAINERS=(
|
||
- "NginxProxyManager" # reverse proxy — internet access
|
||
- "Authelia" # auth — nothing accessible without it
|
||
- "Authelia-Secondary"
|
||
- "Mariadb-Authelia" # Authelia dependency
|
||
- "Mariadb-Authelia-Secondary"
|
||
- "Redis-Authelia" # Authelia dependency
|
||
- "Redis-Authelia-Secondary"
|
||
- "AdGuard-Home" # DNS — all LAN resolution
|
||
- "Emby" # media server — Live TV buffering
|
||
- "Dispatcharr" # Live TV scheduler — loses state if stopped
|
||
- "Dispatcharr-Basic"
|
||
- "Dispatcharr-Iptv-Users"
|
||
- )
|
||
-
|
||
-# ==============================================================================================
|
||
-# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
-# Single-pass system health check — last line of defense before a crash.
|
||
-# Called by watchdog_orchestrator.sh every minute. NOT started by array_started.sh.
|
||
-# Re-sources config at each orchestrator run — config changes take effect immediately.
|
||
-#
|
||
-# ── THREE-TIER RESPONSE SYSTEM ────────────────────────────────────────────────────────────────
|
||
-# CRITICAL — bypass ALL strikes, reboot immediately
|
||
-# Docker daemon down, rootfs 100%, kernel oops, FD exhaustion, /boot read-only
|
||
-#
|
||
-# URGENT — bypass strikes only when OOM confirms active crisis
|
||
-# RAM < MEM_GB AND OOM kills >= OOM_LIMIT in this cycle → reboot NOW
|
||
-# Without OOM confirmation → normal strike system
|
||
-#
|
||
-# STANDARD — strike system (N consecutive failures → reboot)
|
||
-# RAM tiers, high load, CPU temp, zombies, /var/log, /tmp, containers
|
||
-#
|
||
-# ── RAM ───────────────────────────────────────────────────────────────────────────────────────
|
||
-# SYS_WATCHDOG_MEM_GB — strike system → reboot (or OOM bypass)
|
||
-# Warn/shutdown/recover RAM tiers are handled by resource_watchdog.sh
|
||
-
|
||
-# ━━━ State Files ━━━
|
||
- SYS_WATCHDOG_STATE_FILE="$STATE_DIR/system_watchdog_state.db"
|
||
- DOCKER_WATCHDOG_FAILED_FILE="$STATE_DIR/docker_watchdog_failed.db"
|
||
- SYS_WATCHDOG_REBOOT_LOG="$STATE_DIR/system_watchdog_reboots.db"
|
||
- SYS_WATCHDOG_OOM_FILE="$STATE_DIR/system_watchdog_oom.db"
|
||
-
|
||
-# ━━━ Strike and Reboot Loop Settings ━━━
|
||
-# Strike system: a check must fail this many consecutive cycles before action is taken.
|
||
-# Single spikes (one bad reading) are ignored — sustained problems trigger reboot.
|
||
- SYS_WATCHDOG_STRIKE_LIMIT=2 # consecutive failures before reboot trigger
|
||
-
|
||
-# Reboot loop protection — if system keeps rebooting something is seriously wrong.
|
||
-# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS → shutdown instead of reboot.
|
||
- SYS_WATCHDOG_REBOOT_LIMIT=3
|
||
- SYS_WATCHDOG_REBOOT_WINDOW_HRS=12
|
||
-
|
||
-# ━━━ RAM Reboot Threshold ━━━
|
||
-# Reboot trigger only — warn/shutdown/recover handled by resource_watchdog.sh
|
||
-# RW_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always (RM acts before watchdog reboots)
|
||
- SYS_WATCHDOG_MEM_GB=4 # strike system → reboot
|
||
-
|
||
-# ━━━ OOM Bypass Settings ━━━
|
||
-# OOM bypass: if RAM is critically low AND kernel OOM kills exceed this threshold
|
||
-# in a single cycle → bypass strike system and reboot immediately.
|
||
-# Rate-based: kills per 5-minute cycle, not absolute count.
|
||
-# Rationale: 1-2 kills = docker_watchdog handles it ✅
|
||
-# 3+ kills while RAM critical = system dying faster than watchdogs can heal ✅
|
||
- SYS_WATCHDOG_OOM_LIMIT=3 # OOM kills in one cycle to trigger bypass
|
||
-
|
||
-# ━━━ Thresholds ━━━
|
||
-# Set at "about to become unstable" — not "things are a bit high".
|
||
-
|
||
-# rootfs (/) usage — two levels: strike at 95%, critical bypass at 99%
|
||
- SYS_WATCHDOG_ROOTFS_PCT=95 # standard strike threshold
|
||
- SYS_WATCHDOG_ROOTFS_CRITICAL_PCT=99 # bypass strikes — truly full, writes failing
|
||
-
|
||
-# /var/log usage — log spam indicates something broken
|
||
- SYS_WATCHDOG_LOG_PCT=95
|
||
-
|
||
-# /tmp usage — tmpfs fills from downloads, lock files fail at 100%
|
||
- SYS_WATCHDOG_TMP_PCT=90 # warn + attempt clear
|
||
- SYS_WATCHDOG_TMP_CRITICAL_PCT=98 # bypass strikes if clear failed
|
||
-
|
||
-# ZFS ARC pinned percentage — ARC not releasing after reclaim = memory stuck
|
||
- SYS_WATCHDOG_ARC_PINNED_PCT=98
|
||
- SYS_WATCHDOG_ARC_RELEASE_PCT=95
|
||
-
|
||
-# Load average multiplier — threshold = MULTIPLIER × CPU core count
|
||
-# MULTIPLIER=3 on 16-core = load of 48 before triggering
|
||
- SYS_WATCHDOG_LOAD_MULTIPLIER=3
|
||
-
|
||
-# Zombie process count — large numbers = serious process management failure
|
||
- SYS_WATCHDOG_ZOMBIE_LIMIT=50
|
||
-
|
||
-# CPU temperature — sustained high temp causes throttling or kernel panic
|
||
- SYS_WATCHDOG_CPU_TEMP_MAX=95
|
||
-
|
||
-# File descriptor exhaustion — system-wide FD limit near exhaustion
|
||
-# New connections fail silently, Docker can't spawn processes, SSH fails
|
||
- SYS_WATCHDOG_FD_CRITICAL_PCT=95 # bypass strikes — critical tier
|
||
-
|
||
-# Runaway process — single non-container process consuming excessive CPU
|
||
-# Multiple strikes before action — single spikes are normal
|
||
- SYS_WATCHDOG_RUNAWAY_CPU_PCT=90 # % single process must sustain
|
||
- SYS_WATCHDOG_RUNAWAY_STRIKES=3 # consecutive cycles before warning
|
||
-
|
||
-# Array disk errors — accumulating mdstat errors = disk failing NOW
|
||
- SYS_WATCHDOG_MDSTAT_ERROR_LIMIT=5 # new errors in one cycle before acting
|
||
-
|
||
-# ━━━ Check Toggles ━━━
|
||
-# Per-host — moved to host*.conf
|
||
-# Different servers may have different hardware, NICs, and check requirements
|
||
-# See HOST*_SYS_WATCHDOG_CHECK_* in host*.conf
|
||
-
|
||
-# ━━━ Abort Toggles ━━━
|
||
-# Conditions that prevent reboot even when a threshold is hit.
|
||
-# CRITICAL tier bypasses these — a truly critical condition reboots regardless.
|
||
-# true = abort standard reboot if this condition is active
|
||
-# false = reboot anyway
|
||
- SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # rebooting with bad pool risks data loss
|
||
- SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity better than crashing mid-check
|
||
- SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting move better than crashing mid-move
|
||
-
|
||
-# ==============================================================================================
|
||
-# ──────────────────────── End Of User Variables ───────────────────────────────────────────────
|
||
-# ==============================================================================================
|
||
\ No newline at end of file
|
||
diff --git a/Deployment/conf_upgrade.sh b/Deployment/conf_upgrade.sh
|
||
index a7d6a6b..2a5af14 100755
|
||
--- a/Deployment/conf_upgrade.sh
|
||
+++ b/Deployment/conf_upgrade.sh
|
||
@@ -53,13 +53,13 @@
|
||
# RUNTIME MODES
|
||
# ==============================================================================================
|
||
#
|
||
-# conf_upgrade.sh --template master.conf.template --target Configurations/master.conf --dry-run
|
||
+# conf_upgrade.sh --template Configurations/master.conf.template --target Configurations/master.conf --dry-run
|
||
# Preview what would be added, removed, and kept — no changes written.
|
||
#
|
||
-# conf_upgrade.sh --template master.conf.template --target Configurations/master.conf --backup
|
||
+# conf_upgrade.sh --template Configurations/master.conf.template --target Configurations/master.conf --backup
|
||
# Apply the upgrade, writing a .bak first.
|
||
#
|
||
-# conf_upgrade.sh --template master.conf.template --target Configurations/master.conf
|
||
+# conf_upgrade.sh --template Configurations/master.conf.template --target Configurations/master.conf
|
||
# Apply the upgrade in-place with no backup.
|
||
#
|
||
# ==============================================================================================
|
||
diff --git a/Fallback/fallback.sh b/Fallback/fallback.sh
|
||
index 982ce6d..671d672 100755
|
||
--- a/Fallback/fallback.sh
|
||
+++ b/Fallback/fallback.sh
|
||
@@ -321,7 +321,9 @@ state_get() {
|
||
state_set() {
|
||
local key="$1" value="$2"
|
||
if grep -q "^${key}=" "$FALLBACK_STATE_FILE" 2>/dev/null; then
|
||
- sed -i "s|^${key}=.*|${key}=${value}|" "$FALLBACK_STATE_FILE"
|
||
+ # Escape sed replacement metacharacters: \ first, then & and |
|
||
+ local safe="${value//\\/\\\\}"; safe="${safe//&/\\&}"; safe="${safe//|/\\|}"
|
||
+ sed -i "s|^${key}=.*|${key}=${safe}|" "$FALLBACK_STATE_FILE"
|
||
else
|
||
echo "${key}=${value}" >> "$FALLBACK_STATE_FILE"
|
||
fi
|
||
diff --git a/Media/radarr_cleanup.sh b/Media/radarr_cleanup.sh
|
||
index 6e030c1..e3b8c12 100755
|
||
--- a/Media/radarr_cleanup.sh
|
||
+++ b/Media/radarr_cleanup.sh
|
||
@@ -305,24 +305,25 @@ format_bytes() {
|
||
echo ""
|
||
echo "━━━ $ICON_SYNC Pre-flight: Radarr Import Scan ━━━"
|
||
|
||
-# Reverse-lookup container path from path map so Radarr gets its own path, not the host path
|
||
-RADARR_CONTAINER_ROOT=""
|
||
-for _cp in "${!ARR_PATH_MAP[@]}"; do
|
||
- if [[ "${ARR_PATH_MAP[$_cp]}" == "$RADARR_MOVIES_ROOT" ]]; then
|
||
- RADARR_CONTAINER_ROOT="$_cp"
|
||
- break
|
||
- fi
|
||
-done
|
||
-unset _cp
|
||
+# Fetch root folders from Radarr API and translate container paths to host paths
|
||
+mapfile -t SCAN_ROOTS < <(
|
||
+ radarr_api "rootfolder" | \
|
||
+ jq -r '.[].path' 2>/dev/null | \
|
||
+ while IFS= read -r cp; do translate_path "$cp"; done
|
||
+)
|
||
|
||
-if [[ -n "$RADARR_CONTAINER_ROOT" ]]; then
|
||
- info "Triggering DownloadedMoviesScan on: $RADARR_CONTAINER_ROOT"
|
||
- SCAN_PAYLOAD="{\"name\": \"DownloadedMoviesScan\", \"path\": \"$RADARR_CONTAINER_ROOT\"}"
|
||
-else
|
||
- info "No path map match — triggering DownloadedMoviesScan (all root folders)"
|
||
- SCAN_PAYLOAD='{"name": "DownloadedMoviesScan"}'
|
||
+if [[ "${#SCAN_ROOTS[@]}" -eq 0 ]]; then
|
||
+ error "No root folders returned from Radarr API — aborting"
|
||
+ notify "Radarr cleanup aborted on $(hostname) — no root folders from API" \
|
||
+ "Radarr Cleanup" "warning"
|
||
+ exit 1
|
||
fi
|
||
|
||
+info "Scan targets (${#SCAN_ROOTS[@]}): ${SCAN_ROOTS[*]}"
|
||
+
|
||
+info "Triggering DownloadedMoviesScan (all root folders)"
|
||
+SCAN_PAYLOAD='{"name": "DownloadedMoviesScan"}'
|
||
+
|
||
SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \
|
||
-H "X-Api-Key: $RADARR_API_KEY" \
|
||
-H "Content-Type: application/json" \
|
||
@@ -487,7 +488,7 @@ while IFS= read -r filepath; do
|
||
fi
|
||
|
||
done < <(
|
||
- for host_path in "${ARR_PATH_MAP[@]}" "$RADARR_MOVIES_ROOT"; do
|
||
+ for host_path in "${SCAN_ROOTS[@]}"; do
|
||
[[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null
|
||
done | sort -u
|
||
)
|
||
@@ -532,13 +533,13 @@ if [[ "$DRY_RUN" == false ]]; then
|
||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||
|
||
done < <(
|
||
- for host_path in "${ARR_PATH_MAP[@]}" "$RADARR_MOVIES_ROOT"; do
|
||
+ for host_path in "${SCAN_ROOTS[@]}"; do
|
||
[[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null
|
||
done | sort -u
|
||
)
|
||
|
||
info "Cleaning up empty folders..."
|
||
- for host_path in "${ARR_PATH_MAP[@]}" "$RADARR_MOVIES_ROOT"; do
|
||
+ for host_path in "${SCAN_ROOTS[@]}"; do
|
||
[[ -d "$host_path" ]] && \
|
||
find "$host_path" -mindepth 1 -type d -empty -delete 2>/dev/null
|
||
done
|
||
diff --git a/Media/sonarr_cleanup.sh b/Media/sonarr_cleanup.sh
|
||
index cc61da4..507a57c 100755
|
||
--- a/Media/sonarr_cleanup.sh
|
||
+++ b/Media/sonarr_cleanup.sh
|
||
@@ -305,24 +305,25 @@ format_bytes() {
|
||
echo ""
|
||
echo "━━━ $ICON_SYNC Pre-flight: Sonarr Import Scan ━━━"
|
||
|
||
-# Reverse-lookup container path from path map so Sonarr gets its own path, not the host path
|
||
-SONARR_CONTAINER_ROOT=""
|
||
-for _cp in "${!ARR_PATH_MAP[@]}"; do
|
||
- if [[ "${ARR_PATH_MAP[$_cp]}" == "$SONARR_TV_ROOT" ]]; then
|
||
- SONARR_CONTAINER_ROOT="$_cp"
|
||
- break
|
||
- fi
|
||
-done
|
||
-unset _cp
|
||
+# Fetch root folders from Sonarr API and translate container paths to host paths
|
||
+mapfile -t SCAN_ROOTS < <(
|
||
+ sonarr_api "rootfolder" | \
|
||
+ jq -r '.[].path' 2>/dev/null | \
|
||
+ while IFS= read -r cp; do translate_path "$cp"; done
|
||
+)
|
||
|
||
-if [[ -n "$SONARR_CONTAINER_ROOT" ]]; then
|
||
- info "Triggering DownloadedEpisodesScan on: $SONARR_CONTAINER_ROOT"
|
||
- SCAN_PAYLOAD="{\"name\": \"DownloadedEpisodesScan\", \"path\": \"$SONARR_CONTAINER_ROOT\"}"
|
||
-else
|
||
- info "No path map match — triggering DownloadedEpisodesScan (all root folders)"
|
||
- SCAN_PAYLOAD='{"name": "DownloadedEpisodesScan"}'
|
||
+if [[ "${#SCAN_ROOTS[@]}" -eq 0 ]]; then
|
||
+ error "No root folders returned from Sonarr API — aborting"
|
||
+ notify "Sonarr cleanup aborted on $(hostname) — no root folders from API" \
|
||
+ "Sonarr Cleanup" "warning"
|
||
+ exit 1
|
||
fi
|
||
|
||
+info "Scan targets (${#SCAN_ROOTS[@]}): ${SCAN_ROOTS[*]}"
|
||
+
|
||
+info "Triggering DownloadedEpisodesScan (all root folders)"
|
||
+SCAN_PAYLOAD='{"name": "DownloadedEpisodesScan"}'
|
||
+
|
||
SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \
|
||
-H "X-Api-Key: $SONARR_API_KEY" \
|
||
-H "Content-Type: application/json" \
|
||
@@ -486,7 +487,7 @@ while IFS= read -r filepath; do
|
||
fi
|
||
|
||
done < <(
|
||
- for host_path in "${ARR_PATH_MAP[@]}" "$SONARR_TV_ROOT"; do
|
||
+ for host_path in "${SCAN_ROOTS[@]}"; do
|
||
[[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null
|
||
done | sort -u
|
||
)
|
||
@@ -531,13 +532,13 @@ if [[ "$DRY_RUN" == false ]]; then
|
||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||
|
||
done < <(
|
||
- for host_path in "${ARR_PATH_MAP[@]}" "$SONARR_TV_ROOT"; do
|
||
+ for host_path in "${SCAN_ROOTS[@]}"; do
|
||
[[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null
|
||
done | sort -u
|
||
)
|
||
|
||
info "Cleaning up empty folders..."
|
||
- for host_path in "${ARR_PATH_MAP[@]}" "$SONARR_TV_ROOT"; do
|
||
+ for host_path in "${SCAN_ROOTS[@]}"; do
|
||
[[ -d "$host_path" ]] && \
|
||
find "$host_path" -mindepth 1 -type d -empty -delete 2>/dev/null
|
||
done
|
||
diff --git a/Orchestrators/array_started.sh b/Orchestrators/array_started.sh
|
||
index 4e4a317..2616f72 100755
|
||
--- a/Orchestrators/array_started.sh
|
||
+++ b/Orchestrators/array_started.sh
|
||
@@ -123,6 +123,13 @@ fi
|
||
# ==============================================================================================
|
||
# ━━━ Launch Scripts ━━━
|
||
# ==============================================================================================
|
||
+if [[ ${#ARRAY_START_SCRIPTS[@]} -eq 0 ]]; then
|
||
+ notify "Array started on $LOCAL_SERVER_NAME ($MY_ID) but ARRAY_START_SCRIPTS is empty — boot sequence skipped. Check master.conf." \
|
||
+ "Array Start" "alert"
|
||
+ error "ARRAY_START_SCRIPTS is empty — check master.conf"
|
||
+ exit 1
|
||
+fi
|
||
+
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Array Start — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
log "Ecosystem root: $ECOSYSTEM_ROOT"
|
||
diff --git a/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh b/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh
|
||
index dc55c5a..f60fead 100755
|
||
--- a/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh
|
||
+++ b/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh
|
||
@@ -37,7 +37,7 @@ acquire_lock
|
||
detect_hosts
|
||
|
||
# ──────────────────────────────────────────────────────────────────────────────
|
||
-CONF_FILE="$SCRIPT_DIR/../Configurations/${MY_ID,,}.conf"
|
||
+CONF_FILE="$SCRIPT_DIR/../../../Configurations/${MY_ID,,}.conf"
|
||
VAR_NAME="${MY_ID}_UNRAID_API_KEY"
|
||
|
||
# Key name: "Varaverk <hostname>" stripping any unraid- prefix
|
||
@@ -69,8 +69,21 @@ KEY=$(echo "$EXISTING" | jq -r '.key // empty' 2>/dev/null)
|
||
|
||
if [[ -n "$KEY" ]]; then
|
||
PREVIEW="${KEY:0:8}...${KEY: -4}"
|
||
- echo "API key valid ✅ — $VAR_NAME = $PREVIEW"
|
||
- log "Key found in registry — no renewal needed"
|
||
+ # Always sync registry key → conf, even if the key was already there.
|
||
+ # Conf gets wiped on git pull / conf regeneration without touching the registry.
|
||
+ CONF_HAS_KEY=$(grep -oP "(?<=^\s*${VAR_NAME}=\")[^\"]*" "$CONF_FILE" 2>/dev/null || true)
|
||
+ if [[ "$CONF_HAS_KEY" == "$KEY" ]]; then
|
||
+ echo "API key valid ✅ — $VAR_NAME = $PREVIEW"
|
||
+ log "Key in registry and conf — no action needed"
|
||
+ exit 0
|
||
+ fi
|
||
+ log "Key in registry but conf is stale — syncing..."
|
||
+ if grep -q "^\s*${VAR_NAME}\s*=" "$CONF_FILE"; then
|
||
+ sed -i "s|^\(\s*${VAR_NAME}\s*=\s*\)\"[^\"]*\"|\1\"${KEY}\"|" "$CONF_FILE"
|
||
+ else
|
||
+ echo " ${VAR_NAME}=\"${KEY}\"" >> "$CONF_FILE"
|
||
+ fi
|
||
+ echo "API key synced to conf ✅ — $VAR_NAME = $PREVIEW"
|
||
exit 0
|
||
fi
|
||
|
||
diff --git a/Plugin/unraid/api/stop.php b/Plugin/unraid/api/stop.php
|
||
index f8882ca..4a58bdc 100644
|
||
--- a/Plugin/unraid/api/stop.php
|
||
+++ b/Plugin/unraid/api/stop.php
|
||
@@ -76,16 +76,17 @@ if (file_exists($namedLock)) {
|
||
if (!in_array(basename($namedLock), $cleared)) $cleared[] = basename($namedLock);
|
||
}
|
||
|
||
-// Update stat file
|
||
+// Update stat file — only clear pid if actually dead (D-state processes survive SIGKILL)
|
||
$now = time();
|
||
-$stat['status'] = 'stopped';
|
||
-$stat['end'] = $now;
|
||
-$stat['exit'] = -1;
|
||
-unset($stat['pid']);
|
||
+$stat['status'] = $dead ? 'stopped' : 'running';
|
||
+$stat['end'] = $dead ? $now : ($stat['end'] ?? null);
|
||
+$stat['exit'] = $dead ? -1 : ($stat['exit'] ?? null);
|
||
+if ($dead) unset($stat['pid']);
|
||
file_put_contents($statFile, json_encode($stat));
|
||
|
||
echo json_encode([
|
||
- 'ok' => true,
|
||
+ 'ok' => $dead,
|
||
'killed' => $dead,
|
||
'locks' => $cleared,
|
||
+ 'error' => $dead ? null : 'Process still alive after SIGKILL (D-state) — lock may persist',
|
||
]);
|
||
diff --git a/Plugin/unraid/js/varaverk.js b/Plugin/unraid/js/varaverk.js
|
||
index d5ec74c..9a609ca 100644
|
||
--- a/Plugin/unraid/js/varaverk.js
|
||
+++ b/Plugin/unraid/js/varaverk.js
|
||
@@ -1,6 +1,47 @@
|
||
// Varaverk — shared JS utilities
|
||
// Page-specific JS lives inline in each page partial.
|
||
|
||
+// ── Shared formatters ────────────────────────────────────────────────────────
|
||
+const _GB = 1073741824, _MB = 1048576, _TB = 1099511627776;
|
||
+
|
||
+// Relative timestamp → "just now" / "5m ago" / "yesterday" / "3d ago"
|
||
+function _relTime(ts) {
|
||
+ if (!ts) return '—';
|
||
+ const d = Math.floor(Date.now() / 1000) - ts;
|
||
+ if (d < 60) return 'just now';
|
||
+ if (d < 3600) return Math.floor(d / 60) + 'm ago';
|
||
+ if (d < 86400) return Math.floor(d / 3600) + 'h ago';
|
||
+ if (d < 172800) return 'yesterday';
|
||
+ return Math.floor(d / 86400) + 'd ago';
|
||
+}
|
||
+
|
||
+// Bytes → human-readable size (GB / MB / KB)
|
||
+function _fmtBytes(b) {
|
||
+ if (!b) return '—';
|
||
+ if (b >= _GB) return (b / _GB).toFixed(1) + ' GB';
|
||
+ if (b >= _MB) return (b / _MB).toFixed(0) + ' MB';
|
||
+ return (b / 1024).toFixed(0) + ' KB';
|
||
+}
|
||
+
|
||
+// Bytes → auto-scale TB / GB / MB
|
||
+function _sz(b) {
|
||
+ if (!b) return '—';
|
||
+ if (b >= _TB) return (b / _TB).toFixed(1) + ' TB';
|
||
+ if (b >= _GB) return (b / _GB).toFixed(1) + ' GB';
|
||
+ return (b / _MB).toFixed(0) + ' MB';
|
||
+}
|
||
+
|
||
+function _gb(b) { return !b ? '—' : (b / _GB).toFixed(1) + ' GB'; }
|
||
+function _tb(b) { return !b ? '—' : (b / _TB).toFixed(2) + ' TB'; }
|
||
+function _n(v) { return v == null ? '—' : Number(v).toLocaleString(); }
|
||
+
|
||
+// Seconds → uptime string: "2d 3h 5m" / "3h 5m" / "5m"
|
||
+function _uptime(sec) {
|
||
+ if (!sec) return '—';
|
||
+ const d = Math.floor(sec / 86400), h = Math.floor((sec % 86400) / 3600), m = Math.floor((sec % 3600) / 60);
|
||
+ return d > 0 ? `${d}d ${h}h ${m}m` : h > 0 ? `${h}h ${m}m` : `${m}m`;
|
||
+}
|
||
+
|
||
// Flash a status element briefly then fade
|
||
function vvFlashStatus(el, msg, ok) {
|
||
el.textContent = msg;
|
||
diff --git a/Plugin/unraid/pages/arrs.php b/Plugin/unraid/pages/arrs.php
|
||
index 047c483..9272796 100644
|
||
--- a/Plugin/unraid/pages/arrs.php
|
||
+++ b/Plugin/unraid/pages/arrs.php
|
||
@@ -96,26 +96,6 @@
|
||
<script>
|
||
(function() {
|
||
|
||
-// ── Helpers ───────────────────────────────────────────────────────────────────
|
||
-function _rel(ts) {
|
||
- if (!ts) return '—';
|
||
- const d = Math.floor(Date.now() / 1000) - ts;
|
||
- if (d < 60) return 'just now';
|
||
- if (d < 3600) return Math.floor(d/60) + 'm ago';
|
||
- if (d < 86400) return Math.floor(d/3600) + 'h ago';
|
||
- if (d < 172800)return 'yesterday';
|
||
- return Math.floor(d/86400) + 'd ago';
|
||
-}
|
||
-function _n(v) { return v == null ? '—' : Number(v).toLocaleString(); }
|
||
-function _gb(b) { if (!b) return '—'; return (b / 1073741824).toFixed(1) + ' GB'; }
|
||
-function _tb(b) { if (!b) return '—'; return (b / 1099511627776).toFixed(2) + ' TB'; }
|
||
-function _sz(b) {
|
||
- if (!b) return '—';
|
||
- if (b >= 1099511627776) return (b/1099511627776).toFixed(1) + ' TB';
|
||
- if (b >= 1073741824) return (b/1073741824).toFixed(1) + ' GB';
|
||
- return (b/1048576).toFixed(0) + ' MB';
|
||
-}
|
||
-
|
||
// ── Arr card ──────────────────────────────────────────────────────────────────
|
||
function _arrCard(arr) {
|
||
const LABELS = { sonarr:'Sonarr', radarr:'Radarr', lidarr:'Lidarr' };
|
||
@@ -209,7 +189,7 @@ function _arrCard(arr) {
|
||
cleanup = `
|
||
<div class="vv-arr-meta">
|
||
<span class="vv-arr-meta-lbl">Cleanup</span>
|
||
- <span class="vv-arr-meta-val">${_rel(cl.last_run)} ${st}</span>
|
||
+ <span class="vv-arr-meta-val">${_relTime(cl.last_run)} ${st}</span>
|
||
</div>
|
||
${cl.tracked != null ? `<div class="vv-arr-meta-sub" style="${orCol}">
|
||
${_n(cl.tracked)} files · ${orph} orphans${junk ? ' · ' + junk + ' junk' : ''}
|
||
@@ -223,7 +203,7 @@ function _arrCard(arr) {
|
||
const added = dsc.added != null ? ` <span style="color:#6fcf97">+${dsc.added}</span>` : '';
|
||
disc = `<div class="vv-arr-meta">
|
||
<span class="vv-arr-meta-lbl">Discovery</span>
|
||
- <span class="vv-arr-meta-val">${_rel(dsc.last_run)}${added}</span>
|
||
+ <span class="vv-arr-meta-val">${_relTime(dsc.last_run)}${added}</span>
|
||
</div>`;
|
||
}
|
||
|
||
@@ -295,7 +275,7 @@ function _syncSection(sync, recovery, settings) {
|
||
const pillTxt = !data.last_run ? 'never run' : ok ? 'ok' : data.status || 'error';
|
||
let body = '';
|
||
if (data.last_run) {
|
||
- body += `<div style="font-size:11px;color:#555;margin-bottom:10px;">${_rel(data.last_run)}</div>`;
|
||
+ body += `<div style="font-size:11px;color:#555;margin-bottom:10px;">${_relTime(data.last_run)}</div>`;
|
||
}
|
||
body += `<div class="vv-arr-sr-row">${items}</div>`;
|
||
if (extra) body += extra;
|
||
@@ -410,7 +390,12 @@ function _render(data) {
|
||
|
||
function vvArrsLoad() {
|
||
fetch('/plugins/varaverk/api/arrs.php')
|
||
- .then(r => r.json()).then(_render).catch(() => {});
|
||
+ .then(r => r.json())
|
||
+ .then(_render)
|
||
+ .catch(() => {
|
||
+ document.getElementById('vv-arrs-grid').innerHTML =
|
||
+ '<div style="grid-column:1/-1;color:#555;font-size:12px;padding:24px 0;text-align:center;">Error loading arrs data — check API</div>';
|
||
+ });
|
||
}
|
||
|
||
vvArrsLoad();
|
||
diff --git a/Plugin/unraid/pages/docker.php b/Plugin/unraid/pages/docker.php
|
||
index c6150df..85681ad 100644
|
||
--- a/Plugin/unraid/pages/docker.php
|
||
+++ b/Plugin/unraid/pages/docker.php
|
||
@@ -291,7 +291,11 @@ function _bindEvents() {
|
||
if (!fid || !name) return;
|
||
const f = (_data.folders||[]).find(x=>x.id===fid);
|
||
if (f && name===f.name) return;
|
||
- _api({action:'rename_folder',folder_id:fid,name}, r => { if(r.ok) _reload(); else alert('Rename failed: '+(r.error||'?')); });
|
||
+ el.disabled = true; el.style.opacity = '0.5';
|
||
+ _api({action:'rename_folder',folder_id:fid,name}, r => {
|
||
+ el.disabled = false; el.style.opacity = '';
|
||
+ if (r.ok) _reload(); else { alert('Rename failed: '+(r.error||'?')); el.value = f?.name ?? name; }
|
||
+ });
|
||
});
|
||
});
|
||
|
||
diff --git a/Plugin/unraid/pages/fallback.php b/Plugin/unraid/pages/fallback.php
|
||
index 8b24807..4a8c77a 100644
|
||
--- a/Plugin/unraid/pages/fallback.php
|
||
+++ b/Plugin/unraid/pages/fallback.php
|
||
@@ -233,7 +233,10 @@ function vvFbLoad() {
|
||
fetch('/plugins/varaverk/api/fallback.php')
|
||
.then(r => r.json())
|
||
.then(_render)
|
||
- .catch(() => {});
|
||
+ .catch(() => {
|
||
+ document.getElementById('vv-fb-grid').innerHTML =
|
||
+ '<div style="grid-column:1/-1;color:#555;font-size:12px;padding:24px 0;text-align:center;">Error loading fallback data — check API</div>';
|
||
+ });
|
||
}
|
||
|
||
vvFbLoad();
|
||
diff --git a/Plugin/unraid/pages/partnership.php b/Plugin/unraid/pages/partnership.php
|
||
index 72ec10c..9cefa28 100644
|
||
--- a/Plugin/unraid/pages/partnership.php
|
||
+++ b/Plugin/unraid/pages/partnership.php
|
||
@@ -593,24 +593,6 @@ function vvPtSaveSettings(btn) {
|
||
|
||
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||
|
||
-function _relTime(ts) {
|
||
- if (!ts) return '—';
|
||
- const d = Math.floor(Date.now() / 1000) - ts;
|
||
- if (d < 60) return 'just now';
|
||
- if (d < 3600) return Math.floor(d / 60) + 'm ago';
|
||
- if (d < 86400) return Math.floor(d / 3600) + 'h ago';
|
||
- if (d < 172800) return 'yesterday';
|
||
- return Math.floor(d / 86400) + 'd ago';
|
||
-}
|
||
-
|
||
-function _uptime(sec) {
|
||
- if (!sec) return '—';
|
||
- const d = Math.floor(sec / 86400);
|
||
- const h = Math.floor((sec % 86400) / 3600);
|
||
- const m = Math.floor((sec % 3600) / 60);
|
||
- return d > 0 ? `${d}d ${h}h ${m}m` : h > 0 ? `${h}h ${m}m` : `${m}m`;
|
||
-}
|
||
-
|
||
function _row(lbl, val) {
|
||
return `<div class="vv-pt-row"><span class="vv-pt-lbl">${lbl}</span><span class="vv-pt-val">${val}</span></div>`;
|
||
}
|
||
@@ -1083,7 +1065,7 @@ function _renderActions(nodes, cfg) {
|
||
</div>
|
||
<div style="display:flex;flex-direction:column;gap:5px;align-items:flex-end;">
|
||
<button class="vv-pt-action-btn info" onclick="vvPtPhase2(this,'${remote.id}')"
|
||
- style="opacity:.35;font-size:10px;" title="Re-run Phase 2">↻ Re-run</button>
|
||
+ style="font-size:10px;" title="Re-run Phase 2">↻ Re-run</button>
|
||
<button class="vv-pt-action-btn warn" onclick="vvPtShowDeleteKeys('${remote.id}')"
|
||
style="opacity:.4;font-size:10px;">🗑 Keys</button>
|
||
</div>
|
||
diff --git a/Plugin/unraid/pages/setup.php b/Plugin/unraid/pages/setup.php
|
||
index fd92aec..b342f66 100644
|
||
--- a/Plugin/unraid/pages/setup.php
|
||
+++ b/Plugin/unraid/pages/setup.php
|
||
@@ -178,7 +178,9 @@ let _vvRedirect = '?tab=scheduler';
|
||
|
||
// ── Detection banner ──────────────────────────────────────────────────────────
|
||
(function() {
|
||
- fetch('/plugins/varaverk/api/setup.php?action=detect&_=' + Date.now())
|
||
+ const _ac = new AbortController();
|
||
+ setTimeout(() => _ac.abort(), 6000);
|
||
+ fetch('/plugins/varaverk/api/setup.php?action=detect&_=' + Date.now(), {signal: _ac.signal})
|
||
.then(r => r.json()).then(d => {
|
||
const b = document.getElementById('vv-detect-banner');
|
||
if (!d.ok) { b.innerHTML = '<span style="color:#555">Detection unavailable</span>'; return; }
|
||
diff --git a/Plugin/varaverk.plg b/Plugin/varaverk.plg
|
||
index a6c87eb..0e506c3 100644
|
||
--- a/Plugin/varaverk.plg
|
||
+++ b/Plugin/varaverk.plg
|
||
@@ -195,8 +195,8 @@ fi
|
||
|
||
# Seed master.conf from template if absent.
|
||
mkdir -p "$CONF_DIR"
|
||
-if [[ ! -f "$CONF_DIR/master.conf" && -f "$SCRIPTS_DIR/Deployment/conf_templates/master.conf" ]]; then
|
||
- cp "$SCRIPTS_DIR/Deployment/conf_templates/master.conf" "$CONF_DIR/master.conf"
|
||
+if [[ ! -f "$CONF_DIR/master.conf" && -f "$CONF_DIR/master.conf.template" ]]; then
|
||
+ cp "$CONF_DIR/master.conf.template" "$CONF_DIR/master.conf"
|
||
log "seeded master.conf from template"
|
||
fi
|
||
log "install step complete"
|
||
diff --git a/common.sh b/common.sh
|
||
index d842801..e0060c0 100755
|
||
--- a/common.sh
|
||
+++ b/common.sh
|
||
@@ -1490,8 +1490,9 @@ acquire_rsync_lock() {
|
||
local actual_count=0
|
||
for lf in "$LOCK_DIR"/rsync_*.lock; do
|
||
[[ -f "$lf" ]] || continue
|
||
- local lpid
|
||
- lpid=$(cat "$lf" 2>/dev/null)
|
||
+ local lpid lf_content
|
||
+ lf_content=$(cat "$lf" 2>/dev/null)
|
||
+ lpid="${lf_content%%:*}"
|
||
kill -0 "$lpid" 2>/dev/null && ((actual_count++))
|
||
done
|
||
if [[ "$actual_count" -ne "$current_count" ]]; then
|
||
diff --git a/git_pull_execute.sh b/git_pull_execute.sh
|
||
index 32d1d26..bf7d2b3 100755
|
||
--- a/git_pull_execute.sh
|
||
+++ b/git_pull_execute.sh
|
||
@@ -281,7 +281,6 @@ echo ""
|
||
echo "━━━ $ICON_GEAR Conf Upgrade ━━━"
|
||
|
||
UPGRADE_SCRIPT="$TARGET_DIR/Deployment/conf_upgrade.sh"
|
||
-TMPL_DIR="$TARGET_DIR/Deployment/conf_templates"
|
||
CONF_DIR="$TARGET_DIR/Configurations"
|
||
|
||
if [[ ! -f "$UPGRADE_SCRIPT" ]]; then
|
||
@@ -292,20 +291,20 @@ elif [[ "$SYNC_SUCCESS" == true ]]; then
|
||
_DRY=""
|
||
|
||
# master.conf
|
||
- if [[ -f "$TMPL_DIR/master.conf" && -f "$CONF_DIR/master.conf" ]]; then
|
||
+ if [[ -f "$CONF_DIR/master.conf.template" && -f "$CONF_DIR/master.conf" ]]; then
|
||
bash "$UPGRADE_SCRIPT" \
|
||
- --template "$TMPL_DIR/master.conf" \
|
||
+ --template "$CONF_DIR/master.conf.template" \
|
||
--target "$CONF_DIR/master.conf" \
|
||
--backup $_DRY
|
||
else
|
||
- warn "master.conf template or target not found — skipping"
|
||
+ warn "master.conf.template or master.conf not found — skipping"
|
||
fi
|
||
|
||
# This server's host conf only — sparse checkout ensures we have it
|
||
HOST_CONF="$CONF_DIR/${MY_ID,,}.conf"
|
||
- if [[ -f "$TMPL_DIR/host.conf.template" && -f "$HOST_CONF" ]]; then
|
||
+ if [[ -f "$CONF_DIR/host.conf.template" && -f "$HOST_CONF" ]]; then
|
||
bash "$UPGRADE_SCRIPT" \
|
||
- --template "$TMPL_DIR/host.conf.template" \
|
||
+ --template "$CONF_DIR/host.conf.template" \
|
||
--target "$HOST_CONF" \
|
||
--backup $_DRY
|
||
else
|