FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added failover script and other sytem scripts

Browse Source
This commit is contained in:
FailedProxy
2026-04-11 17:46:11 -04:00
parent ee107e5df7
commit 55930a1166
7 changed files with 1388 additions and 386 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Master.conf
+116 -122
View File
@@ -20,10 +20,14 @@
# DAILY SYNC SHARES Media shares synced by daily_sync.sh
# RSYNC PROFILE SYSTEM Per-profile overrides (appdata profiles)
#
# ── FAILOVER ───────────────────────────────────────────────────────────────────────────────
# FAILOVER Mutual container failover between two servers
#
# ── DOCKER ESSENTIALS ──────────────────────────────────────────────────────────────────────
# DOCKER DAILY RESTART Containers restarted daily
# DOCKER WEEKLY RESTART Containers restarted weekly
# DOCKER WATCHDOG Container health monitoring — memory, CPU, HTTP
# DOCKER NETWORK CONNECT Connect containers to extra networks on boot
#
# ── UNRAID ESSENTIALS ──────────────────────────────────────────────────────────────────────
# REBOOT User warning delay before scheduled reboot
@@ -31,6 +35,8 @@
# SYSLOG FILTER Docker veth noise filter file path
# PHP-FPM PHP-FPM max children config
# CLEAR LOGS System log file paths
# WEBGUI WATCHDOG WebGUI nginx + emhttp monitoring and restart
# ZFS MEMORY SNAPSHOT Weekly ZFS health and memory diagnostic report
#
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
# MEDIA PERMISSIONS Share list, mode and owner for permissions script
@@ -45,22 +51,16 @@
# ==============================================================================================
# ━━━ Host Configuration ━━━
# Hostnames must match Tailscale machine names exactly — case sensitive
HOST1="unRAID-Gmer4Lfe"
HOST2="unRAID-Jayred365"
# SSH keys for server-to-server rsync — each server needs the other's key authorised
HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key"
HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key"
# ━━━ Logging ━━━
# true = verbose [LOG] output in scripts / false = user-facing output only
ENABLE_LOGGING=true
# ━━━ Notifications ━━━
# unRAID native — configure Settings → Notification Settings for errors/warnings only
NOTIFY_UNRAID=true
# Discord webhook URL — leave blank to disable
DISCORD_WEBHOOK=""
# ━━━ Git / Repo ━━━
@@ -74,25 +74,19 @@
# ==============================================================================================
# ━━━ Rsync Defaults ━━━
# Global fallback values — used when no profile match is found for a share.
# Shares in DAILY_SYNC_SHARES always use these globals (no profile defined).
BW_LIMIT=12500 # network speed limit KB/s
RETRY_COUNT=3 # number of retry attempts on failure
SLEEP=300 # seconds between retries
CRITICAL_CONTAINER_NAMES=() # containers to stop before rsync
DELAYED_CONTAINERS=() # containers needing delay before start
CONTAINER_DELAY=5 # seconds delay before starting delayed containers
EXCLUDE_DIRS=() # directories to exclude from transfer
BW_LIMIT=12500
RETRY_COUNT=3
SLEEP=300
CRITICAL_CONTAINER_NAMES=()
DELAYED_CONTAINERS=()
CONTAINER_DELAY=5
EXCLUDE_DIRS=()
DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file)
# ━━━ Remote Health Checks ━━━
# Abort rsync if remote rootfs exceeds this percentage.
# Protects against rsync filling rootfs when remote array is down or drives are missing.
ROOTFS_WARN=75
# ━━━ Daily Sync Shares ━━━
# Shares synced once daily by Orchestrators/daily_sync.sh
# No profile needed — all fall through to DEFAULT_RSYNC_OPTS above.
DAILY_SYNC_SHARES=(
/mnt/user/Anime_Movies-Old
/mnt/user/Anime_Shows-Old
@@ -109,23 +103,6 @@ DAILY_SYNC_SHARES=(
)
# ━━━ Rsync Profile System ━━━
# Profiles are matched by directory basename (lowercased).
# Example: /mnt/user/appdata-Failover/Arrs_Stack → profile key = arrs_stack
#
# How fallthrough works:
# - If a key exists in a profile array that value is used
# - If a key is missing the global default above is used instead
# - Shares in DAILY_SYNC_SHARES have no profile and always use globals
#
# To add a new profile:
# 1. Add a key to each array below with your chosen profile name
# 2. Call rsync.sh with a directory whose basename matches that key
# 3. Any array you omit falls back to its global default
#
# Note: PROFILE_RSYNC_OPTS does NOT inherit from DEFAULT_RSYNC_OPTS —
# list all desired options explicitly if you define a profile entry
# SPACE-SEPARATED STRINGS
declare -A PROFILE_RSYNC_OPTS=(
[arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace"
[critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete"
@@ -158,7 +135,6 @@ declare -A PROFILE_SLEEP=(
[emby]=300
)
# SPACE-SEPARATED STRINGS
declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
[arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
[critical-data]="Mariadb-Authelia Redis-Authelia Lldap-Gmer4Lfe NginxProxyManager Authelia"
@@ -167,7 +143,6 @@ declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
[emby]=""
)
# SPACE-SEPARATED STRINGS — containers needing delay before starting
declare -A PROFILE_DELAYED_CONTAINERS=(
[arrs_stack]=""
[critical-data]="Authelia"
@@ -184,7 +159,6 @@ declare -A PROFILE_CONTAINER_DELAY=(
[emby]=5
)
# SPACE-SEPARATED STRINGS
declare -A PROFILE_EXCLUDE_DIRS=(
[arrs_stack]="logs *.tmp"
[critical-data]="logs *.tmp"
@@ -193,12 +167,59 @@ declare -A PROFILE_EXCLUDE_DIRS=(
[emby]="logs *.tmp"
)
# ==============================================================================================
# ── FAILOVER ──────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Mutual container failover between two unRAID servers.
# Each server runs failover.sh independently — no coordination between servers.
# Decisions based solely on two ping checks: remote reachable + internet reachable.
# Comment out any container or rsync job to disable without removing the entry.
EXTERNAL_IP="8.8.8.8"
FAILOVER_CHECK_INTERVAL=120
FAILOVER_HANDBACK_STRIKES=2
FAILOVER_STATE_FILE="/boot/config/failover_state.db"
# Containers HOST1 starts locally when HOST2 goes down
FAILOVER_HOST1_STARTS_FOR_HOST2=(
"Vaultwarden-Jayred365"
"Nextcloud-Jayred365"
"Cloudflare-DDNS-Jayred365"
)
# Containers HOST1 stops when it loses internet
FAILOVER_HOST1_STOP_ON_NO_NET=(
"Emby"
"Cloudflare-DDNS-Gmer4Lfe"
)
# Rsync jobs HOST1 runs before handing containers back to HOST2
FAILOVER_HOST1_RSYNC_JOBS=(
"/mnt/user/appdata-Failover/Jayred365"
"/mnt/user/Media_Server/Emby-Jayred"
)
# Containers HOST2 starts locally when HOST1 goes down
FAILOVER_HOST2_STARTS_FOR_HOST1=(
"Emby"
"Cloudflare-DDNS-Gmer4Lfe"
)
# Containers HOST2 stops when it loses internet
FAILOVER_HOST2_STOP_ON_NO_NET=(
"Cloudflare-DDNS-Jayred365"
)
# Rsync jobs HOST2 runs before handing containers back to HOST1
FAILOVER_HOST2_RSYNC_JOBS=(
"/mnt/user/appdata-Failover/Gmer4Lfe"
)
# ==============================================================================================
# ── DOCKER ESSENTIALS ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ Docker Daily Restart ━━━
# Containers restarted every day — case-sensitive names
DAILY_RESTART_CONTAINERS=(
"NginxProxyManager"
"Authelia"
@@ -209,7 +230,6 @@ DAILY_RESTART_CONTAINERS=(
)
# ━━━ Docker Weekly Restart ━━━
# Containers restarted once per week — case-sensitive names
WEEKLY_RESTART_CONTAINERS=(
"NginxProxyManager"
"Authelia"
@@ -220,11 +240,6 @@ WEEKLY_RESTART_CONTAINERS=(
)
# ━━━ Docker Watchdog ━━━
# First line of defense — monitors and restarts unhealthy containers.
# Runs on a cron schedule (recommended every 15 minutes).
# Strike system prevents restarts on brief spikes.
# Containers to monitor with memory hard limits in MB
# 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240
# 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024
declare -A WATCHDOG_CONTAINERS=(
@@ -235,44 +250,47 @@ declare -A WATCHDOG_CONTAINERS=(
["Code-Server"]=1024
)
# Containers to check HTTP responsiveness — omit to skip
declare -A WATCHDOG_CONTAINER_URLS=(
["Emby"]="http://localhost:8096"
["Jellyfin-Gmer4Lfe"]="http://localhost:8095"
)
# Containers that should always be running — monitored for unexpected stops
# Strike system used — persistent skip list prevents reboot loops
WATCHDOG_REQUIRED_CONTAINERS=(
"NginxProxyManager"
"Authelia"
"Emby"
)
# Strike state file — /tmp resets on reboot, correct for strike tracking
WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db"
SOFT_CPU_THRESHOLD=80
HARD_CPU_THRESHOLD=85
CPU_FAIL_LIMIT=2
SOFT_MEM_THRESHOLD=80
RESP_FAIL_LIMIT=2
CURL_TIMEOUT=5
# CPU thresholds — normalised against total core count at runtime
SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU
HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU
CPU_FAIL_LIMIT=2 # consecutive strikes before restart
# ━━━ Docker Network Connect ━━━
# Containers connected to extra networks on array start — many-to-many
# Every container connects to every network listed
# Comment out entries to disable without removing them
NETWORK_CONNECT_CONTAINERS=(
"memcached"
"Npm-CrowdSec"
)
# Memory threshold
SOFT_MEM_THRESHOLD=80 # warn at this % of per-container hard limit
# Responsiveness check
RESP_FAIL_LIMIT=2 # consecutive failures before restart
CURL_TIMEOUT=5 # seconds before curl gives up
NETWORK_CONNECT_NETWORKS=(
"nextcloud-aio"
)
# ==============================================================================================
# ── UNRAID ESSENTIALS ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ Reboot ━━━
REBOOT_SLEEP=300 # user warning delay before scheduled reboot (seconds)
REBOOT_SLEEP=300
# ━━━ Mover ━━━
MOVER_STOP_TIMEOUT=300 # timeout before stopping the mover (seconds)
MOVER_STOP_TIMEOUT=300
# ━━━ Syslog Filter ━━━
FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"
@@ -284,6 +302,21 @@ WATCHDOG_REQUIRED_CONTAINERS=(
# ━━━ Clear Logs ━━━
LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg)
# ━━━ WebGUI Watchdog ━━━
# Monitors unRAID WebGUI — escalates from nginx restart to emhttp restart if needed
WEBGUI_URL="http://localhost" # adjust port if running non-standard e.g. http://localhost:8080
WEBGUI_TIMEOUT=5 # seconds before curl gives up
WEBGUI_NGINX_WAIT=15 # seconds to wait after nginx restart before recheck
WEBGUI_EMHTTP_WAIT=30 # seconds to wait after emhttp restart before recheck
# ━━━ ZFS Memory Snapshot ━━━
# Weekly ZFS health and memory diagnostic report
ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log"
ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC utilization above this %
ZFS_REPORT_FREE_WARN_GB=10 # warn if free RAM below this GB
ZFS_REPORT_AVAIL_WARN_GB=20 # warn if available RAM below this GB
ZFS_REPORT_DOCKER_TOP=10 # number of top Docker memory users to show
# ==============================================================================================
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -318,9 +351,6 @@ MEDIA_PERMISSION_SHARES=(
)
# ━━━ Media Cleaner ━━━
# Two profiles: anime and media — passed as argument to media_cleaner.sh
# Usage: media_cleaner.sh anime or media_cleaner.sh media
ANIME_CLEAN_FOLDERS=(
/mnt/user/Anime_Movies
/mnt/user/Anime_Movies-Old
@@ -355,70 +385,38 @@ MEDIA_FILE_PATTERNS=(
# ==============================================================================================
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Session-based storage allocator using filesystem indirection.
# New transcode sessions land wherever TRANSCODE_LINK points.
# Existing sessions are never interrupted — ffmpeg resolves path once at session start.
#
# Flow:
# ramdisk_setup.sh — run once at array start, creates ramdisk and symlink
# transcode_manager.sh — runs every 2-3 min, monitors usage and flips symlink
# transcode_cleanup.sh — runs every 5 min, removes old inactive files
#
# Hysteresis gap between RAMDISK_WARN_GB and RAMDISK_LOW_GB prevents flip-flop
# when usage hovers near the threshold. Gap should be at least 0.5-1GB.
# ━━━ Transcode Manager ━━━
# Paths
RAMDISK_PATH="/mnt/ramdisk_transcodes" # tmpfs mount point
RAMDISK_SIZE="8G" # increase if you have RAM headroom
TRANSCODE_LINK="/mnt/ram-transcode" # symlink Emby points at
TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" # SSD fallback location
# Thresholds in GB
RAMDISK_WARN_GB=6.8 # flip symlink to SSD at or above this usage
RAMDISK_LOW_GB=5.5 # flip symlink back to ramdisk when usage drops here
RAMDISK_SSD_MIN_GB=20 # minimum free GB on SSD before allowing flip — abort if below
# Cleanup settings
TRANSCODE_MAX_AGE=20 # minutes before a file is eligible for cleanup
TRANSCODE_ORPHAN_AGE=30 # minutes before an orphaned file is eligible (slightly longer buffer)
# Flip frequency monitoring
TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in one hour
# Permissions — should match your Emby container user
RAMDISK_PATH="/mnt/ramdisk_transcodes"
RAMDISK_SIZE="8G"
TRANSCODE_LINK="/mnt/ram-transcode"
TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
RAMDISK_WARN_GB=6.8
RAMDISK_LOW_GB=5.5
RAMDISK_SSD_MIN_GB=20
TRANSCODE_MAX_AGE=20
TRANSCODE_ORPHAN_AGE=30
TRANSCODE_FLIP_WARN=3
TRANSCODE_OWNER="nobody:users"
TRANSCODE_MODE="755"
# ==============================================================================================
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Last line of defense — reboots the system cleanly if it is about to become unstable.
# Works alongside docker_watchdog.sh — containers first, system second.
# Thresholds set at "about to fall over" levels — not just high usage.
# ━━━ System Watchdog State Files ━━━
SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db"
SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db"
SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db"
# ━━━ Strike and Reboot Loop Settings ━━━
SYS_WATCHDOG_STRIKE_LIMIT=2 # consecutive hits before reboot trigger
SYS_WATCHDOG_REBOOT_LIMIT=3 # max reboots in window before shutdown instead
SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 # window in hours — controls reboot count AND log purge
# ━━━ Thresholds ━━━
SYS_WATCHDOG_ROOTFS_PCT=95 # rootfs % before strike
SYS_WATCHDOG_LOG_PCT=95 # /var/log % before strike
SYS_WATCHDOG_MEM_GB=4 # free RAM GB below which strikes (128GB system)
SYS_WATCHDOG_ARC_PINNED_PCT=98 # ZFS ARC % of max before reclaim attempt
SYS_WATCHDOG_ARC_RELEASE_PCT=95 # ZFS ARC % after reclaim that still triggers
SYS_WATCHDOG_LOAD_MULTIPLIER=3 # strike if load avg > cores x this value
SYS_WATCHDOG_ZOMBIE_LIMIT=50 # strike if zombie count exceeds this
SYS_WATCHDOG_CPU_TEMP_MAX=95 # degrees C — adjust for your CPU tjmax
# ━━━ Check Toggles ━━━
# true = run this check / false = skip entirely
SYS_WATCHDOG_STRIKE_LIMIT=2
SYS_WATCHDOG_REBOOT_LIMIT=3
SYS_WATCHDOG_REBOOT_WINDOW_HRS=12
SYS_WATCHDOG_ROOTFS_PCT=95
SYS_WATCHDOG_LOG_PCT=95
SYS_WATCHDOG_MEM_GB=4
SYS_WATCHDOG_ARC_PINNED_PCT=98
SYS_WATCHDOG_ARC_RELEASE_PCT=95
SYS_WATCHDOG_LOAD_MULTIPLIER=3
SYS_WATCHDOG_ZOMBIE_LIMIT=50
SYS_WATCHDOG_CPU_TEMP_MAX=95
SYS_WATCHDOG_CHECK_ROOTFS=true
SYS_WATCHDOG_CHECK_LOG=true
SYS_WATCHDOG_CHECK_RAM=true
@@ -428,10 +426,6 @@ MEDIA_FILE_PATTERNS=(
SYS_WATCHDOG_CHECK_ZOMBIES=true
SYS_WATCHDOG_CHECK_CONTAINERS=true
SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
# ━━━ Abort Toggles ━━━
# true = abort reboot if condition active / false = reboot anyway
# Default true = conservative — set false only when you want "reboot no matter what"
SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true
SYS_WATCHDOG_ABORT_ON_PARITY=true
SYS_WATCHDOG_ABORT_ON_MOVER=true