added failover script and other sytem scripts

This commit is contained in:
2026-04-11 17:46:11 -04:00
parent ee107e5df7
commit 55930a1166
7 changed files with 1388 additions and 386 deletions
+101 -74
View File
@@ -16,18 +16,18 @@
# Shared runtime functions used by all scripts:
# /mnt/user/appdata/unraid_scripts/common.sh
#
# For full setup instructions see:
# /mnt/user/appdata/unraid_scripts/README.md
#
# ==============================================================================================
# Changelog:
# v1.0 — Initial template
# v1.1 — MAX_RSYNC_PROCS removed — bandwidth limiting handles concurrency
# Typo fixes Master.cong → Master.conf
# Added --status flag to arguments section
# Added full directory tree
# All script calls pre-written and grouped by type
# Changelog added
# Typo fixes, --status flag added, full directory tree, changelog added
# v1.2 — Docker Essentials, Media, Transcodes, System Watchdog added
# Directory tree updated to reflect full ecosystem
# v1.3 — Failover script added, directory tree updated with Failover folder
# v1.4 — WebGUI watchdog added
# ZFS memory snapshot added
# Docker network connect added
# Directory tree updated to reflect full ecosystem
# ==============================================================================================
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
@@ -39,42 +39,60 @@
# ├── common.sh # Shared library — functions used by all scripts
# ├── README.md # Project overview and quick start
# ├── User_Script_Template.sh # This file — copy into User Scripts plugin
# ├── git_pull_execute.sh # Pulls latest scripts from Gitea repo
# │
# ├── Failover/
# │ └── failover.sh # Mutual container failover — runs continuously
# │
# ├── Orchestrators/
# │ └── daily_sync.sh # Runs all daily media share syncs sequentially
# │ # add shares to master.conf, for seqential syncs
# │
# ├── Rsync/
# │ ├── rsync.sh # Core rsync script — called per share or profile
# │ └── README_Rsync_Setup.md # Rsync-specific setup guide
# │
# ├── Docker_Essentials/
# │ ├── docker_watchdog.sh # Container health monitor — memory, CPU, HTTP, stops
# │ ├── docker_daily_restart.sh # Restarts configured containers daily
# │ ├── docker_weekly_restart.sh # Restarts configured containers weekly
# │ └── docker_network_connect.sh # Connects containers to extra networks on boot
# │
# ├── Media/
# │ ├── media_permissions.sh # Applies permissions to all media shares
# │ └── media_cleaner.sh # Removes junk files — profiles: anime, media
# │
# ├── Transcodes/
# │ ├── ramdisk_setup.sh # Creates ramdisk and transcode symlink — run at array start
# │ ├── transcode_manager.sh # Monitors ramdisk usage, manages symlink direction
# │ └── transcode_cleanup.sh # Removes old inactive transcode files
# │
# ├── Tools/
# │ └── recreate_shares.sh # Recreates share dirs from .cfg files after incident
# │
# └── unRAID_Essentials/
# ├── clear_logs.sh # Clears unRAID log files
# ├── docker_syslog_filter.sh # Filters docker veth noise from syslog
# ├── clear_logs.sh # Clears unRAID system and Docker log files
# ├── docker_syslog_filter.sh # Filters Docker veth noise from syslog
# ├── mover_stop.sh # Safely stops the unRAID mover
# ├── php_fpm_max_children.sh # Sets php-fpm max children value
# ├── rsync_stop.sh # Cleanly stops all running rsync processes
# ├── php_fpm_max_children.sh # Sets PHP-FPM max children value
# ├── rsync_stop.sh # Stops all rsync processes on both servers
# ├── server_reboot.sh # Graceful server reboot with user warning
# ├── user_script_stop.sh # Stops a running User Scripts job
# ── zfs_memory_snapshot.sh # Creates a ZFS memory snapshot
# ├── system_watchdog.sh # System health monitor — last line of defense
# ── user_script_stop.sh # Stops running User Scripts plugin jobs
# ├── webgui_restart.sh # WebGUI watchdog — nginx + emhttp escalating restart
# └── zfs_memory_snapshot.sh # Weekly ZFS health and memory diagnostic report
#
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# ━━━ 🚀 Script Commands — Uncomment the one you want to run ━━━
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
#
# ━━━ Orchestrators (README) ━━━
# The daily orchestrator runs all bulk media shares sequentially in a single scheduled job.
# Instead of creating a User Script entry per share, add the share path to DAILY_SYNC_SHARES
# in Master.conf and the orchestrator handles it automatically at 1am.
# ━━━ Failover (README) ━━━
# Runs continuously as a background task — do NOT schedule with a time interval.
# Set schedule to "At Startup of Array" — background task.
# Both servers must have this script running for mutual failover to work.
# Use --status to check current state without restarting the loop.
#
# Media shares are nothing special — they carry no profile and fall through to global defaults
# in Master.conf. Sequential execution means one share finishes before the next starts,
# no concurrency needed, bandwidth limiting keeps things sane if appdata jobs overlap.
#
# This is where 80% of your rsync shares should live. Only create individual scheduled
# entries for shares that need their own timing — like appdata profiles below.
# ━━━ Failover ━━━
#/mnt/user/appdata/unraid_scripts/Failover/failover.sh
#
# ━━━ Orchestrators ━━━
#/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync.sh
@@ -86,9 +104,36 @@
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Emby
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Gmer4Lfe
#
# ━━━ Rsync — Individual Media Share (ad hoc use) ━━━
# ━━━ Rsync — Individual Media Shares (ad hoc use) ━━━
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Movies
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Tv_Shows
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Shows
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Shows-Old
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Movies
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Movies-Old
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Books
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Intros
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Kids_Movies
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Kids_Tv_Shows
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Music_Videos
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Nextcloud
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/stand-up_comedy
#
# ━━━ Docker Essentials ━━━
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_watchdog.sh
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_daily_restart.sh
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh
#
# ━━━ Media ━━━
#/mnt/user/appdata/unraid_scripts/Media/media_permissions.sh
#/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh anime
#/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh media
#
# ━━━ Transcodes ━━━
#/mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh
#/mnt/user/appdata/unraid_scripts/Transcodes/transcode_manager.sh
#/mnt/user/appdata/unraid_scripts/Transcodes/transcode_cleanup.sh
#
# ━━━ Tools ━━━
#/mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh
@@ -100,7 +145,9 @@
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/php_fpm_max_children.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/rsync_stop.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/server_reboot.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/system_watchdog.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/user_script_stop.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/webgui_restart.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/zfs_memory_snapshot.sh
#
@@ -108,66 +155,46 @@
# ━━━ ⚙️ Arguments — Add after the script path ━━━
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
#
# ━━━ Rsync / Orchestrators ━━━
#
# --dry-run Preview what would be transferred — no changes made
# --dry-run Preview what would happen — no changes made
# --log Enable verbose logging output
# --no-log Disable logging (overrides Master.conf setting)
# --status Print resolved profile and configuration then exit
# --status Print resolved configuration and current state then exit
# --help Show usage information
#
# KEY=VALUE Override any Master.conf variable for this run only
# Better to add a profile in Master.conf for permanent changes
#
# ━━━ unRAID Essentials ━━━
# ━━━ Media Cleaner profile (required first argument) ━━━
# media_cleaner.sh anime --dry-run
# media_cleaner.sh media --dry-run
#
# Arguments vary per script — see comments inside each script for details
# ━━━ Failover specific ━━━
# failover.sh --status — check current state
# failover.sh --dry-run --log — test logic without touching containers
#
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# ━━━ 📋 Profile System ━━━
# ━━━ 📋 Recommended Schedules ━━━
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
#
# Profiles are defined in Master.conf and matched automatically by the
# basename of the directory path passed to rsync.sh (lowercased).
# failover.sh — At Startup of Array (background task)
# ramdisk_setup.sh — At Startup of Array
# docker_network_connect.sh — At Startup of Array
# docker_syslog_filter.sh — At Startup of Array
# php_fpm_max_children.sh — At Startup of Array
#
# Example:
# /mnt/user/appdata-Failover/Arrs_Stack → matches profile key [arrs_stack]
# /mnt/user/Movies → no match, uses global defaults
# transcode_manager.sh — */3 * * * * (every 3 minutes)
# transcode_cleanup.sh — */5 * * * * (every 5 minutes)
# docker_watchdog.sh — */15 * * * * (every 15 minutes)
# webgui_restart.sh — */10 * * * * (every 10 minutes)
# system_watchdog.sh — */15 * * * * (every 15 minutes)
#
# If a profile match is found — profile settings are used for that run
# If no profile match is found — global defaults from Master.conf are used
# daily_sync.sh — 0 1 * * * (daily at 1am)
# docker_daily_restart.sh — 0 3 * * * (daily at 3am)
# media_cleaner.sh anime — 0 2 * * * (daily at 2am)
# media_cleaner.sh media — 0 2 * * * (daily at 2am)
# media_permissions.sh — 0 4 * * 0 (weekly Sunday 4am)
# clear_logs.sh — 0 5 * * 0 (weekly Sunday 5am)
# zfs_memory_snapshot.sh — 0 6 * * 0 (weekly Sunday 6am)
# docker_weekly_restart.sh — 0 3 * * 0 (weekly Sunday 3am)
#
# Profile settings control:
# - rsync options PROFILE_RSYNC_OPTS
# - bandwidth limit PROFILE_BW_LIMIT
# - retry count PROFILE_RETRY_COUNT
# - sleep between retry PROFILE_SLEEP
# - containers to stop PROFILE_CRITICAL_CONTAINER_NAMES
# - delayed containers PROFILE_DELAYED_CONTAINERS
# - container delay PROFILE_CONTAINER_DELAY
# - excluded dirs PROFILE_EXCLUDE_DIRS
#
# To add a new profile — add a key to each array in Master.conf
# For permanent changes — always use Master.conf
# For one-off overrides — use KEY=VALUE arguments (see above)
#
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# ━━━ 💡 Examples ━━━
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
#
# Run arrs_stack profile sync:
# /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack
#
# Dry run with logging enabled:
# /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --dry-run --log
#
# Check what profile and settings resolved before running:
# /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --status
#
# Override bandwidth limit for this run only:
# /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Movies BW_LIMIT=5000
#
# Run daily orchestrator:
# /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync.sh
# rsync.sh profiles — schedule individually as needed
#
# ==============================================================================================