Files
Varaverk/user_script_plug-in.sh
T

343 lines
22 KiB
Bash

#!/bin/bash
# ==============================================================================================
# ================================= User Script Template =======================================
# ==============================================================================================
#
# This file is the master template for all scripts run via the unRAID User Scripts plugin.
# Copy and paste the contents of this file into a new User Script entry in the plugin,
# then uncomment the script you want to run and set your schedule.
#
# All scripts live at: /mnt/user/appdata/unraid_scripts/
# All configuration at: /mnt/user/appdata/unraid_scripts/Master.conf
# Shared functions at: /mnt/user/appdata/unraid_scripts/common.sh
#
# ==============================================================================================
# Changelog:
# v1.0 — Initial template
# v1.1 — MAX_RSYNC_PROCS removed, --status flag, full directory tree, changelog
# v1.2 — Docker Essentials, Media, Transcodes, System Watchdog added
# v1.3 — Failover script added
# v1.4 — WebGUI watchdog, ZFS snapshot, Docker network connect, schedules section
# v1.5 — media_management.sh orchestrator, corrected filenames, git repo section
# v1.6 — lidarr_cleanup.sh, sonarr_cleanup.sh, radarr_cleanup.sh added
# v1.7 — cert_monitor.sh added
# v1.8 — Monitors/ folder added with all monitoring scripts
# cert_monitor.sh moved from unRAID_Essentials to Monitors/
# backup_verify.sh, smart_health.sh, bandwidth_monitor.sh added
# weekly_health_digest.sh, emby_session_report.sh added
# ZFS memory snapshot moved to Monitors/ — informational only
# Directory tree updated to reflect full ecosystem
# v1.9 — transcode_management.sh added to Orchestrators/
# replaces separate transcode_manager and transcode_cleanup cron entries
# Tools/ folder expanded with all utility scripts
# Docker watchdog — two-tier monitoring, startup grace, dependency ordering
# Failover — tiered failover, DDNS management, handback sequence
# Emby Docker mount fix — bind-propagation=shared required in Extra Parameters
# transcode_cleanup.sh — transcoding-temp protected from directory deletion
# ramdisk_setup.sh — creates transcoding-temp on ramdisk at array start
# RAMDISK_SIZE bumped to 10G, thresholds updated
# ==============================================================================================
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# ━━━ 📂 Repository Structure ━━━
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
#
# /mnt/user/appdata/unraid_scripts/
# ├── Master.conf # All user configuration — edit this file only
# ├── common.sh # Shared library — functions used by all scripts
# ├── README.md # Project overview and quick start
# ├── user_script_plug-in.sh # This file — copy into User Scripts plugin
# ├── git_pull_execute.sh # Pulls latest scripts from Gitea repo
# │
# ├── Failover/
# │ ├── failover.sh # Mutual container failover — runs continuously
# │ ├── failover_test.sh # Controlled failover simulation — run manually
# │ └── README-Failover.md
# │
# ├── Monitors/
# │ ├── backup_verify.sh # Random sample checksum verification vs remote
# │ ├── bandwidth_monitor.sh # Daily transfer logging + weekly summary
# │ ├── cert_monitor.sh # SSL certificate expiry — direct openssl check
# │ ├── emby_session_report.sh # Weekly Emby usage statistics via API
# │ ├── smart_health.sh # Drive SMART attribute monitoring
# │ ├── weekly_health_digest.sh # Aggregated system health — always/smart/weekly
# │ ├── zfs_memory_snapshot.sh # Weekly ZFS health and memory diagnostic report
# │ └── README-Monitors.md
# │
# ├── Orchestrators/
# │ ├── array_start.sh # Single entry point — launches all array-start scripts
# │ ├── daily_sync_maintenance.sh # Daily — git pull, media sync, media mgmt, docker restart
# │ ├── weekly_sync_maintenance.sh # Weekly — critical sync + updates, docker weekly restart
# │ ├── daily_sync_maintenance.sh # Media shares sync both directions
# │ ├── weekly_sync_maintenance.sh # Clean sync + container updates (Emby + auth stack)
# │ ├── media_management.sh # Permissions + cleaners + arr cleanup — run manually
# │ ├── transcode_management.sh # Cleanup then manager every 3min + daily stats
# │ └── README-Orchestrators.md
# │
# ├── Rsync/
# │ ├── rsync.sh # Core rsync script — called per share or profile
# │ └── README-Rsync_Setup.md # Rsync-specific setup guide
# │
# ├── Docker_Essentials/
# │ ├── docker_watchdog.sh # Two-tier container health monitor — self-healing
# │ ├── docker_daily_restart.sh # Restarts configured containers daily
# │ ├── docker_weekly_restart.sh # Restarts configured containers weekly
# │ ├── docker_network_connect.sh # Connects containers to extra networks on boot
# │ └── README-Docker_Essentials.md
# │
# ├── Media/
# │ ├── media_shares_permissions.sh # Applies permissions to all media shares
# │ ├── media_cleaner.sh # Removes junk files — profiles: anime, media
# │ ├── lidarr_cleanup.sh # Removes orphaned music files via Lidarr API
# │ ├── sonarr_cleanup.sh # Removes orphaned TV files via Sonarr API
# │ ├── radarr_cleanup.sh # Removes orphaned movie files via Radarr API
# │ └── README-Media.md
# │
# ├── Transcodes/
# │ ├── ramdisk_setup.sh # Creates ramdisk, symlink, and transcoding-temp
# │ ├── transcode_manager.sh # Monitors ramdisk usage, manages symlink
# │ ├── transcode_cleanup.sh # Removes old inactive transcode files
# │ └── README-Transcoding.md
# │
# ├── Tools/
# │ ├── bulk_permissions_repair.sh # Targeted permission repair for one share
# │ ├── container_data_export.sh # Export container appdata to tar archive
# │ ├── emby_database_repair.sh # SQLite integrity check on Emby databases
# │ ├── failover_state_reset.sh # Reset failover state file to NORMAL
# │ ├── recreate_shares.sh # Recreates share dirs from .cfg files after incident
# │ ├── watchdog_skip_list_manager.sh # Manage container watchdog skip lists
# │ ├── zfs_pool_scrub.sh # Trigger ZFS scrub with completion report
# │ └── README-Tools.md
# │
# └── unRAID_Essentials/
# ├── clear_logs.sh # Clears unRAID system and Docker log files
# ├── docker_syslog_filter.sh # Filters Docker veth noise from syslog
# ├── mover_stop.sh # Safely stops the unRAID mover
# ├── php_fpm_max_children.sh # Sets PHP-FPM max children value
# ├── rsync_stop.sh # Stops all rsync processes on both servers
# ├── server_reboot.sh # Graceful server reboot with user warning
# ├── system_watchdog.sh # System health monitor — last line of defense
# ├── user_scripts_stop.sh # Stops running User Scripts plugin jobs
# ├── webgui_restart.sh # WebGUI watchdog — nginx + emhttp restart
# └── README-Unraid_Essentials.md
#
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# ━━━ ⚠️ Critical Setup Notes ━━━
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
#
# EMBY TRANSCODE DOCKER MOUNT — MUST USE EXTRA PARAMETERS:
# Do NOT add /mnt/ram-transcode as a standard path mapping in the Emby template.
# Standard bind mounts use rprivate propagation — Docker locks the mount inode on
# the first symlink flip and new sessions land on SSD permanently for that run.
#
# In Emby Extra Parameters add:
# --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared
#
# shared propagation means host symlink changes are visible inside the container
# in real time — symlink flips work correctly for the lifetime of the container.
#
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# ━━━ 🚀 Script Commands — Uncomment the one you want to run ━━━
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
#
# ━━━ Failover ━━━
# Runs continuously as background task — set schedule to "At Startup of Array (Background)".
# Both servers must have this running for mutual failover to work.
#
#/mnt/user/appdata/unraid_scripts/Failover/failover.sh
#
# ━━━ Failover Test ━━━
# Run manually during a maintenance window — starts and stops real containers.
# Always --dry-run first to walk through phases without making changes.
#
#/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh --dry-run
#/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh
#
# ━━━ Monitors ━━━
#/mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh
#/mnt/user/appdata/unraid_scripts/Monitors/bandwidth_monitor.sh --report
#/mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh
#/mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh
#/mnt/user/appdata/unraid_scripts/Monitors/smart_health.sh
#/mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh
#/mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh
#
# ━━━ Orchestrators ━━━
# array_start.sh is the single User Scripts entry for array startup.
# All other orchestrators are scheduled via cron — not started at array start.
#
#/mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh
# ^^ set to: At Startup of Array
#/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh
#/mnt/user/appdata/unraid_scripts/Orchestrators/weekly_sync_maintenance.sh
#/mnt/user/appdata/unraid_scripts/Orchestrators/transcode_management.sh
# ^^ media_management.sh is absorbed into daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS
# ^^ run manually: bash Orchestrators/media_management.sh --dry-run
#
# ━━━ Rsync — Appdata Profiles ━━━
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Critical-Data
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Important-Data
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Media_Server/Emby --profile=emby-failover
# ^^ schedule every 30-60min — dirty sync, Emby running, critical data only
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Media_Server/Emby
# ^^ do NOT schedule — called by weekly_sync_maintenance.sh Sunday 2:30am only
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Gmer4Lfe
#
# ━━━ Rsync — Individual Media Shares (ad hoc) ━━━
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Movies
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Tv_Shows
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Music
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Music_Videos
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Shows
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Shows-Old
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Movies
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Movies-Old
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Books
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Intros
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Kids_Movies
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Kids_Tv_Shows
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Nextcloud
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Sports
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/stand-up_comedy
#
# ━━━ Docker Essentials ━━━
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_watchdog.sh
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_daily_restart.sh
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh
#
# ━━━ Media ━━━
# media_management.sh is absorbed into daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS.
# Scripts below available for manual runs only — always --dry-run first on arr cleanup scripts.
#
#/mnt/user/appdata/unraid_scripts/Media/media_shares_permissions.sh
#/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh anime --dry-run
#/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh anime
#/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh media --dry-run
#/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh media
#/mnt/user/appdata/unraid_scripts/Media/lidarr_cleanup.sh --dry-run --log
#/mnt/user/appdata/unraid_scripts/Media/lidarr_cleanup.sh
#/mnt/user/appdata/unraid_scripts/Media/sonarr_cleanup.sh --dry-run --log
#/mnt/user/appdata/unraid_scripts/Media/sonarr_cleanup.sh
#/mnt/user/appdata/unraid_scripts/Media/radarr_cleanup.sh --dry-run --log
#/mnt/user/appdata/unraid_scripts/Media/radarr_cleanup.sh
#/mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh --dry-run
#/mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh
# ^^ schedule: 0 5 * * * (5am daily) — always dry-run first
#
# ━━━ Transcodes ━━━
# transcode_management.sh runs cleanup then manager — schedule that, not the individuals.
# ramdisk_setup.sh runs once at array start.
# transcode_manager.sh and transcode_cleanup.sh available for individual manual runs.
#
#/mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh
#/mnt/user/appdata/unraid_scripts/Orchestrators/transcode_management.sh
#/mnt/user/appdata/unraid_scripts/Transcodes/transcode_manager.sh --dry-run
#/mnt/user/appdata/unraid_scripts/Transcodes/transcode_manager.sh
#/mnt/user/appdata/unraid_scripts/Transcodes/transcode_cleanup.sh --dry-run
#/mnt/user/appdata/unraid_scripts/Transcodes/transcode_cleanup.sh
#
# ━━━ Tools ━━━
#/mnt/user/appdata/unraid_scripts/Tools/bulk_permissions_repair.sh /mnt/user/ShareName
#/mnt/user/appdata/unraid_scripts/Tools/container_data_export.sh ContainerName /path/to/appdata /path/to/output
#/mnt/user/appdata/unraid_scripts/Tools/emby_database_repair.sh
#/mnt/user/appdata/unraid_scripts/Tools/failover_state_reset.sh --status
#/mnt/user/appdata/unraid_scripts/Tools/failover_state_reset.sh
#/mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh
#/mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --status
#/mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --clear ContainerName
#/mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --clear-all
#/mnt/user/appdata/unraid_scripts/Tools/zfs_pool_scrub.sh
#
# ━━━ unRAID Essentials ━━━
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/clear_logs.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/docker_syslog_filter.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/mover_stop.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/php_fpm_max_children.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/rsync_stop.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/server_reboot.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/system_watchdog.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/user_scripts_stop.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/webgui_restart.sh
#
# ━━━ Git repo (Gitea) ━━━
#/mnt/user/appdata/unraid_scripts/git_pull_execute.sh
#
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# ━━━ ⚙️ Arguments ━━━
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
#
# --dry-run Preview what would happen — no changes made
# --log Enable verbose logging output
# --no-log Disable logging (overrides Master.conf)
# --status Print resolved config and current state then exit
# --help Show usage information
# KEY=VALUE Override any Master.conf variable for this run only
#
# ━━━ Special arguments ━━━
# failover.sh --status — check current state without restarting loop
# failover.sh --dry-run --log — test logic without touching containers
# media_cleaner.sh anime --dry-run — profile required as first argument
# bandwidth_monitor.sh --report — generate weekly summary report
# lidarr/sonarr/radarr --dry-run --log — always test arr cleanup before running live
# transcode_management.sh --dry-run — passes through to both child scripts
# watchdog_skip_list_manager.sh --status — view skip list and restart history
# watchdog_skip_list_manager.sh --clear Name — clear specific container
# failover_state_reset.sh --status — view current failover state file
#
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# ━━━ 📋 Recommended Schedules ━━━
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
#
# ━━━ At Startup of Array ━━━
# array_start.sh — single entry point, launches everything below
# configure what runs in ARRAY_START_SCRIPTS in Master.conf
#
# Launched by array_start.sh:
# ramdisk_setup.sh — creates ramdisk before Emby starts (one-shot)
# docker_syslog_filter.sh — suppress veth noise before logs fill (one-shot)
# php_fpm_max_children.sh — WebGUI performance tuning (one-shot)
# docker_network_connect.sh — connect containers to extra networks (one-shot)
# system_watchdog.sh — system health monitor (continuous)
# docker_watchdog.sh — container health monitor (continuous)
# failover.sh — mutual failover (continuous)
#
# ━━━ Frequent (cron) ━━━
# */3 * * * * transcode_management.sh (cleanup + manager every 3min)
# */30 * * * * rsync.sh /mnt/user/Media_Server/Emby (emby-failover dirty sync)
# --profile=emby-failover
# */6 * * * * arrs_failed_stalled_recovery.sh (blocklist + re-search)
# */15 * * * * system_watchdog.sh
#
# ━━━ Daily ━━━
# 0 1 * * * daily_sync_maintenance.sh (git pull + media sync + media_management + docker restart)
# 0 5 * * * arrs_failed_stalled_recovery.sh (blocklist + re-search failed imports + stalled)
# 0 8 * * * weekly_health_digest.sh (profile controls if it sends)
#
# ━━━ Rsync profiles — schedule individually ━━━
# Arrs_Stack — daily or every few days (arr databases change on every download)
# Critical-Data — handled by weekly_sync_maintenance.sh — no separate schedule needed
# Important-Data — daily (NextCloud file changes)
# Gmer4Lfe — daily or weekly (personal appdata, rarely changes)
# Emby — handled by weekly_sync_maintenance.sh — no separate schedule needed
# emby-failover — every 30-60min via frequent cron above
#
# ━━━ Weekly — Sunday morning block ━━━
# 30 2 * * 0 weekly_sync_maintenance.sh (clean Emby + auth stack — cache resets weekly)
# 0 3 * * 0 docker_weekly_restart.sh
# 0 5 * * 0 clear_logs.sh
# 0 6 * * 0 zfs_memory_snapshot.sh
# 0 7 * * 0 smart_health.sh
# 0 8 * * 0 weekly_health_digest.sh (weekly profile sends today)
# 0 9 * * 0 cert_monitor.sh
# 0 10 * * 0 backup_verify.sh
# 0 11 * * 0 emby_session_report.sh
# 0 11 * * 0 bandwidth_monitor.sh --report
#
# ━━━ Bandwidth monitor ━━━
# bandwidth_monitor.sh --log-transfer called automatically by rsync.sh
# bandwidth_monitor.sh --report runs standalone for weekly summary
#
# ==============================================================================================