From f1529db3a0ffafbfd1f0b3d3c0125b517e54b77f Mon Sep 17 00:00:00 2001 From: FailedProxy Date: Tue, 14 Apr 2026 17:11:49 -0400 Subject: [PATCH] did way to much,,,,,, mostly added monitors, but almost every file was edited in some way --- Master.conf | 635 ++++++++++++++++++++----------- Media/lidarr_cleanup.sh | 291 ++++++++++++++ Media/radarr_cleanup.sh | 0 Media/sonarr_cleanup.sh | 281 ++++++++++++++ Monitors/backup_verify.sh | 187 +++++++++ Monitors/bandwidth_monitor.sh | 152 ++++++++ Monitors/cert_monitor.sh | 182 +++++++++ Monitors/emby_session_report.sh | 0 Monitors/smart_health.sh | 249 ++++++++++++ Monitors/weekly_health_digest.sh | 245 ++++++++++++ common.sh | 95 +++-- user_script_pluin.sh | 142 ++++--- 12 files changed, 2154 insertions(+), 305 deletions(-) create mode 100644 Media/lidarr_cleanup.sh create mode 100644 Media/radarr_cleanup.sh create mode 100644 Media/sonarr_cleanup.sh create mode 100644 Monitors/backup_verify.sh create mode 100644 Monitors/bandwidth_monitor.sh create mode 100644 Monitors/cert_monitor.sh create mode 100644 Monitors/emby_session_report.sh create mode 100644 Monitors/smart_health.sh create mode 100644 Monitors/weekly_health_digest.sh diff --git a/Master.conf b/Master.conf index 86df17e..fcb9bd9 100644 --- a/Master.conf +++ b/Master.conf @@ -5,6 +5,13 @@ # All user-facing variables for the unRAID script ecosystem. # Scripts source this file — edit here, changes apply everywhere on next git pull. # +# ── HOW THIS FILE WORKS ─────────────────────────────────────────────────────────────────────── +# Every script sources Master.conf and common.sh at startup. +# Change a value here and it affects all scripts that use it — no hunting through files. +# To disable something: comment it out with # rather than deleting it. +# To add a new rsync profile: add a key to each PROFILE_* array. +# To add a new media maintenance job: add a line to MEDIA_MAINTENANCE_JOBS. +# # ── INDEX ───────────────────────────────────────────────────────────────────────────────────── # # Section Description @@ -36,47 +43,70 @@ # PHP-FPM PHP-FPM max children config # CLEAR LOGS System log file paths # WEBGUI WATCHDOG WebGUI nginx + emhttp monitoring and restart -# ZFS MEMORY SNAPSHOT Weekly ZFS health and memory diagnostic report # # ── MEDIA ────────────────────────────────────────────────────────────────────────────────── # MEDIA PERMISSIONS Share list, mode and owner for permissions script # MEDIA CLEANER Anime and media folder lists and file patterns +# MEDIA MANAGEMENT Orchestrator job list for media_management.sh +# ARR CLEANUP Lidarr, Sonarr, Radarr orphan file cleanup # # ── TRANSCODES ───────────────────────────────────────────────────────────────────────────── # TRANSCODE MANAGER Ramdisk and SSD fallback transcode management # +# ── MONITOR ──────────────────────────────────────────────────────────────────────────────── +# CERTIFICATE MONITOR SSL certificate expiry monitoring +# BACKUP VERIFY Random sample checksum verification against remote +# SMART HEALTH Drive SMART attribute monitoring +# BANDWIDTH MONITOR Daily rsync transfer logging and weekly summary +# HEALTH DIGEST Aggregated system health digest — always/smart/weekly +# EMBY SESSION REPORT Weekly Emby usage statistics via API +# # ── SYSTEM WATCHDOG ──────────────────────────────────────────────────────────────────────── # SYSTEM WATCHDOG System health monitoring — last line of defense # # ============================================================================================== # ━━━ Host Configuration ━━━ -# Hostnames must match Tailscale machine names exactly — case sensitive +# Hostnames must match Tailscale machine names exactly — case sensitive. +# Used by detect_hosts() in common.sh to determine which server is local and which is remote. +# Both servers run identical scripts — host detection makes them bidirectional. HOST1="unRAID-Gmer4Lfe" HOST2="unRAID-Jayred365" -# SSH keys for server-to-server rsync and failover operations -# Each server authenticates with its own key — both must be authorised on the remote + +# SSH keys for server-to-server rsync and failover container operations. +# HOST1_SSH_KEY is used when HOST1 SSHes to HOST2 and vice versa. +# Both keys must be in /root/.ssh/ and authorised in the remote server's authorized_keys. HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" # ━━━ Logging ━━━ -# true = verbose [LOG] lines in all script output / false = user-facing output only +# Controls verbose [LOG] output across all scripts. +# true = show detailed [LOG] lines — useful for debugging or first-time setup +# false = show only user-facing output — cleaner for scheduled runs ENABLE_LOGGING=true # ━━━ Notifications ━━━ -# unRAID native — fires through Settings → Notification Settings -# Recommended: configure unRAID to send errors/warnings only so normal completions stay quiet +# Two independent notification channels — either or both can be active simultaneously. + +# unRAID native notification system — integrates with the bell icon in the WebGUI. +# Recommended: set Settings → Notification Settings to errors/warnings only so +# normal completions don't create noise. The ecosystem sends: +# normal — job completed successfully (informational) +# warning — something failed or needs attention NOTIFY_UNRAID=true -# Discord webhook — paste full webhook URL to enable, leave blank to disable + +# Discord webhook URL — paste the full webhook URL from your Discord server settings. +# Leave blank to disable Discord notifications entirely. DISCORD_WEBHOOK="" # ━━━ Git / Repo ━━━ -# Gitea self-hosted repository for the script ecosystem -# git_pull_execute.sh uses these to clone or pull the latest version on both servers +# Gitea self-hosted repository settings used by git_pull_execute.sh. +# Running git_pull_execute.sh on either server pulls the latest scripts and sets +# executable permissions automatically — keeps both servers in sync with one command. REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git" TARGET_DIR="/mnt/user/appdata/unraid_scripts" - GITEA_SSH_KEY="/root/.ssh/unraid_gitea" - SSH_PORT=221 + GITEA_SSH_KEY="/root/.ssh/unraid_gitea" # SSH key for authenticating to Gitea + SSH_PORT=221 # Gitea SSH port — default Gitea uses 22 # ============================================================================================== # ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── @@ -86,39 +116,35 @@ # Global fallback values used when no profile match is found for a directory. # Shares in DAILY_SYNC_SHARES always use these globals — no profile is defined for them. # Appdata shares (Arrs_Stack, Critical-Data etc.) match profiles by directory basename. +# If a profile key exists in a PROFILE_* array that value overrides the global. +# If a profile key is missing the global below is used as the fallback. -# Network transfer speed cap in KB/s — 12500 ≈ 100Mbit - BW_LIMIT=12500 -# Number of retry attempts if rsync fails before giving up - RETRY_COUNT=3 -# Seconds to wait between retry attempts - SLEEP=300 -# Containers to stop on remote before rsync and restart after — empty by default -# Profiles below override this per appdata share - CRITICAL_CONTAINER_NAMES=() -# Containers that need a delay before starting — e.g. Authelia needs its DB ready first - DELAYED_CONTAINERS=() -# Seconds to wait before starting delayed containers - CONTAINER_DELAY=5 -# Directories to exclude from transfer — empty by default, profiles override per share - EXCLUDE_DIRS=() -# Default rsync options used when no profile match is found -# --delete removes files on remote that no longer exist on source -# --inplace writes directly to destination file rather than temp file — better for large files -# --no-whole-file forces delta transfer even on local-like connections + BW_LIMIT=12500 # network transfer speed cap in KB/s — 12500 ≈ 100Mbit + RETRY_COUNT=3 # number of retry attempts if rsync fails before giving up + SLEEP=300 # seconds to wait between retry attempts + CRITICAL_CONTAINER_NAMES=() # containers to stop on REMOTE before rsync — profiles override + DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override + CONTAINER_DELAY=5 # seconds to wait before starting delayed containers + EXCLUDE_DIRS=() # directories to exclude from transfer — profiles override + +# Default rsync options used when no profile match is found. +# --delete removes files on remote that no longer exist on source (mirror behaviour) +# --inplace writes directly to destination file — better for large files, avoids temp copies +# --no-whole-file forces delta transfer even on fast local-like connections DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file) # ━━━ Remote Health Checks ━━━ -# Abort rsync if remote rootfs (/) usage is at or above this percentage. -# When the remote array is down or drives are missing, rsync writes land on rootfs instead -# of /mnt/user — this fills the filesystem rapidly and can crash the remote server. -# 75% gives enough headroom to detect the problem before it becomes critical. +# Pre-flight check run before every rsync — aborts if remote rootfs (/) usage is at or +# above this percentage. When the remote array is down or drives are missing, rsync +# writes land on rootfs instead of /mnt/user — this fills the filesystem rapidly and +# can crash the remote server. 75% gives headroom to detect the problem early. ROOTFS_WARN=75 # ━━━ Daily Sync Shares ━━━ -# Media shares synced once daily by Orchestrators/daily_sync.sh -# These shares have no profile — all use DEFAULT_RSYNC_OPTS above -# Add or remove paths to control what gets synced each night +# Media shares synced once daily by Orchestrators/daily_sync.sh. +# These shares have no profile entry — all use DEFAULT_RSYNC_OPTS above. +# Add or remove paths here to control what syncs each night. +# For shares needing custom bandwidth or container stops — create a profile below instead. DAILY_SYNC_SHARES=( /mnt/user/Anime_Movies-Old /mnt/user/Anime_Shows-Old @@ -136,36 +162,38 @@ DAILY_SYNC_SHARES=( # ━━━ Rsync Profile System ━━━ # Profiles allow per-share rsync behaviour without touching script logic. -# Each profile is matched automatically by the basename of the directory passed to rsync.sh -# (lowercased). Example: /mnt/user/appdata-Failover/Arrs_Stack → profile key = arrs_stack +# The profile key is matched automatically by the basename of the directory +# passed to rsync.sh (lowercased). # -# How fallthrough works: -# If a key exists in a profile array → that value is used for this run -# If a key is missing → the global default above is used instead -# Shares in DAILY_SYNC_SHARES → always use globals, no profile defined +# Example: +# rsync.sh /mnt/user/appdata-Failover/Arrs_Stack +# basename = Arrs_Stack → lowercased = arrs_stack → matches [arrs_stack] profile # # To add a new profile: -# 1. Add a key to each array below with your chosen profile name +# 1. Add a key to each PROFILE_* array below using your chosen name # 2. Call rsync.sh with a directory whose basename matches that key # 3. Any array you omit falls back to its global default automatically # -# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit from DEFAULT_RSYNC_OPTS — -# if you define a profile entry you must list all desired options explicitly +# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit from DEFAULT_RSYNC_OPTS. +# If you define a profile entry you must list ALL desired options explicitly. # -# Profile descriptions: -# arrs_stack — Sonarr, Radarr, Lidarr, Readarr, Prowlarr, Bazarr, Pinchflat appdata -# Lower bandwidth — runs alongside media syncs, containers stopped during sync -# critical-data — Auth stack: NPM, Authelia, Mariadb, Redis, LLDAP -# High bandwidth — small data, synced frequently, Authelia needs delayed start +# Current profiles: +# arrs_stack — Sonarr, Radarr, Lidarr, Readarr, Prowlarr, Bazarr, Pinchflat +# Lower bandwidth — runs alongside media syncs +# Containers stopped during sync for data consistency +# critical-data — Auth stack: NPM, Authelia, Mariadb-Authelia, Redis-Authelia, LLDAP +# High bandwidth — small data, synced frequently +# Authelia needs delayed start — database containers must be ready first # gmer4lfe — Server-specific appdata: Organizr, UptimeKuma, VaultWarden # Medium bandwidth — personal services, no container stop needed # important-data — NextCloud + Postgres database # High bandwidth — NextCloud needs graceful stop before sync -# emby — Emby media server appdata and metadata -# Medium bandwidth — large appdata, no containers stopped (metadata only) +# NextCloud needs delayed start — Postgres must be accepting connections +# emby — Emby media server appdata and metadata only +# Medium bandwidth — large appdata directory, no containers stopped -# SPACE-SEPARATED STRINGS — rsync options per profile -# If defined for a profile, these replace DEFAULT_RSYNC_OPTS entirely for that run +# Rsync options per profile — replaces DEFAULT_RSYNC_OPTS entirely for that profile run +# SPACE-SEPARATED STRINGS — converted to array at runtime by rsync.sh declare -A PROFILE_RSYNC_OPTS=( [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete" @@ -174,16 +202,16 @@ declare -A PROFILE_RSYNC_OPTS=( [emby]="-av --human-readable --bwlimit=$BW_LIMIT" ) -# Bandwidth limit in KB/s per profile — overrides global BW_LIMIT for this profile +# Bandwidth limit in KB/s per profile — overrides global BW_LIMIT for this profile only declare -A PROFILE_BW_LIMIT=( - [arrs_stack]=5000 # lower — runs alongside other jobs - [critical-data]=9500 # high — small data, sync fast + [arrs_stack]=5000 # lower — runs alongside other jobs, avoids saturating link + [critical-data]=9500 # high — small data, get it synced fast [gmer4lfe]=8000 # medium - [important-data]=9500 # high — database sync, prioritise speed - [emby]=8000 # medium — large files, steady transfer + [important-data]=9500 # high — database sync needs to be fast and clean + [emby]=8000 # medium — large files, steady sustained transfer ) -# Retry attempts per profile — overrides global RETRY_COUNT +# Retry attempts per profile before giving up — overrides global RETRY_COUNT declare -A PROFILE_RETRY_COUNT=( [arrs_stack]=3 [critical-data]=3 @@ -192,7 +220,7 @@ declare -A PROFILE_RETRY_COUNT=( [emby]=3 ) -# Sleep between retries in seconds — overrides global SLEEP +# Seconds between retry attempts — overrides global SLEEP declare -A PROFILE_SLEEP=( [arrs_stack]=300 [critical-data]=300 @@ -201,9 +229,10 @@ declare -A PROFILE_SLEEP=( [emby]=300 ) -# Containers stopped on REMOTE before rsync and restarted after — SPACE-SEPARATED STRINGS -# Only containers that need to be stopped for data consistency — running containers are fine -# for most media but databases and auth stacks need clean state during sync +# Containers stopped on REMOTE before rsync and restarted after completion. +# Only include containers that need to be stopped for data consistency. +# Databases and auth stacks need clean state — media servers generally do not. +# SPACE-SEPARATED STRINGS — converted to array at runtime declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" [critical-data]="Mariadb-Authelia Redis-Authelia Lldap-Gmer4Lfe NginxProxyManager Authelia" @@ -212,28 +241,30 @@ declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( [emby]="" ) -# Containers that need a delay before starting after rsync — SPACE-SEPARATED STRINGS -# Authelia needs its database containers (Mariadb, Redis) to be ready before it starts -# NextCloud needs Postgres to be accepting connections before it starts +# Containers that need a delay before starting after rsync completes. +# Used when a container depends on another that was also stopped — it needs its +# dependency to be ready before it can start successfully. +# SPACE-SEPARATED STRINGS — converted to array at runtime declare -A PROFILE_DELAYED_CONTAINERS=( [arrs_stack]="" - [critical-data]="Authelia" + [critical-data]="Authelia" # Authelia needs Mariadb + Redis ready before starting [gmer4lfe]="" - [important-data]="NextCloud" + [important-data]="NextCloud" # NextCloud needs Postgres accepting connections first [emby]="" ) -# Seconds to wait before starting delayed containers — per profile +# Seconds to wait before starting delayed containers — gives dependencies time to initialise declare -A PROFILE_CONTAINER_DELAY=( [arrs_stack]=5 - [critical-data]=10 # Authelia needs DB ready — 10s gives Mariadb/Redis time to start + [critical-data]=10 # 10s gives Mariadb and Redis time to accept connections [gmer4lfe]=5 - [important-data]=10 # NextCloud needs Postgres ready + [important-data]=10 # 10s gives Postgres time to accept connections [emby]=5 ) -# Directories excluded from transfer per profile — SPACE-SEPARATED STRINGS -# logs and *.tmp are excluded universally — they are ephemeral and regenerated on start +# Directories excluded from rsync transfer per profile. +# logs and *.tmp are safe to exclude — they are ephemeral and regenerated on container start. +# SPACE-SEPARATED STRINGS — converted to array at runtime declare -A PROFILE_EXCLUDE_DIRS=( [arrs_stack]="logs *.tmp" [critical-data]="logs *.tmp" @@ -242,75 +273,78 @@ declare -A PROFILE_EXCLUDE_DIRS=( [emby]="logs *.tmp" ) -# ━━━ Profile Disk Check Toggle ━━━ -# true = skip per-disk check for this profile — use when share lives on a ZFS pool -# false = run per-disk check — use for traditional unRAID array with individual disks -# Per-disk check looks for /mnt/disk*/sharename — ZFS pools don't have this structure -# The share existence and content checks still run regardless of this setting +# Per-disk check toggle — controls whether rsync.sh runs check_remote_disks() for this profile. +# true = skip per-disk check — use when remote share lives on a ZFS pool +# ZFS pools don't have /mnt/disk* structure so the check always fails incorrectly +# false = run per-disk check — use for traditional unRAID array with individual disk mounts +# Verifies all disks backing the share are online before syncing +# Note: rootfs and share existence checks always run regardless of this setting declare -A PROFILE_SKIP_DISK_CHECK=( - [arrs_stack]=true - [critical-data]=true - [gmer4lfe]=true - [important-data]=true - [emby]=true + [arrs_stack]=true # remote uses ZFS pool — no individual disk mounts + [critical-data]=true + [gmer4lfe]=true + [important-data]=true + [emby]=true ) + # ============================================================================================== # ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Mutual container failover between two unRAID servers 50 miles apart. -# Each server runs failover.sh independently — no coordination between servers. +# Each server runs Failover/failover.sh independently — no coordination between servers. # All decisions are based solely on two ping checks: remote reachable + internet reachable. # +# How it works: +# Every FAILOVER_CHECK_INTERVAL seconds each server pings the remote and pings the internet. +# Based on those two results it determines its state and takes the appropriate action. +# No SSH, no signaling — each server acts autonomously based only on what it can see. +# # States: # NORMAL — remote up, internet up — own containers only, silent operation # FAILOVER — remote down, internet up — start remote's containers locally (additive) -# NO_INTERNET — internet down — stop public-facing containers, wait -# DARK — remote down + internet down — same as NO_INTERNET +# Own normal containers keep running — failover containers added on top +# NO_INTERNET — internet down — stop public-facing containers, wait for recovery +# DARK — remote down + internet down — same actions as NO_INTERNET # # Handback sequence when remote returns after FAILOVER: -# Strike confirmation (FAILOVER_HANDBACK_STRIKES consecutive remote-up checks) -# → pre-flight checks (remote array, docker daemon, rootfs) -# → rsync data back via rsync.sh (uses profile system) -# → start failover containers on remote via SSH -# → stop failover containers locally -# → return to NORMAL +# 1. Strike confirmation — FAILOVER_HANDBACK_STRIKES consecutive remote-up checks +# Prevents handing back during a brief network blip +# 2. Pre-flight checks — remote array started, Docker daemon healthy, rootfs not full +# 3. Rsync data back via rsync.sh — uses existing profile system for options +# 4. Start failover containers on remote via SSH +# 5. Stop failover containers locally — only after remote confirmed started +# 6. Return to NORMAL state # -# Comment out any container or rsync job to disable without removing the entry. # Script runs on BOTH servers — detect_hosts() selects the correct arrays automatically. +# Comment out any container or rsync job to disable without removing the entry. -# External IP to ping for internet connectivity check — Google DNS, reliable and fast - EXTERNAL_IP="8.8.8.8" -# Seconds between state checks — 120s = 2 minute polling interval -# With 1 minute DNS TTL this means failover is visible to users within ~3 minutes - FAILOVER_CHECK_INTERVAL=120 -# Consecutive remote-up confirmations required before initiating handback -# Prevents handing back during a brief network blip — 2 strikes = 4 minutes confirmation - FAILOVER_HANDBACK_STRIKES=2 -# State file path — on /boot/ so it survives reboots -# Script re-evaluates from scratch on restart using live pings — state file is reference only + EXTERNAL_IP="8.8.8.8" # external IP to ping for internet connectivity check + FAILOVER_CHECK_INTERVAL=120 # seconds between state checks — 120s = 2 minute polling + FAILOVER_HANDBACK_STRIKES=2 # consecutive remote-up confirmations required before handback + # 2 strikes at 120s interval = 4 minutes confirmation window FAILOVER_STATE_FILE="/boot/config/failover_state.db" + # persists on /boot/ so it survives reboots + # script re-evaluates from live pings on restart # ━━━ HOST1 Failover Config (unRAID-Gmer4Lfe — Primary) ━━━ -# Containers HOST1 starts locally when HOST2 goes down -# These run on top of HOST1's normal containers — additive, not replacement -# HOST2's DDNS containers must start here so DNS stays pointing at HOST1 during HOST2 outage +# Containers HOST1 starts locally when HOST2 goes down. +# These run ON TOP OF HOST1's normal containers — additive, not a replacement. FAILOVER_HOST1_STARTS_FOR_HOST2=( "Gmer4Lfe.com" "Gmer4Lfe.us" ) -# Containers HOST1 stops when it loses internet -# No point serving DDNS or media if HOST1 itself has no internet — stops unnecessary churn -# HOST2 will independently detect HOST1 is gone and start its own failover +# Containers HOST1 stops when it loses internet connectivity. +# No point serving DDNS or public services if HOST1 itself has no internet. FAILOVER_HOST1_STOP_ON_NO_NET=( "Gmer4Lfe.com" "Gmer4Lfe.us" ) -# Rsync jobs HOST1 runs before handing containers back to HOST2 -# Uses rsync.sh with existing profile system — basename matched to profile keys -# Comment out jobs that don't need syncing back or aren't ready yet +# Rsync jobs HOST1 runs before handing containers back to HOST2 after recovery. +# Uses rsync.sh with the existing profile system — basename matched to profile keys. +# Comment out jobs that are not yet ready or not needed for handback. FAILOVER_HOST1_RSYNC_JOBS=( # "/mnt/user/appdata-Failover/Jayred365" # "/mnt/user/Media_Server/Emby-Jayred" @@ -318,24 +352,22 @@ FAILOVER_HOST1_RSYNC_JOBS=( # ━━━ HOST2 Failover Config (unRAID-Jayred365 — Secondary) ━━━ -# Containers HOST2 starts locally when HOST1 goes down -# Emby starts on HOST2 so media keeps working during HOST1 outage -# HOST1's DDNS containers start here so DNS updates to point at HOST2 +# Containers HOST2 starts locally when HOST1 goes down. +# Emby starts on HOST2 so media keeps working during HOST1 outage. +# HOST1's DDNS containers start here so DNS updates to point at HOST2's IP. FAILOVER_HOST2_STARTS_FOR_HOST1=( "Emby" "Gmer4Lfe.com" "Gmer4Lfe.us" ) -# Containers HOST2 stops when it loses internet -# HOST2's DDNS containers serve no purpose without internet connectivity +# Containers HOST2 stops when it loses internet connectivity. FAILOVER_HOST2_STOP_ON_NO_NET=( "Gmer4Lfe.com" "Gmer4Lfe.us" ) -# Rsync jobs HOST2 runs before handing containers back to HOST1 -# Comment out jobs that don't need syncing back or aren't ready yet +# Rsync jobs HOST2 runs before handing containers back to HOST1 after recovery. FAILOVER_HOST2_RSYNC_JOBS=( # "/mnt/user/appdata-Failover/Gmer4Lfe" ) @@ -345,8 +377,9 @@ FAILOVER_HOST2_RSYNC_JOBS=( # ============================================================================================== # ━━━ Docker Daily Restart ━━━ -# Containers restarted every day — keeps services fresh, clears memory leaks -# Case-sensitive — must match exact Docker container names +# Containers restarted every day by Docker_Essentials/docker_daily_restart.sh. +# Keeps services fresh and clears memory leaks that accumulate over time. +# Case-sensitive — must match exact Docker container names shown in the unRAID Docker tab. DAILY_RESTART_CONTAINERS=( "NginxProxyManager" "Authelia" @@ -357,7 +390,8 @@ DAILY_RESTART_CONTAINERS=( ) # ━━━ Docker Weekly Restart ━━━ -# Containers restarted once per week — less critical services that benefit from periodic restart +# Containers restarted once per week by Docker_Essentials/docker_weekly_restart.sh. +# For less critical services that benefit from periodic restart but don't need daily cycling. WEEKLY_RESTART_CONTAINERS=( "NextCloud" "Organizrv2-Gmer4Lfe" @@ -366,30 +400,33 @@ WEEKLY_RESTART_CONTAINERS=( ) # ━━━ Docker Watchdog ━━━ -# First line of defense — monitors containers for memory, CPU and HTTP responsiveness. -# Restarts containers that exceed thresholds using a strike system to avoid false positives. -# Works alongside system_watchdog.sh — containers first, system reboot second. +# First line of defense for container health — runs every 15 minutes via cron. +# Monitors memory usage, CPU usage and HTTP responsiveness per container. +# Uses a strike system to avoid restarting on brief spikes — sustained issues trigger restart. +# Works alongside system_watchdog.sh — containers first, system reboot is the last resort. -# Containers to monitor with their memory hard limits in MB -# Strike system used for CPU and responsiveness — immediate restart for memory hard limit +# Containers to monitor with their memory hard limits in MB. +# Memory hard limit exceeded → immediate restart (no strike system for memory). +# CPU and HTTP use strike system — see CPU_FAIL_LIMIT and RESP_FAIL_LIMIT below. # 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240 # 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024 declare -A WATCHDOG_CONTAINERS=( - ["Emby"]=16384 # 16GB — large media server, transcoding can spike + ["Emby"]=16384 # 16GB — media server, transcoding can spike high ["LidaTube"]=6144 # 6GB — YouTube downloader ["Tdarr"]=6144 # 6GB — transcoding node ["Code-Server"]=1024 # 1GB — VS Code server ) -# Containers to check HTTP responsiveness — curl check per interval -# Omit a container entirely to skip its HTTP check +# Containers to check HTTP responsiveness via curl — omit a container to skip its HTTP check. +# curl checks the URL and considers the container unresponsive if it times out or errors. declare -A WATCHDOG_CONTAINER_URLS=( ["Emby"]="http://localhost:8096" ) -# Containers that should always be running — monitored for unexpected stops -# Strike system used — persistent skip list on /boot/ prevents reboot loops -# Auto-clears from skip list when container recovers after reboot or manual fix +# Containers that should always be running — monitored for unexpected stops. +# Strike system used — tries restart on each strike up to SYS_WATCHDOG_STRIKE_LIMIT. +# If restart fails after strike limit → added to persistent skip list on /boot/ +# Skip list auto-clears when container recovers after reboot or manual fix. WATCHDOG_REQUIRED_CONTAINERS=( "NginxProxyManager" "Lldap-Gmer4Lfe" @@ -400,33 +437,36 @@ WATCHDOG_REQUIRED_CONTAINERS=( "Redis-Authelia-Secondary" ) -# Strike state file — /tmp resets on reboot which is correct for strike tracking +# Strike state file — /tmp resets on reboot which is correct behaviour for strike tracking WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db" -# CPU thresholds — normalised against total core count at runtime +# CPU thresholds — normalised against total core count automatically at runtime. +# A container using 85% of one core on a 16-core system = ~5.3% normalised. SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU CPU_FAIL_LIMIT=2 # consecutive strikes before container restart -# Memory — warn at this % of per-container hard limit defined in WATCHDOG_CONTAINERS +# Memory soft threshold — warn when container reaches this % of its hard limit. +# Hard limit exceeded triggers immediate restart regardless of strikes. SOFT_MEM_THRESHOLD=80 -# HTTP responsiveness check +# HTTP responsiveness check settings RESP_FAIL_LIMIT=2 # consecutive failed curl checks before restart CURL_TIMEOUT=5 # seconds before curl gives up per check # ━━━ Docker Network Connect ━━━ -# Connects containers to extra Docker networks on array start — many-to-many -# Every container in the list connects to every network in the list -# Useful when containers need to communicate across networks they weren't configured with -# Comment out entries to disable without removing them +# Connects containers to extra Docker networks on array start. +# Useful when containers need to communicate across networks they were not originally +# configured with — e.g. memcached needing access to the nextcloud-aio network. +# Every container in the list connects to every network in the list (many-to-many). +# Comment out entries to disable without removing them. NETWORK_CONNECT_CONTAINERS=( "memcached" "Npm-CrowdSec" ) NETWORK_CONNECT_NETWORKS=( - "nextcloud-aio" + "nextcloud-aio" # Docker network name — must exist before array start ) # ============================================================================================== @@ -434,52 +474,56 @@ NETWORK_CONNECT_NETWORKS=( # ============================================================================================== # ━━━ Reboot ━━━ -# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots +# Seconds of warning broadcast to all logged-in users before server_reboot.sh reboots. +# Gives users time to save work or finish what they are doing before the system goes down. REBOOT_SLEEP=300 # ━━━ Mover ━━━ -# Seconds to wait after warning users before mover_stop.sh kills the mover process +# Seconds to wait after warning users before mover_stop.sh sends SIGTERM to the mover. +# Gives the mover time to finish its current file operation cleanly before being killed. MOVER_STOP_TIMEOUT=300 # ━━━ Syslog Filter ━━━ -# Path for the rsyslog filter file that suppresses Docker veth/docker0 noise +# Path for the rsyslog filter file created by docker_syslog_filter.sh. +# The filter suppresses noisy Docker veth and docker0 messages from syslog on boot. +# Without this filter, every Docker network interface change floods the syslog. FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" # ━━━ PHP-FPM ━━━ -# Config file path and max children value for php_fpm_max_children.sh +# Config file path and max children value for php_fpm_max_children.sh. +# Higher max_children allows more concurrent PHP requests to the unRAID WebGUI. +# Set based on available RAM — too high can cause memory pressure on low-RAM systems. PHP_CONF="/etc/php-fpm.d/www.conf" PHP_MAX_CHILDREN=250 # ━━━ Clear Logs ━━━ -# System log files cleared by clear_logs.sh — Docker logs cleared automatically too +# System log files cleared by clear_logs.sh — Docker container logs are cleared too. +# Run weekly to prevent logs from filling the rootfs over time. LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) # ━━━ WebGUI Watchdog ━━━ -# Monitors unRAID WebGUI and restarts services if unresponsive -# Escalation: nginx restart → recheck → emhttp restart → recheck → notify warning +# Monitors the unRAID WebGUI and restarts services if it becomes unresponsive. +# Escalation path: nginx restart → recheck → emhttp restart → recheck → notify warning. +# emhttp is the core unRAID management daemon — restarting it is more disruptive than nginx +# but both recover cleanly. Notification sent on any restart so you know what happened. WEBGUI_URL="http://localhost" # adjust port if non-standard e.g. http://localhost:8080 - WEBGUI_TIMEOUT=5 # seconds before curl gives up - WEBGUI_NGINX_WAIT=15 # seconds to wait after nginx restart before recheck - WEBGUI_EMHTTP_WAIT=30 # seconds to wait after emhttp restart before recheck - -# ━━━ ZFS Memory Snapshot ━━━ -# Weekly ZFS pool health and memory diagnostic report — informational only, no action taken -# system_watchdog.sh handles threshold-based intervention - ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log" - ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC utilization above this % - ZFS_REPORT_FREE_WARN_GB=10 # warn if free RAM below this GB - ZFS_REPORT_AVAIL_WARN_GB=20 # warn if available RAM below this GB - ZFS_REPORT_DOCKER_TOP=10 # number of top Docker memory users to show in report + WEBGUI_TIMEOUT=5 # seconds before curl gives up on the WebGUI check + WEBGUI_NGINX_WAIT=15 # seconds to wait after nginx restart before rechecking + WEBGUI_EMHTTP_WAIT=30 # seconds to wait after emhttp restart — emhttp takes longer # ============================================================================================== # ── MEDIA ───────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ Media Permissions ━━━ -# Mode and owner applied recursively to all shares in MEDIA_PERMISSION_SHARES +# Mode and owner applied recursively to all shares in MEDIA_PERMISSION_SHARES. +# Run by Media/media_shares_permissions.sh via the media_management.sh orchestrator. +# 777 and nobody:users is standard for unRAID media shares accessible by Docker containers. PERMISSIONS_MODE="777" PERMISSIONS_OWNER="nobody:users" +# Shares to apply permissions to — add or remove paths as your library grows. +# These are applied recursively so large shares take time — run overnight via orchestrator. MEDIA_PERMISSION_SHARES=( /mnt/user/Anime_Movies /mnt/user/Anime_Movies-Old @@ -506,10 +550,12 @@ MEDIA_PERMISSION_SHARES=( ) # ━━━ Media Cleaner ━━━ -# Two profiles: anime and media — passed as argument to media_cleaner.sh -# Run via Orchestrators/media_management.sh for permissions + both cleaners in one job -# Usage: media_cleaner.sh anime or media_cleaner.sh media +# Removes junk files from media shares using configurable file pattern lists. +# Two profiles: anime and media — each with their own folder list and patterns. +# Run via Media/media_cleaner.sh anime or Media/media_cleaner.sh media +# Called automatically by media_management.sh via MEDIA_MAINTENANCE_JOBS below. +# Folders scanned by the anime profile — anime downloads commonly include these junk files ANIME_CLEAN_FOLDERS=( /mnt/user/Anime_Movies /mnt/user/Anime_Movies-Old @@ -517,6 +563,7 @@ ANIME_CLEAN_FOLDERS=( /mnt/user/Anime_Shows-Old ) +# Folders scanned by the media profile MEDIA_CLEAN_FOLDERS=( /mnt/user/Kids_Movies /mnt/user/Kids_Tv_Shows @@ -527,7 +574,7 @@ MEDIA_CLEAN_FOLDERS=( /mnt/user/Tv_Shows ) -# Junk file patterns common in anime downloads +# File patterns deleted by the anime profile — common junk from anime download groups ANIME_FILE_PATTERNS=( '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' @@ -535,7 +582,7 @@ ANIME_FILE_PATTERNS=( '*.log' '*.json' ) -# Junk file patterns for general media — includes *.iso and *.lrc not needed in anime +# File patterns deleted by the media profile — includes *.iso and *.lrc not needed in anime MEDIA_FILE_PATTERNS=( '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' @@ -544,108 +591,252 @@ MEDIA_FILE_PATTERNS=( ) # ━━━ Media Management Orchestrator ━━━ -# Scripts run sequentially by Orchestrators/media_management.sh -# Order matters — permissions runs first so cleaner operates on correctly owned files -# Format: "script_path [optional_argument]" -# Add a new line to run another script — comment out to disable without removing +# Job list for Orchestrators/media_management.sh — runs scripts sequentially in order. +# Format: "folder/script.sh optional_argument" +# Order matters — permissions runs first so cleaners and arr scripts see correct ownership. +# Arr cleanup scripts run last — they depend on clean folders from the cleaner steps. +# Comment out any job to disable without removing it — easy to re-enable later. MEDIA_MAINTENANCE_JOBS=( - "Media/media_shares_permissions.sh" - "Media/media_cleaner.sh anime" - "Media/media_cleaner.sh media" + "Media/media_shares_permissions.sh" # apply permissions first + "Media/media_cleaner.sh anime" # remove junk from anime shares + "Media/media_cleaner.sh media" # remove junk from media shares + "Media/lidarr_cleanup.sh" # remove orphaned music files + "Media/sonarr_cleanup.sh" # remove orphaned TV files + "Media/radarr_cleanup.sh" # remove orphaned movie files ) +# ━━━ Arr Cleanup ━━━ +# Lidarr, Sonarr and Radarr orphan file cleanup via their respective APIs. +# Each arr script queries its API to get all tracked file paths, then compares against +# what exists on disk. Files not tracked by the arr and older than ORPHAN_AGE days are deleted. +# +# Why the age threshold matters: +# The arr downloads a file then processes it — there is a window where the file exists +# on disk but the arr hasn't imported it yet. ORPHAN_AGE prevents deleting files that +# are mid-import. 7 days is conservative and safe for any normal workflow. +# +# Protected patterns are NEVER deleted regardless of tracking status or age. +# These protect arr-generated metadata (cover art, .nfo files, subtitles) that the arr +# depends on but does not include in its tracked file API response. +# Add new patterns here if arr metadata formats change in future versions. + +# Lidarr — music library + LIDARR_URL="http://192.168.50.2:8686" + LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc" + LIDARR_MUSIC_ROOT="/mnt/user/Music-New" # must match the root path set in Lidarr + LIDARR_ORPHAN_AGE=7 # days before untracked file is eligible for deletion + LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") + # file extensions considered valid music files + LIDARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.lrc") + # never deleted — cover art, metadata, lyrics + +# Sonarr — TV library + SONARR_URL="http://192.168.50.2:8989" + SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f" + SONARR_TV_ROOT="/mnt/user/Tv_Shows" # must match the root path set in Sonarr + SONARR_ORPHAN_AGE=7 + SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") + SONARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa") + # never deleted — artwork, metadata, subtitles + +# Radarr — movie library + RADARR_URL="http://192.168.50.2:7878" + RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" + RADARR_MOVIES_ROOT="/mnt/user/Movies" # must match the root path set in Radarr + RADARR_ORPHAN_AGE=7 + RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") + RADARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa") + # ============================================================================================== # ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Session-based storage allocator using filesystem indirection. -# ffmpeg resolves the symlink path once at session start — existing sessions are never affected. -# New sessions land wherever TRANSCODE_LINK points at the moment they start. +# ffmpeg resolves the symlink path ONCE at session start — existing sessions are never affected +# by symlink changes. Only NEW sessions care about where the symlink currently points. # -# Flow: -# ramdisk_setup.sh — run once at array start, creates ramdisk and sets symlink -# transcode_manager.sh — every 3 min, monitors usage and flips symlink if needed +# How it works: +# ramdisk_setup.sh — run once at array start, creates tmpfs and sets symlink +# transcode_manager.sh — every 3 min, monitors ramdisk usage and flips symlink if needed # transcode_cleanup.sh — every 5 min, removes old inactive files from both locations # # Hysteresis gap between RAMDISK_WARN_GB and RAMDISK_LOW_GB prevents flip-flop -# when usage hovers near the threshold — gap should be at least 0.5-1GB +# when usage hovers near the threshold. Gap should be at least 0.5-1GB. +# Ramdisk is dynamic tmpfs — only uses RAM actually needed, RAMDISK_SIZE is the ceiling. - RAMDISK_PATH="/mnt/ramdisk_transcodes" # tmpfs mount point — created at array start - RAMDISK_SIZE="8G" # ceiling — tmpfs is dynamic, only uses what's needed - TRANSCODE_LINK="/mnt/ram-transcode" # symlink Emby points at — never changes location - TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" # SSD fallback for edge cases + RAMDISK_PATH="/mnt/ramdisk_transcodes" # tmpfs mount point created at array start + RAMDISK_SIZE="8G" # ceiling — tmpfs only uses RAM actually needed + TRANSCODE_LINK="/mnt/ram-transcode" # symlink Emby points at — location never changes + TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" # SSD fallback for edge cases only -# Thresholds in GB — flip symlink to SSD at WARN, flip back to ramdisk at LOW - RAMDISK_WARN_GB=6.8 # flip to SSD at or above this — "getting full" - RAMDISK_LOW_GB=5.5 # flip back to ramdisk when cleanup brings usage here - RAMDISK_SSD_MIN_GB=20 # minimum free GB on SSD before allowing flip — safety net +# Usage thresholds in GB + RAMDISK_WARN_GB=6.8 # flip symlink to SSD at or above this usage + RAMDISK_LOW_GB=5.5 # flip symlink back to ramdisk when usage drops here + RAMDISK_SSD_MIN_GB=20 # minimum free GB on SSD required before allowing flip to SSD -# Cleanup — files must be older than MAX_AGE and not open by any process to be deleted - TRANSCODE_MAX_AGE=20 # minutes before a file is eligible for cleanup - TRANSCODE_ORPHAN_AGE=30 # minutes before an orphaned file is eligible — extra caution +# Cleanup age thresholds — files must be older than these AND not open by any process + TRANSCODE_MAX_AGE=20 # minutes before a transcode file is eligible for cleanup + TRANSCODE_ORPHAN_AGE=30 # minutes before an orphaned file is eligible — extra caution buffer -# Flip frequency monitoring — notify if symlink flips too often (indicates sizing issue) +# Flip frequency alert — too many flips per hour may indicate ramdisk needs to be larger TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in one hour -# Permissions — must match your Emby container user +# Permissions applied to ramdisk and SSD fallback — must match your Emby container user TRANSCODE_OWNER="nobody:users" TRANSCODE_MODE="755" +# ============================================================================================== +# ── MONITOR ─────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Monitoring scripts — watch and report only, never take action. +# Lives in Monitor/ folder — distinct from unRAID_Essentials (which acts) and +# Docker_Essentials (which manages containers). +# These scripts are the data sources for the future plugin dashboard. + +# ━━━ Certificate Monitor ━━━ +# Checks SSL cert expiry via direct openssl connection — no NPM dependency. +# Reads the actual cert the server is presenting — catches real-world issues API checks miss. +# Each domain and subdomain is a separate entry — they have independent certs. +# Add your public-facing domains — uncomment and replace with your actual domains. +CERT_MONITOR_DOMAINS=( + # "yourdomain.com" + # "auth.yourdomain.com" + # "emby.yourdomain.com" + # "nextcloud.yourdomain.com" +) + CERT_WARN_DAYS=30 # notify warning when cert expires within this many days + CERT_CRIT_DAYS=7 # notify critical when cert expires within this many days + CERT_TIMEOUT=10 # seconds before openssl connection attempt gives up per domain + +# ━━━ Backup Verify ━━━ +# Verifies the rsync mirror is healthy by comparing random file checksums between servers. +# Uses existing SSH keys — no additional configuration needed beyond the share list. +# Leave BACKUP_VERIFY_SHARES empty to automatically use DAILY_SYNC_SHARES as the target list. +BACKUP_VERIFY_SHARES=( + # leave empty to use DAILY_SYNC_SHARES automatically + # or specify individual shares to verify: + # /mnt/user/Movies + # /mnt/user/Tv_Shows +) + BACKUP_VERIFY_SAMPLE=10 # number of files to randomly sample per share per run + BACKUP_VERIFY_MIN_SIZE=1M # skip files smaller than this — avoids tiny junk files + +# ━━━ SMART Health ━━━ +# Monitors drive SMART attributes — reads live from each drive, no persistent writes. +# Discovers all drives automatically via /dev/sd* and /dev/nvme* — no drive list needed. +# Add drives to SMART_IGNORE_DRIVES to skip specific drives (e.g. your unRAID boot USB). + SMART_TEMP_WARN=45 # degrees C — warn if drive temperature exceeds this + SMART_TEMP_CRIT=55 # degrees C — critical if drive temperature exceeds this +SMART_IGNORE_DRIVES=( + # "sda" # uncomment to ignore sda — common choice if sda is your unRAID boot USB +) + +# ━━━ Bandwidth Monitor ━━━ +# Logs daily rsync transfer totals to a bounded file on /boot/ — minimal flash wear. +# bandwidth_monitor.sh --log-transfer is called by rsync.sh after each successful sync. +# bandwidth_monitor.sh --report generates the weekly summary standalone. +# File stays bounded to BANDWIDTH_LOG_RETENTION lines — old entries auto-purged on each write. + BANDWIDTH_LOG="/boot/config/bandwidth_history.db" + BANDWIDTH_LOG_RETENTION=90 # days to keep — file never grows beyond ~90 lines + BANDWIDTH_WARN_GB=50 # flag in reports if a single sync transfer exceeds this GB + +# ━━━ Health Digest ━━━ +# Aggregated system health summary from across the ecosystem. +# Reads existing state files — no new writes to flash drive. +# +# Three profiles — switch by changing DIGEST_PROFILE, no cron changes needed: +# always — sends every run (schedule daily = daily digest, weekly = weekly digest) +# smart — sends only if findings worth reporting (intelligent quiet operation) +# weekly — sends once per week on DIGEST_DAY only, silent all other days +# +# Data sources (reads only — no writes): +# Transcode ramdisk state, container watchdog strikes, system watchdog strikes, +# failover state, container skip list, bandwidth history, SSL cert days remaining + DIGEST_PROFILE="weekly" # always | smart | weekly + DIGEST_DAY="Sunday" # day name for weekly profile — must match date +%A output + +# Smart profile triggers — set true to send digest when this condition is found + DIGEST_SMART_ON_WATCHDOG=true # send if any watchdog strikes are active + DIGEST_SMART_ON_FAILOVER=true # send if failover state is not NORMAL + DIGEST_SMART_ON_CERT_WARN=true # send if any cert is under CERT_WARN_DAYS + DIGEST_SMART_ON_BANDWIDTH=true # send if any transfer exceeded BANDWIDTH_WARN_GB + +# ━━━ Emby Session Report ━━━ +# Weekly Emby usage report via API — no persistent writes, queries fresh each run. +# Shows active streams, library counts, transcode vs direct play ratio. +# Requires an API key from Emby Settings → API Keys in the Emby WebUI. + EMBY_URL="http://localhost:8096" + EMBY_API_KEY="" # paste your Emby API key here + EMBY_REPORT_DAYS=7 # number of days to include in the report period + EMBY_REPORT_TOP_N=10 # number of top content items to show in report + # ============================================================================================== # ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── # ============================================================================================== -# Last line of defense — reboots cleanly if the system is about to become unstable. -# Works alongside docker_watchdog.sh — containers first, system reboot second. -# Thresholds set at "about to fall over" levels — not just high usage. -# Strike system prevents rebooting on single spikes — sustained threshold hits trigger action. +# Last line of defense — reboots the system cleanly when it is about to become unstable. +# Runs every 15 minutes via cron. Works alongside docker_watchdog.sh: +# docker_watchdog.sh — container level, minimal disruption, tries to self-heal first +# system_watchdog.sh — system level, last resort, reboots when healing has failed +# +# Strike system: sustained threshold hits trigger reboot — single spikes are ignored. +# Each check that exceeds its threshold adds a strike. Strikes reset when recovered. +# When strike limit is hit the reboot sequence begins. +# +# Reboot loop protection: tracks reboot timestamps on /boot/ (survives reboots). +# If the server reboots too many times in the window it shuts down instead — a reboot +# loop means something is fundamentally wrong that a reboot is not fixing. # ━━━ State Files ━━━ -# Strike counts reset on reboot — /tmp is correct for this +# Strike counts reset on reboot — /tmp is correct (fresh start after each reboot) SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" -# Persistent container skip list — survives reboots, auto-clears when container recovers +# Persistent container skip list — on /boot/ so it survives reboots +# Containers added here when docker_watchdog.sh exhausts all restart attempts +# Auto-clears when container is found running again after reboot or manual fix SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" -# Reboot timestamp log — survives reboots for loop detection +# Reboot timestamp log — on /boot/ for reboot loop detection across reboots SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db" # ━━━ Strike and Reboot Loop Settings ━━━ -# Consecutive threshold hits before triggering reboot +# Consecutive threshold hits required before triggering reboot SYS_WATCHDOG_STRIKE_LIMIT=2 -# Maximum reboots allowed in window before shutdown instead — prevents reboot loops +# Maximum reboots allowed within the window before shutting down instead +# A reboot loop means something fundamental is broken that rebooting is not fixing SYS_WATCHDOG_REBOOT_LIMIT=3 -# Window in hours — controls both reboot count window AND rolling log purge -# 12 = entries older than 12hrs purge automatically, fresh window starts +# Window in hours — controls BOTH the reboot count window AND the rolling log purge +# Entries older than this many hours are automatically removed from the reboot log SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 # ━━━ Thresholds ━━━ -# Set these at "I am about to become unstable" levels — not just "things are a bit tight" +# Set these at "I am about to become unstable" levels — not just "things are a bit high" SYS_WATCHDOG_ROOTFS_PCT=95 # rootfs % — at 95% something is seriously wrong - SYS_WATCHDOG_LOG_PCT=95 # /var/log % — log spam filling disk - SYS_WATCHDOG_MEM_GB=4 # free RAM GB — 4GB on 128GB system is critical + SYS_WATCHDOG_LOG_PCT=95 # /var/log % — log spam filling the filesystem + SYS_WATCHDOG_MEM_GB=4 # free RAM GB — 4GB free on 128GB system is critical SYS_WATCHDOG_ARC_PINNED_PCT=98 # ZFS ARC % of max before attempting reclaim SYS_WATCHDOG_ARC_RELEASE_PCT=95 # ZFS ARC % after reclaim that still triggers reboot - SYS_WATCHDOG_LOAD_MULTIPLIER=3 # strike if load avg > cores x this value + SYS_WATCHDOG_LOAD_MULTIPLIER=3 # strike if load avg > cores x this multiplier SYS_WATCHDOG_ZOMBIE_LIMIT=50 # zombie process count before strike - SYS_WATCHDOG_CPU_TEMP_MAX=95 # degrees C — adjust for your CPU tjmax + SYS_WATCHDOG_CPU_TEMP_MAX=95 # degrees C — adjust for your specific CPU tjmax # ━━━ Check Toggles ━━━ -# true = run this check / false = skip entirely -# Disable checks that are not relevant to your hardware or cause false positives +# true = run this check on every watchdog cycle / false = skip entirely +# Disable checks not relevant to your hardware or that cause false positives SYS_WATCHDOG_CHECK_ROOTFS=true SYS_WATCHDOG_CHECK_LOG=true SYS_WATCHDOG_CHECK_RAM=true SYS_WATCHDOG_CHECK_ARC=true SYS_WATCHDOG_CHECK_CPU_TEMP=true - SYS_WATCHDOG_CHECK_LOAD=false + SYS_WATCHDOG_CHECK_LOAD=false # disabled — load spikes during transcoding are normal SYS_WATCHDOG_CHECK_ZOMBIES=true - SYS_WATCHDOG_CHECK_CONTAINERS=true + SYS_WATCHDOG_CHECK_CONTAINERS=true # checks persistent skip list from docker_watchdog.sh SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true # ━━━ Abort Toggles ━━━ -# true = abort reboot if condition is active / false = reboot anyway -# Default true = conservative — set false only when "reboot no matter what" is wanted -# Goal: graceful reboot before crash is always better than hard crash mid-operation - SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # unhealthy pool + reboot = potential data loss - SYS_WATCHDOG_ABORT_ON_PARITY=true # aborting parity better than crashing mid-check - SYS_WATCHDOG_ABORT_ON_MOVER=true # aborting move better than crashing mid-move +# Controls whether certain conditions prevent a reboot from happening. +# true = abort reboot if this condition is active (conservative — default) +# false = reboot anyway regardless of this condition (aggressive) +# Philosophy: a graceful reboot before crash is always better than a hard crash mid-operation + SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # unhealthy pool + reboot risks data loss + SYS_WATCHDOG_ABORT_ON_PARITY=true # aborting parity check beats crashing mid-check + SYS_WATCHDOG_ABORT_ON_MOVER=true # aborting mover beats crashing mid-move # ============================================================================================== # ──────────────────────── End Of User Variables ─────────────────────────────────────────────── diff --git a/Media/lidarr_cleanup.sh b/Media/lidarr_cleanup.sh new file mode 100644 index 0000000..9466428 --- /dev/null +++ b/Media/lidarr_cleanup.sh @@ -0,0 +1,291 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Lidarr Cleanup Script -------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Removes orphaned music files from the library that Lidarr no longer tracks. +# Uses the Lidarr API to build a complete list of tracked file paths then compares +# against what exists on disk — anything not tracked and older than LIDARR_ORPHAN_AGE +# days is considered an orphan and deleted. +# +# File classification: +# TRACKED — Lidarr API knows about this exact file path → leave it alone +# PROTECTED — matches LIDARR_PROTECTED_PATTERNS → never delete (cover art, .nfo etc.) +# ORPHAN — music file, not tracked, older than LIDARR_ORPHAN_AGE days → delete +# JUNK — not a music extension, not protected → delete regardless of age +# RECENT — not tracked, under LIDARR_ORPHAN_AGE days old → skip (may be mid-import) +# +# Why protected patterns matter: +# Lidarr generates cover art (*.jpg), metadata (*.nfo) and lyrics (*.lrc) but does +# not include these in its tracked file API response. Without protection these would +# be classified as orphans and deleted — breaking Lidarr and Emby metadata display. +# +# All configuration in Master.conf under Arr Cleanup section. +# Supports --dry-run to preview what would be deleted without making changes. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +if ! command -v curl >/dev/null 2>&1; then + error "curl not found — required for Lidarr API calls" + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + error "jq not found — required for JSON parsing" + notify "Lidarr cleanup failed on $(hostname) — jq not installed" "Lidarr Cleanup" "warning" + exit 1 +fi + +require_var LIDARR_URL +require_var LIDARR_API_KEY +require_var LIDARR_MUSIC_ROOT + +if [[ ! -d "$LIDARR_MUSIC_ROOT" ]]; then + error "Music root not found: $LIDARR_MUSIC_ROOT" + notify "Lidarr cleanup failed on $(hostname) — music root not found: $LIDARR_MUSIC_ROOT" "Lidarr Cleanup" "warning" + exit 1 +fi + +success "Config validated" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_GEAR Lidarr URL: $LIDARR_URL" + echo "$ICON_GEAR Music root: $LIDARR_MUSIC_ROOT" + echo "$ICON_TIME Orphan age: ${LIDARR_ORPHAN_AGE} days" + echo "$ICON_GEAR Extensions: ${LIDARR_EXTENSIONS[*]}" + echo "$ICON_GEAR Protected patterns: ${LIDARR_PROTECTED_PATTERNS[*]}" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted" + +# ----------------------------------------------------------------------------------------------- +# HELPERS +# ----------------------------------------------------------------------------------------------- + +# Makes a GET request to the Lidarr API — exits on connection failure +lidarr_api() { + local endpoint="$1" + local response http_code body + + response=$(curl -sf \ + --max-time 30 \ + -H "X-Api-Key: $LIDARR_API_KEY" \ + -w "\n%{http_code}" \ + "${LIDARR_URL}/api/v1/${endpoint}" 2>/dev/null) + + http_code=$(echo "$response" | tail -1) + body=$(echo "$response" | head -n -1) + + if [[ "$http_code" != "200" ]]; then + error "Lidarr API returned HTTP $http_code for endpoint: $endpoint" + return 1 + fi + + echo "$body" +} + +# Returns 0 if file extension matches LIDARR_EXTENSIONS list +is_music_file() { + local ext="${1##*.}" + ext="${ext,,}" + for valid_ext in "${LIDARR_EXTENSIONS[@]}"; do + [[ "$ext" == "$valid_ext" ]] && return 0 + done + return 1 +} + +# Returns 0 if filename matches any pattern in LIDARR_PROTECTED_PATTERNS +is_protected_file() { + local filename + filename=$(basename "$1") + for pattern in "${LIDARR_PROTECTED_PATTERNS[@]}"; do + # shellcheck disable=SC2254 + case "$filename" in + $pattern) return 0 ;; + esac + done + return 1 +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SYNC Fetching Lidarr Tracked Files ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_SYNC Fetching Lidarr Tracked Files ━━━" + +info "Querying Lidarr API: $LIDARR_URL" + +TRACKFILE_RESPONSE=$(lidarr_api "trackfile") || { + error "Failed to fetch track files from Lidarr — check URL and API key" + notify "Lidarr cleanup failed on $(hostname) — API unreachable" "Lidarr Cleanup" "warning" + exit 1 +} + +TMP_DIR="/tmp/lidarr_cleanup_$$" +mkdir -p "$TMP_DIR" +trap "rm -rf $TMP_DIR" EXIT + +TRACKED_FILE="$TMP_DIR/tracked_paths.txt" +echo "$TRACKFILE_RESPONSE" | jq -r '.[].path' 2>/dev/null | sort > "$TRACKED_FILE" + +TRACKED_COUNT=$(wc -l < "$TRACKED_FILE") +success "Lidarr tracks $TRACKED_COUNT files" + +if [[ "$TRACKED_COUNT" -eq 0 ]]; then + warn "No tracked files returned — Lidarr may not have scanned yet or library is empty" + warn "Aborting to prevent mass deletion" + notify "Lidarr cleanup aborted on $(hostname) — no tracked files returned from API" "Lidarr Cleanup" "warning" + exit 1 +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_CLEAN Scanning Music Root ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_CLEAN Scanning Music Root ━━━" +info "Root: $LIDARR_MUSIC_ROOT" +info "Orphan age: ${LIDARR_ORPHAN_AGE} days" +info "Protected: ${LIDARR_PROTECTED_PATTERNS[*]}" +echo "" + +START=$(date +%s) +ORPHAN_COUNT=0 +JUNK_COUNT=0 +RECENT_COUNT=0 +PROTECTED_COUNT=0 +ORPHAN_BYTES=0 +JUNK_BYTES=0 + +AGE_SECONDS=$(( LIDARR_ORPHAN_AGE * 86400 )) +NOW=$(date +%s) + +while IFS= read -r filepath; do + [[ -z "$filepath" ]] && continue + + # TRACKED — Lidarr knows about this file + if grep -qF "$filepath" "$TRACKED_FILE" 2>/dev/null; then + log "TRACKED: $filepath" + continue + fi + + # PROTECTED — never delete regardless of tracking status + if is_protected_file "$filepath"; then + log "PROTECTED: $filepath" + ((PROTECTED_COUNT++)) + continue + fi + + FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0) + + if is_music_file "$filepath"; then + # Music file not tracked — check age + FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0) + FILE_AGE=$(( NOW - FILE_MTIME )) + + if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]]; then + log "RECENT (skipping): $filepath" + ((RECENT_COUNT++)) + continue + fi + + # ORPHAN — old enough to delete + warn "$ICON_TRASH ORPHAN: $filepath" + if [[ "$DRY_RUN" == false ]]; then + rm -f "$filepath" && { + ((ORPHAN_COUNT++)) + ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE)) + } || error "Failed to delete: $filepath" + else + ((ORPHAN_COUNT++)) + ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE)) + fi + else + # JUNK — not a music extension, not protected + log "JUNK: $filepath" + if [[ "$DRY_RUN" == false ]]; then + rm -f "$filepath" && { + ((JUNK_COUNT++)) + JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE)) + } || error "Failed to delete: $filepath" + else + ((JUNK_COUNT++)) + JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE)) + fi + fi + +done < <(find "$LIDARR_MUSIC_ROOT" -type f 2>/dev/null) + +# Empty folder cleanup +if [[ "$DRY_RUN" == false ]]; then + echo "" + info "Cleaning up empty folders..." + find "$LIDARR_MUSIC_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null + success "Empty folders removed" +fi + +END=$(date +%s) + +# Format bytes helper +format_bytes() { + local bytes=$1 + if (( bytes > 1073741824 )); then + awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}" + elif (( bytes > 1048576 )); then + awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}" + else + echo "${bytes}B" + fi +} + +ORPHAN_HUMAN=$(format_bytes $ORPHAN_BYTES) +JUNK_HUMAN=$(format_bytes $JUNK_BYTES) +TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━━━ $ICON_SUMMARY LIDARR CLEANUP SUMMARY ━━━━━" +echo "$ICON_SYNC Tracked by Lidarr: $TRACKED_COUNT files" +echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (cover art, metadata)" +echo "$ICON_TRASH Orphans removed: $ORPHAN_COUNT files ($ORPHAN_HUMAN)" +echo "$ICON_TRASH Junk removed: $JUNK_COUNT files ($JUNK_HUMAN)" +echo "$ICON_TIME Recent skipped: $RECENT_COUNT files (under ${LIDARR_ORPHAN_AGE} days)" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "" +if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Status: DRY RUN — no files deleted" +elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then + echo "$ICON_DONE Status: $ICON_SUCCESS CLEAN — nothing to remove" + notify "Lidarr cleanup complete on $(hostname) — library is clean" "Lidarr Cleanup" "normal" +else + echo "$ICON_DONE Status: $ICON_SUCCESS DONE — $TOTAL_REMOVED files removed" + notify "Lidarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "normal" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Media/radarr_cleanup.sh b/Media/radarr_cleanup.sh new file mode 100644 index 0000000..e69de29 diff --git a/Media/sonarr_cleanup.sh b/Media/sonarr_cleanup.sh new file mode 100644 index 0000000..2a4a8db --- /dev/null +++ b/Media/sonarr_cleanup.sh @@ -0,0 +1,281 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Sonarr Cleanup Script -------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Removes orphaned TV episode files from the library that Sonarr no longer tracks. +# Uses the Sonarr API to build a complete list of tracked episode file paths then compares +# against what exists on disk — anything not tracked and older than SONARR_ORPHAN_AGE +# days is considered an orphan and deleted. +# +# File classification: +# TRACKED — Sonarr API knows about this exact file path → leave it alone +# PROTECTED — matches SONARR_PROTECTED_PATTERNS → never delete (artwork, subtitles, .nfo) +# ORPHAN — video file, not tracked, older than SONARR_ORPHAN_AGE days → delete +# JUNK — not a video extension, not protected → delete regardless of age +# RECENT — not tracked, under SONARR_ORPHAN_AGE days old → skip (may be mid-import) +# +# Why protected patterns matter: +# Sonarr generates show artwork (*.jpg), metadata (*.nfo) and manages subtitles +# (*.srt, *.sub, *.ass) but does not include these in its tracked file API response. +# Without protection these would be classified as orphans and deleted. +# +# All configuration in Master.conf under Arr Cleanup section. +# Supports --dry-run to preview what would be deleted without making changes. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +if ! command -v curl >/dev/null 2>&1; then + error "curl not found — required for Sonarr API calls" + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + error "jq not found — required for JSON parsing" + notify "Sonarr cleanup failed on $(hostname) — jq not installed" "Sonarr Cleanup" "warning" + exit 1 +fi + +require_var SONARR_URL +require_var SONARR_API_KEY +require_var SONARR_TV_ROOT + +if [[ ! -d "$SONARR_TV_ROOT" ]]; then + error "TV root not found: $SONARR_TV_ROOT" + notify "Sonarr cleanup failed on $(hostname) — TV root not found: $SONARR_TV_ROOT" "Sonarr Cleanup" "warning" + exit 1 +fi + +success "Config validated" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_GEAR Sonarr URL: $SONARR_URL" + echo "$ICON_GEAR TV root: $SONARR_TV_ROOT" + echo "$ICON_TIME Orphan age: ${SONARR_ORPHAN_AGE} days" + echo "$ICON_GEAR Extensions: ${SONARR_EXTENSIONS[*]}" + echo "$ICON_GEAR Protected patterns: ${SONARR_PROTECTED_PATTERNS[*]}" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted" + +# ----------------------------------------------------------------------------------------------- +# HELPERS +# ----------------------------------------------------------------------------------------------- + +sonarr_api() { + local endpoint="$1" + local response http_code body + + response=$(curl -sf \ + --max-time 30 \ + -H "X-Api-Key: $SONARR_API_KEY" \ + -w "\n%{http_code}" \ + "${SONARR_URL}/api/v3/${endpoint}" 2>/dev/null) + + http_code=$(echo "$response" | tail -1) + body=$(echo "$response" | head -n -1) + + if [[ "$http_code" != "200" ]]; then + error "Sonarr API returned HTTP $http_code for endpoint: $endpoint" + return 1 + fi + + echo "$body" +} + +is_video_file() { + local ext="${1##*.}" + ext="${ext,,}" + for valid_ext in "${SONARR_EXTENSIONS[@]}"; do + [[ "$ext" == "$valid_ext" ]] && return 0 + done + return 1 +} + +is_protected_file() { + local filename + filename=$(basename "$1") + for pattern in "${SONARR_PROTECTED_PATTERNS[@]}"; do + # shellcheck disable=SC2254 + case "$filename" in + $pattern) return 0 ;; + esac + done + return 1 +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SYNC Fetching Sonarr Tracked Files ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_SYNC Fetching Sonarr Tracked Files ━━━" + +info "Querying Sonarr API: $SONARR_URL" + +EPISODEFILE_RESPONSE=$(sonarr_api "episodefile") || { + error "Failed to fetch episode files from Sonarr — check URL and API key" + notify "Sonarr cleanup failed on $(hostname) — API unreachable" "Sonarr Cleanup" "warning" + exit 1 +} + +TMP_DIR="/tmp/sonarr_cleanup_$$" +mkdir -p "$TMP_DIR" +trap "rm -rf $TMP_DIR" EXIT + +TRACKED_FILE="$TMP_DIR/tracked_paths.txt" +echo "$EPISODEFILE_RESPONSE" | jq -r '.[].path' 2>/dev/null | sort > "$TRACKED_FILE" + +TRACKED_COUNT=$(wc -l < "$TRACKED_FILE") +success "Sonarr tracks $TRACKED_COUNT episode files" + +if [[ "$TRACKED_COUNT" -eq 0 ]]; then + warn "No tracked files returned — Sonarr may not have scanned yet or library is empty" + warn "Aborting to prevent mass deletion" + notify "Sonarr cleanup aborted on $(hostname) — no tracked files returned from API" "Sonarr Cleanup" "warning" + exit 1 +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_CLEAN Scanning TV Root ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_CLEAN Scanning TV Root ━━━" +info "Root: $SONARR_TV_ROOT" +info "Orphan age: ${SONARR_ORPHAN_AGE} days" +info "Protected: ${SONARR_PROTECTED_PATTERNS[*]}" +echo "" + +START=$(date +%s) +ORPHAN_COUNT=0 +JUNK_COUNT=0 +RECENT_COUNT=0 +PROTECTED_COUNT=0 +ORPHAN_BYTES=0 +JUNK_BYTES=0 + +AGE_SECONDS=$(( SONARR_ORPHAN_AGE * 86400 )) +NOW=$(date +%s) + +while IFS= read -r filepath; do + [[ -z "$filepath" ]] && continue + + if grep -qF "$filepath" "$TRACKED_FILE" 2>/dev/null; then + log "TRACKED: $filepath" + continue + fi + + if is_protected_file "$filepath"; then + log "PROTECTED: $filepath" + ((PROTECTED_COUNT++)) + continue + fi + + FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0) + + if is_video_file "$filepath"; then + FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0) + FILE_AGE=$(( NOW - FILE_MTIME )) + + if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]]; then + log "RECENT (skipping): $filepath" + ((RECENT_COUNT++)) + continue + fi + + warn "$ICON_TRASH ORPHAN: $filepath" + if [[ "$DRY_RUN" == false ]]; then + rm -f "$filepath" && { + ((ORPHAN_COUNT++)) + ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE)) + } || error "Failed to delete: $filepath" + else + ((ORPHAN_COUNT++)) + ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE)) + fi + else + log "JUNK: $filepath" + if [[ "$DRY_RUN" == false ]]; then + rm -f "$filepath" && { + ((JUNK_COUNT++)) + JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE)) + } || error "Failed to delete: $filepath" + else + ((JUNK_COUNT++)) + JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE)) + fi + fi + +done < <(find "$SONARR_TV_ROOT" -type f 2>/dev/null) + +if [[ "$DRY_RUN" == false ]]; then + echo "" + info "Cleaning up empty folders..." + find "$SONARR_TV_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null + success "Empty folders removed" +fi + +END=$(date +%s) + +format_bytes() { + local bytes=$1 + if (( bytes > 1073741824 )); then + awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}" + elif (( bytes > 1048576 )); then + awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}" + else + echo "${bytes}B" + fi +} + +ORPHAN_HUMAN=$(format_bytes $ORPHAN_BYTES) +JUNK_HUMAN=$(format_bytes $JUNK_BYTES) +TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━━━ $ICON_SUMMARY SONARR CLEANUP SUMMARY ━━━━━" +echo "$ICON_SYNC Tracked by Sonarr: $TRACKED_COUNT files" +echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles, metadata)" +echo "$ICON_TRASH Orphans removed: $ORPHAN_COUNT files ($ORPHAN_HUMAN)" +echo "$ICON_TRASH Junk removed: $JUNK_COUNT files ($JUNK_HUMAN)" +echo "$ICON_TIME Recent skipped: $RECENT_COUNT files (under ${SONARR_ORPHAN_AGE} days)" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "" +if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Status: DRY RUN — no files deleted" +elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then + echo "$ICON_DONE Status: $ICON_SUCCESS CLEAN — nothing to remove" + notify "Sonarr cleanup complete on $(hostname) — library is clean" "Sonarr Cleanup" "normal" +else + echo "$ICON_DONE Status: $ICON_SUCCESS DONE — $TOTAL_REMOVED files removed" + notify "Sonarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Sonarr Cleanup" "normal" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Monitors/backup_verify.sh b/Monitors/backup_verify.sh new file mode 100644 index 0000000..ac87237 --- /dev/null +++ b/Monitors/backup_verify.sh @@ -0,0 +1,187 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Backup Verify ---------------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Verifies the rsync mirror is healthy by comparing random file samples between +# local and remote servers using MD5 checksums. +# +# Randomly samples BACKUP_VERIFY_SAMPLE files per share, computes checksums locally, +# then computes the same checksums on the remote via SSH and compares results. +# +# Results per file: +# MATCH — checksums identical, file is correctly mirrored +# MISMATCH — file exists on both but checksums differ — sync may have failed +# MISSING — file exists locally but not on remote — not yet synced or deleted +# +# Silent when all files match. Notifies on any mismatch or missing file. +# Uses existing SSH keys — no additional configuration needed beyond share list. +# +# All configuration in Master.conf under Backup Verify section. +# Supports --dry-run to show what would be checked without running checksums. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +detect_hosts +resolve_remote_ip + +# Use BACKUP_VERIFY_SHARES if defined, fall back to DAILY_SYNC_SHARES +if [[ ${#BACKUP_VERIFY_SHARES[@]} -gt 0 ]]; then + VERIFY_SHARES=("${BACKUP_VERIFY_SHARES[@]}") + info "Using BACKUP_VERIFY_SHARES (${#VERIFY_SHARES[@]} shares)" +else + VERIFY_SHARES=("${DAILY_SYNC_SHARES[@]}") + info "BACKUP_VERIFY_SHARES not set — using DAILY_SYNC_SHARES (${#VERIFY_SHARES[@]} shares)" +fi + +if [[ ${#VERIFY_SHARES[@]} -eq 0 ]]; then + warn "No shares configured — nothing to verify" + exit 0 +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_VERIFY Shares: ${#VERIFY_SHARES[@]}" + echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share" + echo "$ICON_VERIFY Min size: $BACKUP_VERIFY_MIN_SIZE" + echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing sample selection only, no checksums computed" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_VERIFY Backup Verification ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_VERIFY Backup Verification — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)" +echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share (min size: $BACKUP_VERIFY_MIN_SIZE)" +echo "" + +START=$(date +%s) +TOTAL_CHECKED=0 +TOTAL_MATCH=0 +TOTAL_MISMATCH=0 +TOTAL_MISSING=0 +SHARES_WITH_ISSUES=() + +for share in "${VERIFY_SHARES[@]}"; do + SHARE_NAME=$(basename "$share") + echo "━━━ $ICON_VERIFY $SHARE_NAME ━━━" + + if [[ ! -d "$share" ]]; then + warn "$SHARE_NAME not found locally — skipping" + echo "" + continue + fi + + # Find files above minimum size and randomly sample + SAMPLE_FILES=$(find "$share" -type f -size +"$BACKUP_VERIFY_MIN_SIZE" 2>/dev/null | \ + shuf | head -n "$BACKUP_VERIFY_SAMPLE") + + SAMPLE_COUNT=$(echo "$SAMPLE_FILES" | grep -c "." 2>/dev/null || echo 0) + + if [[ "$SAMPLE_COUNT" -eq 0 ]]; then + info "No files found above $BACKUP_VERIFY_MIN_SIZE — skipping" + echo "" + continue + fi + + info "Sampled $SAMPLE_COUNT files" + + if [[ "$DRY_RUN" == true ]]; then + echo "$SAMPLE_FILES" | while IFS= read -r f; do + warn "DRY RUN — would check: $(basename "$f")" + done + echo "" + continue + fi + + SHARE_MISMATCH=0 + SHARE_MISSING=0 + SHARE_MATCH=0 + + while IFS= read -r local_file; do + [[ -z "$local_file" ]] && continue + + # Compute local checksum + local_md5=$(md5sum "$local_file" 2>/dev/null | awk '{print $1}') + + if [[ -z "$local_md5" ]]; then + warn "Could not checksum: $local_file — skipping" + continue + fi + + # Compute remote checksum via SSH + remote_md5=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ + "md5sum '$local_file' 2>/dev/null | awk '{print \$1}'" 2>/dev/null) + + ((TOTAL_CHECKED++)) + + if [[ -z "$remote_md5" ]]; then + warn "$ICON_ERROR MISSING: $(basename "$local_file")" + ((SHARE_MISSING++)) + ((TOTAL_MISSING++)) + elif [[ "$local_md5" == "$remote_md5" ]]; then + log "MATCH: $(basename "$local_file")" + ((SHARE_MATCH++)) + ((TOTAL_MATCH++)) + else + error "$ICON_ERROR MISMATCH: $(basename "$local_file")" + ((SHARE_MISMATCH++)) + ((TOTAL_MISMATCH++)) + fi + + done <<< "$SAMPLE_FILES" + + echo " $ICON_SUCCESS Match: $SHARE_MATCH $ICON_WARN Missing: $SHARE_MISSING $ICON_ERROR Mismatch: $SHARE_MISMATCH" + + if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 ]]; then + SHARES_WITH_ISSUES+=("$SHARE_NAME") + fi + + echo "" +done + +END=$(date +%s) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "━━━━━ $ICON_SUMMARY BACKUP VERIFY SUMMARY ━━━━━" +echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME" +echo "$ICON_VERIFY Checked: $TOTAL_CHECKED files" +echo "$ICON_SUCCESS Match: $TOTAL_MATCH" +echo "$ICON_WARN Missing: $TOTAL_MISSING" +echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "" + +if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Status: DRY RUN — no checksums computed" +elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then + echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention" + notify "Backup verify failed on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" "Backup Verify" "warning" +else + echo "$ICON_DONE Status: $ICON_SUCCESS ALL FILES MATCH" + notify "Backup verify passed on $(hostname) → $REMOTE_SERVER_NAME — $TOTAL_CHECKED files checked across ${#VERIFY_SHARES[@]} shares" "Backup Verify" "normal" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Monitors/bandwidth_monitor.sh b/Monitors/bandwidth_monitor.sh new file mode 100644 index 0000000..7e72097 --- /dev/null +++ b/Monitors/bandwidth_monitor.sh @@ -0,0 +1,152 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Bandwidth Monitor ------------------------------------------ +# ----------------------------------------------------------------------------------------------- +# Logs daily rsync transfer totals and generates weekly summary reports. +# Designed for minimal flash drive impact — one bounded write per day. +# +# Two modes: +# --log-transfer "profile" bytes duration — called by rsync.sh after each sync +# appends one line, trims old entries +# --report — generates weekly summary from log +# (no args) — generates summary report +# +# Log format (one line per transfer): +# YYYY-MM-DD|profile|bytes|duration_seconds +# +# Log file stays bounded to BANDWIDTH_LOG_RETENTION days — old entries trimmed on write. +# Minimal writes: one append per rsync run, one trim per append. +# +# All configuration in Master.conf under Bandwidth Monitor section. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# Check for log-transfer mode +LOG_TRANSFER_MODE=false +REPORT_MODE=false +TRANSFER_PROFILE="" +TRANSFER_BYTES=0 +TRANSFER_DURATION=0 + +for arg in "${PARSED_ARGS[@]}"; do + case "$arg" in + --log-transfer) LOG_TRANSFER_MODE=true ;; + --report) REPORT_MODE=true ;; + *) + if [[ "$LOG_TRANSFER_MODE" == true && -z "$TRANSFER_PROFILE" ]]; then + TRANSFER_PROFILE="$arg" + elif [[ "$LOG_TRANSFER_MODE" == true && "$TRANSFER_BYTES" -eq 0 ]]; then + TRANSFER_BYTES="$arg" + elif [[ "$LOG_TRANSFER_MODE" == true ]]; then + TRANSFER_DURATION="$arg" + fi + ;; + esac +done + +# Ensure log file exists +touch "$BANDWIDTH_LOG" 2>/dev/null || { + error "Cannot create bandwidth log: $BANDWIDTH_LOG" + exit 1 +} + +# ----------------------------------------------------------------------------------------------- +# LOG TRANSFER MODE +# Called by rsync.sh after each successful sync — appends one line and trims old entries +# Usage: bandwidth_monitor.sh --log-transfer "profile" bytes duration +# ----------------------------------------------------------------------------------------------- +if [[ "$LOG_TRANSFER_MODE" == true ]]; then + TODAY=$(date '+%Y-%m-%d') + echo "${TODAY}|${TRANSFER_PROFILE}|${TRANSFER_BYTES}|${TRANSFER_DURATION}" >> "$BANDWIDTH_LOG" + log "Logged transfer: $TRANSFER_PROFILE — $TRANSFER_BYTES bytes in ${TRANSFER_DURATION}s" + + # Trim entries older than retention period — keeps file bounded + CUTOFF=$(date -d "${BANDWIDTH_LOG_RETENTION} days ago" '+%Y-%m-%d') + TEMP_FILE="${BANDWIDTH_LOG}.tmp" + awk -F'|' -v cutoff="$CUTOFF" '$1 >= cutoff' "$BANDWIDTH_LOG" > "$TEMP_FILE" + mv "$TEMP_FILE" "$BANDWIDTH_LOG" + log "Log trimmed — keeping entries from $CUTOFF onwards" + exit 0 +fi + +# ----------------------------------------------------------------------------------------------- +# REPORT MODE — generate summary from log +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_BANDWIDTH Bandwidth Report — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "" + +if [[ ! -s "$BANDWIDTH_LOG" ]]; then + warn "No bandwidth data yet — log is empty" + warn "Data accumulates as rsync jobs run" + exit 0 +fi + +START=$(date +%s) + +# Calculate date range in log +OLDEST=$(awk -F'|' 'NR==1{print $1}' "$BANDWIDTH_LOG") +NEWEST=$(awk -F'|' 'END{print $1}' "$BANDWIDTH_LOG") +ENTRY_COUNT=$(wc -l < "$BANDWIDTH_LOG") + +info "Log covers: $OLDEST → $NEWEST ($ENTRY_COUNT entries)" +echo "" + +# Total bytes transferred +TOTAL_BYTES=$(awk -F'|' '{sum += $3} END {print sum+0}' "$BANDWIDTH_LOG") +TOTAL_GB=$(awk "BEGIN {printf \"%.2f\", $TOTAL_BYTES / 1073741824}") + +# Per-profile breakdown +echo "━━━ $ICON_BANDWIDTH Per-Profile Totals ━━━" +awk -F'|' '{ + bytes[$2] += $3 + runs[$2]++ + duration[$2] += $4 +} +END { + for (profile in bytes) { + gb = bytes[profile] / 1073741824 + printf " %-20s %6.2f GB (%d runs)\n", profile, gb, runs[profile] + } +}' "$BANDWIDTH_LOG" | sort -k3 -rn + +echo "" + +# Daily totals for the last 7 days +echo "━━━ $ICON_BANDWIDTH Last 7 Days ━━━" +for i in 6 5 4 3 2 1 0; do + day=$(date -d "$i days ago" '+%Y-%m-%d') + day_bytes=$(awk -F'|' -v d="$day" '$1==d{sum+=$3} END{print sum+0}' "$BANDWIDTH_LOG") + day_gb=$(awk "BEGIN {printf \"%.2f\", $day_bytes / 1073741824}") + + # Flag days that exceeded warning threshold + over_warn=$(awk "BEGIN {print ($day_bytes > $BANDWIDTH_WARN_GB * 1073741824) ? 1 : 0}") + if [[ "$over_warn" == "1" ]]; then + echo " $ICON_WARN $day ${day_gb} GB ← exceeded ${BANDWIDTH_WARN_GB}GB threshold" + else + echo " $ICON_TIME $day ${day_gb} GB" + fi +done + +echo "" +echo "━━━ $ICON_SUMMARY Totals ━━━" +echo " $ICON_BANDWIDTH Total transferred: ${TOTAL_GB} GB" +echo " $ICON_TIME Log period: $OLDEST → $NEWEST" +echo " $ICON_GEAR Retention: ${BANDWIDTH_LOG_RETENTION} days" + +END=$(date +%s) + +echo "" +echo "━━━━━ $ICON_SUMMARY BANDWIDTH SUMMARY ━━━━━" +echo "$ICON_BANDWIDTH Total: ${TOTAL_GB} GB" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_DONE Status: $ICON_SUCCESS DONE" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +notify "Bandwidth report on $(hostname) — ${TOTAL_GB}GB transferred (${OLDEST} to ${NEWEST})" "Bandwidth Monitor" "normal" \ No newline at end of file diff --git a/Monitors/cert_monitor.sh b/Monitors/cert_monitor.sh new file mode 100644 index 0000000..bf9cf41 --- /dev/null +++ b/Monitors/cert_monitor.sh @@ -0,0 +1,182 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Certificate Monitor ---------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Monitors SSL certificate expiry for all configured domains by connecting directly +# via openssl — no dependency on NPM or any other service. Reads the actual certificate +# the server is presenting to the outside world. +# +# This approach catches real-world cert issues that API-based checks miss: +# - Cert renewed but server not reloaded +# - Wrong cert being served +# - Cert chain issues +# +# Each domain and subdomain is a separate entry — they have independent certs. +# Silent when all certs are healthy. Notifies when any approach warning threshold. +# Notifications batched per severity — one message for warnings, one for criticals. +# +# All configuration in Master.conf under Certificate Monitor section. +# Supports --dry-run to check certs and show results without sending notifications. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if ! command -v openssl >/dev/null 2>&1; then + error "openssl not found — required for certificate checks" + exit 1 +fi + +success "openssl available" + +if [[ ${#CERT_MONITOR_DOMAINS[@]} -eq 0 ]]; then + warn "CERT_MONITOR_DOMAINS is empty in Master.conf — add your domains to enable monitoring" + exit 0 +fi + +info "$ICON_CERT Domains to check: ${#CERT_MONITOR_DOMAINS[@]}" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_CERT Domains: ${CERT_MONITOR_DOMAINS[*]}" + echo "$ICON_WARN Warn at: ${CERT_WARN_DAYS} days remaining" + echo "$ICON_ERROR Crit at: ${CERT_CRIT_DAYS} days remaining" + echo "$ICON_TIME Timeout: ${CERT_TIMEOUT}s per domain" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — results shown but no notifications sent" + +# ----------------------------------------------------------------------------------------------- +# CERT CHECK FUNCTION +# Connects to domain:443 via openssl, extracts expiry date, calculates days remaining. +# Returns 0=healthy 1=warning 2=critical 3=failed +# ----------------------------------------------------------------------------------------------- +check_cert() { + local domain="$1" + local port="${2:-443}" + + local expiry_str + expiry_str=$(echo | timeout "$CERT_TIMEOUT" openssl s_client \ + -connect "${domain}:${port}" \ + -servername "$domain" \ + 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) + + if [[ -z "$expiry_str" ]]; then + error "$ICON_CERT $domain — could not retrieve certificate" + return 3 + fi + + local expiry_epoch + expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null) + + if [[ -z "$expiry_epoch" ]]; then + error "$ICON_CERT $domain — could not parse expiry date: $expiry_str" + return 3 + fi + + local now days_remaining expiry_display + now=$(date +%s) + days_remaining=$(( (expiry_epoch - now) / 86400 )) + expiry_display=$(date -d "$expiry_str" '+%Y-%m-%d' 2>/dev/null) + + if [[ "$days_remaining" -le "$CERT_CRIT_DAYS" ]]; then + error "$ICON_CERT $domain — CRITICAL: ${days_remaining} days remaining (expires $expiry_display)" + return 2 + elif [[ "$days_remaining" -le "$CERT_WARN_DAYS" ]]; then + warn "$ICON_CERT $domain — WARNING: ${days_remaining} days remaining (expires $expiry_display)" + return 1 + else + success "$ICON_CERT $domain — OK: ${days_remaining} days remaining (expires $expiry_display)" + return 0 + fi +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_CERT Certificate Monitor ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_CERT Certificate Monitor — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "$ICON_WARN Warn threshold: ${CERT_WARN_DAYS} days" +echo "$ICON_ERROR Crit threshold: ${CERT_CRIT_DAYS} days" +echo "" + +START=$(date +%s) +HEALTHY=() +WARNING=() +CRITICAL=() +FAILED=() +declare -A DOMAIN_STATUS + +for domain in "${CERT_MONITOR_DOMAINS[@]}"; do + [[ -z "$domain" ]] && continue + echo "━━━ $ICON_CERT $domain ━━━" + check_cert "$domain" + result=$? + case $result in + 0) HEALTHY+=("$domain"); DOMAIN_STATUS["$domain"]="OK" ;; + 1) WARNING+=("$domain"); DOMAIN_STATUS["$domain"]="WARN" ;; + 2) CRITICAL+=("$domain"); DOMAIN_STATUS["$domain"]="CRIT" ;; + 3) FAILED+=("$domain"); DOMAIN_STATUS["$domain"]="FAIL" ;; + esac + echo "" +done + +END=$(date +%s) + +if [[ "$DRY_RUN" == false ]]; then + [[ ${#CRITICAL[@]} -gt 0 ]] && \ + notify "Certificate CRITICAL on $(hostname) — expiring within ${CERT_CRIT_DAYS} days: ${CRITICAL[*]}" "Certificate Monitor" "warning" + [[ ${#WARNING[@]} -gt 0 ]] && \ + notify "Certificate WARNING on $(hostname) — expiring within ${CERT_WARN_DAYS} days: ${WARNING[*]}" "Certificate Monitor" "warning" + [[ ${#FAILED[@]} -gt 0 ]] && \ + notify "Certificate check FAILED on $(hostname) — could not reach: ${FAILED[*]}" "Certificate Monitor" "warning" +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "━━━━━ $ICON_SUMMARY CERTIFICATE MONITOR SUMMARY ━━━━━" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "" +echo " $ICON_SUCCESS Healthy: ${#HEALTHY[@]} $ICON_WARN Warning: ${#WARNING[@]} $ICON_ERROR Critical: ${#CRITICAL[@]} Failed: ${#FAILED[@]}" +echo "" +for domain in "${CERT_MONITOR_DOMAINS[@]}"; do + [[ -z "$domain" ]] && continue + case "${DOMAIN_STATUS[$domain]:-UNKN}" in + OK) echo " $ICON_SUCCESS $domain" ;; + WARN) echo " $ICON_WARN $domain" ;; + CRIT) echo " $ICON_ERROR $domain" ;; + FAIL) echo " $ICON_ERROR $domain (unreachable)" ;; + esac +done +echo "" +if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Status: DRY RUN — no notifications sent" +elif [[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]]; then + echo "$ICON_ERROR Status: ACTION REQUIRED" +elif [[ ${#WARNING[@]} -gt 0 ]]; then + echo "$ICON_WARN Status: WARNINGS — renewal recommended" +else + echo "$ICON_DONE Status: $ICON_SUCCESS ALL CERTS HEALTHY" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/Monitors/emby_session_report.sh b/Monitors/emby_session_report.sh new file mode 100644 index 0000000..e69de29 diff --git a/Monitors/smart_health.sh b/Monitors/smart_health.sh new file mode 100644 index 0000000..333e51c --- /dev/null +++ b/Monitors/smart_health.sh @@ -0,0 +1,249 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- SMART Health Monitor --------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Checks SMART health attributes for all drives on the system. +# Reads data live from each drive via smartctl — no persistent writes. +# +# Monitored attributes: +# Reallocated_Sector_Ct — bad sectors remapped — any > 0 is concerning +# Current_Pending_Sector — sectors waiting for reallocation — any > 0 is concerning +# Offline_Uncorrectable — sectors that could not be corrected — any > 0 is critical +# Temperature_Celsius — drive temperature vs SMART_TEMP_WARN / SMART_TEMP_CRIT +# Power_On_Hours — informational — drive age estimation +# SMART overall status — pass/fail per drive +# +# Discovers drives automatically — no configuration needed for drive list. +# SMART_IGNORE_DRIVES allows skipping specific drives (e.g. USB flash drives). +# +# All configuration in Master.conf under SMART Health section. +# Supports --dry-run to show which drives would be checked without running smartctl. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +if ! command -v smartctl >/dev/null 2>&1; then + error "smartctl not found — install smartmontools" + notify "SMART health check failed on $(hostname) — smartmontools not installed" "SMART Health" "warning" + exit 1 +fi + +success "smartctl available" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_SMART Temp warn: ${SMART_TEMP_WARN}°C" + echo "$ICON_SMART Temp crit: ${SMART_TEMP_CRIT}°C" + echo "$ICON_SMART Ignore drives: ${SMART_IGNORE_DRIVES[*]:-none}" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "" + + # Show drives that would be checked + echo "━━━ Discovered Drives ━━━" + for drive in /dev/sd? /dev/nvme?; do + [[ ! -e "$drive" ]] && continue + drive_name=$(basename "$drive") + ignored=false + for ignore in "${SMART_IGNORE_DRIVES[@]}"; do + [[ "$drive_name" == "$ignore" ]] && ignored=true && break + done + if [[ "$ignored" == true ]]; then + echo " $ICON_WARN $drive — ignored" + else + echo " $ICON_SMART $drive — would check" + fi + done + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing drive list only, no SMART data read" + +# ----------------------------------------------------------------------------------------------- +# HELPER — extract SMART attribute value +# Usage: get_smart_attr "/dev/sda" "Reallocated_Sector_Ct" +# ----------------------------------------------------------------------------------------------- +get_smart_attr() { + local drive="$1" attr="$2" + smartctl -A "$drive" 2>/dev/null | \ + awk -v attr="$attr" '$2 == attr {print $10}' +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SMART SMART Health Check ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_SMART SMART Health Check — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "" + +START=$(date +%s) +DRIVES_OK=() +DRIVES_WARN=() +DRIVES_CRIT=() +DRIVES_SKIP=() + +for drive in /dev/sd? /dev/nvme?; do + [[ ! -e "$drive" ]] && continue + drive_name=$(basename "$drive") + + # Check ignore list + ignored=false + for ignore in "${SMART_IGNORE_DRIVES[@]}"; do + [[ "$drive_name" == "$ignore" ]] && ignored=true && break + done + + if [[ "$ignored" == true ]]; then + info "$drive_name — ignored (in SMART_IGNORE_DRIVES)" + DRIVES_SKIP+=("$drive_name") + continue + fi + + echo "━━━ $ICON_SMART $drive_name ━━━" + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would check $drive_name" + echo "" + continue + fi + + # Check if drive supports SMART + if ! smartctl -i "$drive" 2>/dev/null | grep -q "SMART support is: Enabled"; then + warn "$drive_name — SMART not enabled or not supported" + DRIVES_SKIP+=("$drive_name") + echo "" + continue + fi + + # Overall SMART status + SMART_STATUS=$(smartctl -H "$drive" 2>/dev/null | grep "overall-health" | awk '{print $NF}') + if [[ "$SMART_STATUS" == "PASSED" ]]; then + success "Overall status: PASSED" + else + error "Overall status: $SMART_STATUS" + fi + + # Key attributes + DRIVE_WARN=false + DRIVE_CRIT=false + + # Reallocated sectors + REALLOC=$(get_smart_attr "$drive" "Reallocated_Sector_Ct") + if [[ -n "$REALLOC" ]]; then + if [[ "$REALLOC" -gt 0 ]]; then + warn "$ICON_SMART Reallocated sectors: $REALLOC — drive showing wear" + DRIVE_WARN=true + else + success "$ICON_SMART Reallocated sectors: $REALLOC" + fi + fi + + # Pending sectors + PENDING=$(get_smart_attr "$drive" "Current_Pending_Sector") + if [[ -n "$PENDING" ]]; then + if [[ "$PENDING" -gt 0 ]]; then + warn "$ICON_SMART Pending sectors: $PENDING — sectors awaiting reallocation" + DRIVE_WARN=true + else + success "$ICON_SMART Pending sectors: $PENDING" + fi + fi + + # Uncorrectable sectors + UNCORR=$(get_smart_attr "$drive" "Offline_Uncorrectable") + if [[ -n "$UNCORR" ]]; then + if [[ "$UNCORR" -gt 0 ]]; then + error "$ICON_SMART Uncorrectable sectors: $UNCORR — CRITICAL" + DRIVE_CRIT=true + else + success "$ICON_SMART Uncorrectable sectors: $UNCORR" + fi + fi + + # Temperature + TEMP=$(get_smart_attr "$drive" "Temperature_Celsius") + # NVMe uses different attribute name + [[ -z "$TEMP" ]] && TEMP=$(smartctl -A "$drive" 2>/dev/null | \ + awk '/Temperature/{print $2}' | head -1) + + if [[ -n "$TEMP" ]]; then + if [[ "$TEMP" -ge "$SMART_TEMP_CRIT" ]]; then + error "$ICON_SMART Temperature: ${TEMP}°C — CRITICAL (threshold: ${SMART_TEMP_CRIT}°C)" + DRIVE_CRIT=true + elif [[ "$TEMP" -ge "$SMART_TEMP_WARN" ]]; then + warn "$ICON_SMART Temperature: ${TEMP}°C — warning (threshold: ${SMART_TEMP_WARN}°C)" + DRIVE_WARN=true + else + success "$ICON_SMART Temperature: ${TEMP}°C" + fi + fi + + # Power on hours — informational + POH=$(get_smart_attr "$drive" "Power_On_Hours") + if [[ -n "$POH" ]]; then + POH_DAYS=$(( POH / 24 )) + info "$ICON_SMART Power on hours: $POH (${POH_DAYS} days)" + fi + + # Classify drive + if [[ "$DRIVE_CRIT" == true ]]; then + DRIVES_CRIT+=("$drive_name") + elif [[ "$DRIVE_WARN" == true ]]; then + DRIVES_WARN+=("$drive_name") + else + DRIVES_OK+=("$drive_name") + fi + + echo "" +done + +END=$(date +%s) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "━━━━━ $ICON_SUMMARY SMART HEALTH SUMMARY ━━━━━" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "" +echo " $ICON_SUCCESS Healthy: ${#DRIVES_OK[@]} $ICON_WARN Warning: ${#DRIVES_WARN[@]} $ICON_ERROR Critical: ${#DRIVES_CRIT[@]} skipped: ${#DRIVES_SKIP[@]}" +echo "" + +[[ ${#DRIVES_OK[@]} -gt 0 ]] && echo " $ICON_SUCCESS ${DRIVES_OK[*]}" +[[ ${#DRIVES_WARN[@]} -gt 0 ]] && echo " $ICON_WARN ${DRIVES_WARN[*]}" +[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && echo " $ICON_ERROR ${DRIVES_CRIT[*]}" + +echo "" +if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Status: DRY RUN" +elif [[ ${#DRIVES_CRIT[@]} -gt 0 ]]; then + echo "$ICON_ERROR Status: CRITICAL — ${DRIVES_CRIT[*]}" + notify "SMART CRITICAL on $(hostname) — drives need immediate attention: ${DRIVES_CRIT[*]}" "SMART Health" "warning" +elif [[ ${#DRIVES_WARN[@]} -gt 0 ]]; then + echo "$ICON_WARN Status: WARNING — ${DRIVES_WARN[*]}" + notify "SMART WARNING on $(hostname) — drives showing wear: ${DRIVES_WARN[*]}" "SMART Health" "warning" +else + echo "$ICON_DONE Status: $ICON_SUCCESS ALL DRIVES HEALTHY" + notify "SMART health check passed on $(hostname) — ${#DRIVES_OK[@]} drives healthy" "SMART Health" "normal" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Monitors/weekly_health_digest.sh b/Monitors/weekly_health_digest.sh new file mode 100644 index 0000000..1256d5c --- /dev/null +++ b/Monitors/weekly_health_digest.sh @@ -0,0 +1,245 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Health Digest ---------------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Aggregates system health data from across the ecosystem into a single digest report. +# Reads existing state files — no new writes to flash drive. +# +# Three profiles controlled by DIGEST_PROFILE in Master.conf: +# always — sends every run regardless of findings (schedule daily for daily digest) +# smart — sends only if something worth reporting was found (intelligent filtering) +# weekly — sends once per week on DIGEST_DAY regardless of schedule frequency +# +# The cron schedule stays the same regardless of profile — just change DIGEST_PROFILE +# in Master.conf to switch behavior. Run daily, profile controls when it actually notifies. +# +# Data sources (reads only — no writes): +# /tmp/transcode_state.db — ramdisk symlink and usage +# /tmp/container_watchdog_state.db — active container strikes +# /tmp/system_watchdog_state.db — active system strikes +# /boot/config/failover_state.db — current failover state +# /boot/config/system_watchdog_failed.db — container skip list +# /boot/config/bandwidth_history.db — recent transfer totals +# SSL certs via openssl (live check) — days remaining per domain +# +# All configuration in Master.conf under Health Digest section. +# Supports --dry-run to generate report without sending notification. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +success "Health Digest — profile: $DIGEST_PROFILE" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — report generated but no notification sent" + +# ----------------------------------------------------------------------------------------------- +# Profile check — should we send today? +# ----------------------------------------------------------------------------------------------- +SHOULD_SEND=false + +case "$DIGEST_PROFILE" in + always) + SHOULD_SEND=true + log "Profile: always — will send" + ;; + weekly) + TODAY_NAME=$(date '+%A') + if [[ "$TODAY_NAME" == "$DIGEST_DAY" ]]; then + SHOULD_SEND=true + log "Profile: weekly — today is $DIGEST_DAY, will send" + else + info "Profile: weekly — today is $TODAY_NAME, digest day is $DIGEST_DAY — skipping" + exit 0 + fi + ;; + smart) + log "Profile: smart — will evaluate findings before deciding" + SHOULD_SEND=false # determined after gathering data + ;; + *) + warn "Unknown DIGEST_PROFILE: $DIGEST_PROFILE — defaulting to weekly behavior" + TODAY_NAME=$(date '+%A') + [[ "$TODAY_NAME" == "$DIGEST_DAY" ]] && SHOULD_SEND=true + ;; +esac + +# ----------------------------------------------------------------------------------------------- +# DATA GATHERING +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_DIGEST Gathering System Data ━━━" + +FINDINGS=() # things worth noting +ISSUES=() # things that need attention +DIGEST_LINES=() # full report lines + +# ── Failover State ────────────────────────────────────────────────────────────────────────── +if [[ -f "$FAILOVER_STATE_FILE" ]]; then + FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) + FAILOVER_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) + DIGEST_LINES+=("$ICON_FAILOVER Failover: $FAILOVER_STATE (last change: ${FAILOVER_CHANGE:-unknown})") + + if [[ "$FAILOVER_STATE" != "NORMAL" && -n "$FAILOVER_STATE" ]]; then + ISSUES+=("Failover state: $FAILOVER_STATE") + [[ "$DIGEST_SMART_ON_FAILOVER" == true ]] && SHOULD_SEND=true + fi +else + DIGEST_LINES+=("$ICON_FAILOVER Failover: state file not found") +fi + +# ── Transcode Ramdisk ─────────────────────────────────────────────────────────────────────── +if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then + RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ') + RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}") + RAMDISK_AVAIL_KB=$(df "$RAMDISK_PATH" --output=avail | tail -1 | tr -d ' ') + RAMDISK_AVAIL_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_AVAIL_KB / 1048576}") + SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown") + DIGEST_LINES+=("$ICON_RAM Transcodes: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB free → $SYMLINK_TARGET") +else + DIGEST_LINES+=("$ICON_RAM Transcodes: ramdisk not mounted") + ISSUES+=("Ramdisk not mounted") +fi + +# ── Container Watchdog Strikes ────────────────────────────────────────────────────────────── +if [[ -f "$WATCHDOG_STATE_FILE" ]]; then + ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | wc -l) + if [[ "$ACTIVE_STRIKES" -gt 0 ]]; then + STRIKE_LIST=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ') + DIGEST_LINES+=("$ICON_WATCHDOG Container strikes: $ACTIVE_STRIKES active — $STRIKE_LIST") + FINDINGS+=("Container watchdog: $ACTIVE_STRIKES active strikes") + [[ "$DIGEST_SMART_ON_WATCHDOG" == true ]] && SHOULD_SEND=true + else + DIGEST_LINES+=("$ICON_WATCHDOG Container watchdog: no active strikes") + fi +fi + +# ── System Watchdog Strikes ───────────────────────────────────────────────────────────────── +if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then + SYS_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | wc -l) + if [[ "$SYS_STRIKES" -gt 0 ]]; then + SYS_STRIKE_LIST=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ') + DIGEST_LINES+=("$ICON_REBOOT_SMART System strikes: $SYS_STRIKES active — $SYS_STRIKE_LIST") + FINDINGS+=("System watchdog: $SYS_STRIKES active strikes") + [[ "$DIGEST_SMART_ON_WATCHDOG" == true ]] && SHOULD_SEND=true + else + DIGEST_LINES+=("$ICON_REBOOT_SMART System watchdog: no active strikes") + fi +fi + +# ── Container Skip List ───────────────────────────────────────────────────────────────────── +if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then + SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE") + SKIP_LIST=$(cat "$SYS_WATCHDOG_FAILED_FILE" | tr '\n' ' ') + DIGEST_LINES+=("$ICON_NOT_RUNNING Skip list: $SKIP_COUNT containers — $SKIP_LIST") + ISSUES+=("Containers on skip list: $SKIP_LIST") + SHOULD_SEND=true +else + DIGEST_LINES+=("$ICON_RUNNING Skip list: empty — all containers healthy") +fi + +# ── Bandwidth (yesterday's total) ─────────────────────────────────────────────────────────── +if [[ -f "$BANDWIDTH_LOG" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then + YESTERDAY=$(date -d "yesterday" '+%Y-%m-%d') + YESTERDAY_BYTES=$(awk -F'|' -v d="$YESTERDAY" '$1==d{sum+=$3} END{print sum+0}' "$BANDWIDTH_LOG") + YESTERDAY_GB=$(awk "BEGIN {printf \"%.2f\", $YESTERDAY_BYTES / 1073741824}") + OVER_WARN=$(awk "BEGIN {print ($YESTERDAY_BYTES > $BANDWIDTH_WARN_GB * 1073741824) ? 1 : 0}") + + if [[ "$OVER_WARN" == "1" ]]; then + DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday's transfers: ${YESTERDAY_GB}GB ← exceeded ${BANDWIDTH_WARN_GB}GB threshold") + FINDINGS+=("High bandwidth day: ${YESTERDAY_GB}GB transferred") + [[ "$DIGEST_SMART_ON_BANDWIDTH" == true ]] && SHOULD_SEND=true + else + DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday's transfers: ${YESTERDAY_GB}GB") + fi +else + DIGEST_LINES+=("$ICON_BANDWIDTH Bandwidth: no data yet") +fi + +# ── SSL Certificates ──────────────────────────────────────────────────────────────────────── +if [[ ${#CERT_MONITOR_DOMAINS[@]} -gt 0 ]] && command -v openssl >/dev/null 2>&1; then + CERT_ISSUES=() + for domain in "${CERT_MONITOR_DOMAINS[@]}"; do + [[ -z "$domain" ]] && continue + expiry_str=$(echo | timeout "$CERT_TIMEOUT" openssl s_client \ + -connect "${domain}:443" -servername "$domain" \ + 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) + + if [[ -n "$expiry_str" ]]; then + expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null) + days_remaining=$(( (expiry_epoch - $(date +%s)) / 86400 )) + + if [[ "$days_remaining" -le "$CERT_CRIT_DAYS" ]]; then + CERT_ISSUES+=("$domain: ${days_remaining}d CRITICAL") + SHOULD_SEND=true + elif [[ "$days_remaining" -le "$CERT_WARN_DAYS" ]]; then + CERT_ISSUES+=("$domain: ${days_remaining}d WARNING") + [[ "$DIGEST_SMART_ON_CERT_WARN" == true ]] && SHOULD_SEND=true + fi + fi + done + + if [[ ${#CERT_ISSUES[@]} -gt 0 ]]; then + DIGEST_LINES+=("$ICON_CERT Certificates: ${CERT_ISSUES[*]}") + FINDINGS+=("Cert issues: ${CERT_ISSUES[*]}") + else + DIGEST_LINES+=("$ICON_CERT Certificates: all healthy") + fi +fi + +# ── Smart profile final decision ──────────────────────────────────────────────────────────── +if [[ "$DIGEST_PROFILE" == "smart" && "$SHOULD_SEND" == false ]]; then + info "Profile: smart — no findings worth reporting — skipping notification" + echo "" + echo "━━━━━ $ICON_SUMMARY DIGEST SUMMARY ━━━━━" + echo "$ICON_SUCCESS Everything looks healthy — no digest sent (smart profile)" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_DIGEST Build and Send Digest ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_DIGEST Health Digest ━━━" + +DIGEST_DATE=$(date '+%Y-%m-%d %H:%M:%S') +DIGEST_HOST=$(hostname) + +# Build notification message +NOTIFY_MSG="Health Digest — $DIGEST_HOST — $DIGEST_DATE" +if [[ ${#ISSUES[@]} -gt 0 ]]; then + NOTIFY_MSG+=" | Issues: ${ISSUES[*]}" +fi +if [[ ${#FINDINGS[@]} -gt 0 ]]; then + NOTIFY_MSG+=" | Findings: ${FINDINGS[*]}" +fi + +# Print full digest to console +for line in "${DIGEST_LINES[@]}"; do + echo " $line" +done + +echo "" +echo "━━━━━ $ICON_SUMMARY DIGEST SUMMARY ━━━━━" +echo "$ICON_DIGEST Profile: $DIGEST_PROFILE" +echo "$ICON_TIME Generated: $DIGEST_DATE" +echo "$ICON_ERROR Issues: ${#ISSUES[@]}" +echo "$ICON_WARN Findings: ${#FINDINGS[@]}" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — digest generated but not sent" +elif [[ "$SHOULD_SEND" == true ]]; then + notify "$NOTIFY_MSG" "Health Digest" "$([[ ${#ISSUES[@]} -gt 0 ]] && echo "warning" || echo "normal")" + success "Digest sent" +fi \ No newline at end of file diff --git a/common.sh b/common.sh index f358b7c..13a28de 100644 --- a/common.sh +++ b/common.sh @@ -2,7 +2,7 @@ # ----------------------------------------------------------------------------------------------- # ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ---------------------------- # ----------------------------------------------------------------------------------------------- -# Version: 2.7 +# Version: 2.9 # ----------------------------------------------------------------------------------------------- # Changelog: # v1.0 — Initial stable framework @@ -51,11 +51,20 @@ # ping_internet added — non-fatal external ping for failover use # v2.7 — ICON_WEBGUI added for WebGUI watchdog operations # ICON_DOCKER_NET added for Docker network connect operations +# v2.8 — ICON_CERT added for SSL certificate monitoring operations +# v2.9 — ICON_MONITOR added for monitoring section headers +# ICON_SMART added for drive SMART health operations +# ICON_BANDWIDTH added for bandwidth tracking operations +# ICON_DIGEST added for health digest operations +# ICON_EMBY added for Emby session reporting +# ICON_VERIFY added for backup verification operations +# Monitor/ folder added to ecosystem # ----------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------- # ICONS # Each icon has one job — do not reuse across different contexts. +# Adding a new icon: add it to the appropriate group below with a comment describing its job. # ----------------------------------------------------------------------------------------------- # System / Host @@ -117,6 +126,17 @@ ICON_FAILOVER="🔀" # failover state changes and operations # Docker Network Operations ICON_DOCKER_NET="🔌" # Docker network connect operations +# Security / Certificate Operations +ICON_CERT="🔒" # SSL certificate monitoring + +# Monitor Operations +ICON_MONITOR="📈" # monitoring section headers and general monitoring +ICON_SMART="🔧" # drive SMART health attribute monitoring +ICON_BANDWIDTH="📶" # bandwidth usage tracking and reporting +ICON_DIGEST="📰" # health digest — aggregated system summary +ICON_EMBY="🎬" # Emby media server session reporting +ICON_VERIFY="✔️" # backup verification — checksum comparison + # Notifications ICON_NOTIFY="🔔" # notification operations @@ -128,6 +148,8 @@ ICON_SUCCESS="✅" # ----------------------------------------------------------------------------------------------- # OUTPUT HELPERS +# Standardised output functions used across all scripts. +# log() is gated by ENABLE_LOGGING — set in Master.conf or via --log flag. # ----------------------------------------------------------------------------------------------- info() { echo "$ICON_INFO [INFO] $*"; } warn() { echo "$ICON_WARN [WARN] $*"; } @@ -140,8 +162,10 @@ log() { # ----------------------------------------------------------------------------------------------- # NOTIFICATION +# Sends a notification via unRAID native system and/or Discord webhook. +# Both channels are optional and independently controlled via Master.conf. +# Severity levels: normal, warning, alert # Usage: notify "message" "subject" "severity" -# Severity: normal, warning, alert # ----------------------------------------------------------------------------------------------- notify() { local message="$1" @@ -175,6 +199,7 @@ notify() { # ----------------------------------------------------------------------------------------------- # DURATION FORMATTER +# Converts raw seconds into a human readable string — e.g. 10m53s or 47s # ----------------------------------------------------------------------------------------------- format_duration() { local secs=$1 @@ -185,6 +210,10 @@ format_duration() { # ----------------------------------------------------------------------------------------------- # ARG PARSER +# Processes all flags and key=value pairs passed to any script. +# Supported flags: --dry-run, --log, --no-log, --status, --help +# Supported key=value: LOG=true/false, or any declared variable e.g. BW_LIMIT=5000 +# Unparsed positional args returned in PARSED_ARGS array. # ----------------------------------------------------------------------------------------------- parse_args() { ENABLE_LOGGING=${ENABLE_LOGGING:-false} @@ -229,12 +258,17 @@ parse_args() { } # ----------------------------------------------------------------------------------------------- -# VALIDATION +# VALIDATION HELPERS # ----------------------------------------------------------------------------------------------- + +# Exits with error if a required variable is empty or unset. +# Usage: require_var VAR_NAME require_var() { [[ -z "${!1:-}" ]] && error "Missing required: $1" && exit 1 } +# Exits with error if a variable is not a valid positive integer. +# Usage: validate_int VAR_NAME "$VAR_VALUE" validate_int() { local name="$1" value="$2" if [[ -z "$value" ]]; then @@ -250,6 +284,9 @@ validate_int() { # ----------------------------------------------------------------------------------------------- # HOST DETECTION +# Determines which server is local and which is remote by comparing hostname against +# HOST1 and HOST2 in Master.conf. Sets LOCAL_SERVER_NAME, REMOTE_SERVER_NAME and SSH_KEY. +# Both servers run identical scripts — this is what makes them bidirectional. # ----------------------------------------------------------------------------------------------- detect_hosts() { LOCAL_HOSTNAME="$(hostname)" @@ -271,12 +308,14 @@ detect_hosts() { SSH_KEY="${SSH_KEYS[$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME]}" [[ -z "$SSH_KEY" ]] && error "Missing SSH key mapping" && exit 1 - info "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" } # ----------------------------------------------------------------------------------------------- # REMOTE IP RESOLUTION +# Resolves the Tailscale IPv4 address of the remote server. +# Sets REMOTE_SERVER used by all subsequent SSH and rsync calls. +# Exits if resolution fails — Tailscale may be down or peer offline. # ----------------------------------------------------------------------------------------------- resolve_remote_ip() { log "Resolving remote IP for $REMOTE_SERVER_NAME..." @@ -287,6 +326,8 @@ resolve_remote_ip() { # ----------------------------------------------------------------------------------------------- # CONNECTIVITY CHECK — fatal, used by rsync scripts +# Pings remote and exits if unreachable. +# For failover use ping_remote() which returns status without exiting. # ----------------------------------------------------------------------------------------------- check_connectivity() { log "Checking connectivity to $REMOTE_SERVER..." @@ -300,15 +341,15 @@ check_connectivity() { # ----------------------------------------------------------------------------------------------- # PING REMOTE — non-fatal, used by failover -# Returns 0 if reachable, 1 if not +# Returns 0 if reachable, 1 if not — does NOT exit. # ----------------------------------------------------------------------------------------------- ping_remote() { ping -c2 -W3 "$REMOTE_SERVER" &>/dev/null } # ----------------------------------------------------------------------------------------------- -# PING INTERNET — non-fatal, used by failover -# Returns 0 if internet reachable, 1 if not +# PING INTERNET — non-fatal external connectivity check +# Returns 0 if internet reachable, 1 if not — does NOT exit. # ----------------------------------------------------------------------------------------------- ping_internet() { ping -c2 -W3 "${EXTERNAL_IP:-8.8.8.8}" &>/dev/null @@ -316,7 +357,9 @@ ping_internet() { # ----------------------------------------------------------------------------------------------- # LOCAL ARRAY CHECK — non-fatal, returns status -# Used by failover before starting remote containers locally +# Verifies local /mnt/user is mounted and has shares. +# Used by failover before starting remote containers locally. +# Returns 0 if healthy, 1 if not. # ----------------------------------------------------------------------------------------------- check_local_array() { log "Checking local array..." @@ -336,7 +379,9 @@ check_local_array() { # ----------------------------------------------------------------------------------------------- # REMOTE ARRAY CHECK — non-fatal, returns status -# Used before handback rsync +# Verifies remote /mnt/user is mounted via SSH. +# Used before handback rsync — syncing to remote with no array fills rootfs. +# Returns 0 if healthy, 1 if not. # ----------------------------------------------------------------------------------------------- check_remote_array() { log "Checking remote array on $REMOTE_SERVER_NAME..." @@ -353,7 +398,9 @@ check_remote_array() { # ----------------------------------------------------------------------------------------------- # REMOTE DOCKER CHECK — non-fatal, returns status -# Used before starting containers on remote +# Verifies remote Docker daemon is responding before container operations. +# A hung daemon means start/stop commands will silently fail. +# Returns 0 if healthy, 1 if not. # ----------------------------------------------------------------------------------------------- check_remote_docker() { log "Checking remote Docker daemon on $REMOTE_SERVER_NAME..." @@ -368,28 +415,26 @@ check_remote_docker() { # ----------------------------------------------------------------------------------------------- # REMOTE ROOTFS SPACE CHECK — fatal +# Aborts if remote rootfs exceeds ROOTFS_WARN threshold. # ----------------------------------------------------------------------------------------------- check_remote_rootfs() { log "Checking remote rootfs usage..." REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null) - if [[ -z "$REMOTE_USAGE" ]]; then error "Could not retrieve rootfs usage from $REMOTE_SERVER_NAME" exit 1 fi - if [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then error "$ICON_HEALTH Remote rootfs ${REMOTE_USAGE}% — threshold ${ROOTFS_WARN:-75}%" - warn "Array may be down or drives missing on $REMOTE_SERVER_NAME" exit 1 fi - info "$ICON_HEALTH Remote rootfs: ${REMOTE_USAGE}% (threshold: ${ROOTFS_WARN:-75}%)" } # ----------------------------------------------------------------------------------------------- # REMOTE SHARE VALIDATION — fatal +# Verifies target directory exists and is not empty on remote. # Usage: check_remote_share "/mnt/user/Movies" # ----------------------------------------------------------------------------------------------- check_remote_share() { @@ -398,7 +443,6 @@ check_remote_share() { SHARE_EXISTS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null) - if [[ "$SHARE_EXISTS" != "yes" ]]; then error "$ICON_HEALTH Remote share does not exist: $dir" exit 1 @@ -406,7 +450,6 @@ check_remote_share() { SHARE_EMPTY=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null) - if [[ "$SHARE_EMPTY" == "yes" ]]; then warn "$ICON_HEALTH Remote share exists but is empty: $dir — aborting to protect data" exit 1 @@ -417,6 +460,8 @@ check_remote_share() { # ----------------------------------------------------------------------------------------------- # REMOTE DISK CHECK — fatal +# Verifies all physical disks backing a share are online on the remote server. +# Skipped when PROFILE_SKIP_DISK_CHECK is true (ZFS pools have no /mnt/disk* structure). # Usage: check_remote_disks "/mnt/user/Movies" # ----------------------------------------------------------------------------------------------- check_remote_disks() { @@ -439,10 +484,8 @@ check_remote_disks() { local disk_mount disk_name disk_mount=$(dirname "$disk_share_path") disk_name=$(basename "$disk_mount") - MOUNTED=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "mountpoint -q '$disk_mount' && echo yes || echo no" 2>/dev/null) - if [[ "$MOUNTED" == "yes" ]]; then info "$ICON_DISK $disk_name $ICON_RUNNING — $share_name present" else @@ -455,12 +498,13 @@ check_remote_disks() { error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME" exit 1 fi - success "All disks backing $share_name are online" } # ----------------------------------------------------------------------------------------------- # CONTAINER MANAGEMENT — STOP (remote via SSH) +# Stops containers in CRITICAL_CONTAINER_NAMES on remote server. +# Tracks running containers in RUNNING_CONTAINERS for restart after rsync. # ----------------------------------------------------------------------------------------------- RUNNING_CONTAINERS=() @@ -470,15 +514,12 @@ stop_containers() { log "No containers configured for this profile, skipping stop." return fi - info "Stopping containers..." RUNNING_CONTAINERS=() - for c in "${CRITICAL_CONTAINER_NAMES[@]}"; do [[ -z "$c" ]] && continue STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "docker inspect -f '{{.State.Running}}' $c 2>/dev/null" 2>/dev/null || echo "false") - if [[ "$STATUS" == "true" ]]; then echo "$ICON_STOP Stopping $c..." RUNNING_CONTAINERS+=("$c") @@ -495,28 +536,25 @@ stop_containers() { # ----------------------------------------------------------------------------------------------- # CONTAINER MANAGEMENT — START (remote via SSH) +# Restarts only containers tracked in RUNNING_CONTAINERS. +# Delayed containers receive CONTAINER_DELAY seconds before starting. # ----------------------------------------------------------------------------------------------- start_containers() { if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then log "No containers to restart." return fi - info "Starting containers..." - for c in "${RUNNING_CONTAINERS[@]}"; do [[ -z "$c" ]] && continue - local needs_delay=false for d in "${DELAYED_CONTAINERS[@]}"; do [[ "$c" == "$d" ]] && needs_delay=true && break done - if [[ "$needs_delay" == true ]]; then info "Waiting ${CONTAINER_DELAY}s before starting $c..." sleep "$CONTAINER_DELAY" fi - echo "$ICON_START Starting $c..." if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $c" >/dev/null; then echo "$ICON_STARTED $c started" @@ -528,6 +566,8 @@ start_containers() { # ----------------------------------------------------------------------------------------------- # RSYNC OPTIONS +# Loads rsync options for current profile. Falls back to DEFAULT_RSYNC_OPTS if no match. +# Profile opts do NOT inherit from defaults — list all desired flags explicitly. # ----------------------------------------------------------------------------------------------- get_rsync_opts() { if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then @@ -541,6 +581,7 @@ get_rsync_opts() { # ----------------------------------------------------------------------------------------------- # STATUS DISPLAY +# Prints current runtime configuration — triggered by --status flag. # ----------------------------------------------------------------------------------------------- show_status() { echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" diff --git a/user_script_pluin.sh b/user_script_pluin.sh index 61237c1..1e2a780 100644 --- a/user_script_pluin.sh +++ b/user_script_pluin.sh @@ -7,30 +7,26 @@ # Copy and paste the contents of this file into a new User Script entry in the plugin, # then uncomment the script you want to run and set your schedule. # -# All scripts in this ecosystem live at: -# /mnt/user/appdata/unraid_scripts/ -# -# Configuration for all scripts is managed in one place: -# /mnt/user/appdata/unraid_scripts/Master.conf -# -# Shared runtime functions used by all scripts: -# /mnt/user/appdata/unraid_scripts/common.sh +# All scripts live at: /mnt/user/appdata/unraid_scripts/ +# All configuration at: /mnt/user/appdata/unraid_scripts/Master.conf +# Shared functions at: /mnt/user/appdata/unraid_scripts/common.sh # # ============================================================================================== # Changelog: # v1.0 — Initial template -# v1.1 — MAX_RSYNC_PROCS removed — bandwidth limiting handles concurrency -# Typo fixes, --status flag added, full directory tree, changelog added +# v1.1 — MAX_RSYNC_PROCS removed, --status flag, full directory tree, changelog # v1.2 — Docker Essentials, Media, Transcodes, System Watchdog added +# v1.3 — Failover script added +# v1.4 — WebGUI watchdog, ZFS snapshot, Docker network connect, schedules section +# v1.5 — media_management.sh orchestrator, corrected filenames, git repo section +# v1.6 — lidarr_cleanup.sh, sonarr_cleanup.sh, radarr_cleanup.sh added +# v1.7 — cert_monitor.sh added +# v1.8 — Monitor/ folder added with all monitoring scripts +# cert_monitor.sh moved from unRAID_Essentials to Monitor/ +# backup_verify.sh, smart_health.sh, bandwidth_monitor.sh added +# weekly_health_digest.sh, emby_session_report.sh added +# ZFS memory snapshot moved to Monitor/ — informational only # Directory tree updated to reflect full ecosystem -# v1.3 — Failover script added, directory tree updated with Failover folder -# v1.4 — WebGUI watchdog, ZFS memory snapshot, Docker network connect added -# Recommended schedules section added -# v1.5 — media_management.sh orchestrator added to Orchestrators folder -# Corrected filenames: media_shares_permissions.sh, user_scripts_stop.sh -# Git repo section added -# Individual media cleaner entries replaced by media_management.sh -# Directory tree updated # ============================================================================================== # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ @@ -47,9 +43,18 @@ # ├── Failover/ # │ └── failover.sh # Mutual container failover — runs continuously # │ +# ├── Monitor/ +# │ ├── backup_verify.sh # Random sample checksum verification vs remote +# │ ├── cert_monitor.sh # SSL certificate expiry — direct openssl check +# │ ├── emby_session_report.sh # Weekly Emby usage statistics via API +# │ ├── smart_health.sh # Drive SMART attribute monitoring +# │ ├── bandwidth_monitor.sh # Daily transfer logging + weekly summary +# │ ├── weekly_health_digest.sh # Aggregated system health — always/smart/weekly +# │ └── zfs_memory_snapshot.sh # Weekly ZFS health and memory diagnostic report +# │ # ├── Orchestrators/ # │ ├── daily_sync.sh # Runs all daily media share syncs sequentially -# │ └── media_management.sh # Runs permissions + anime + media cleaner sequentially +# │ └── media_management.sh # Runs permissions + cleaners + arr cleanup sequentially # │ # ├── Rsync/ # │ ├── rsync.sh # Core rsync script — called per share or profile @@ -63,11 +68,14 @@ # │ # ├── Media/ # │ ├── media_shares_permissions.sh # Applies permissions to all media shares -# │ └── media_cleaner.sh # Removes junk files — profiles: anime, media +# │ ├── media_cleaner.sh # Removes junk files — profiles: anime, media +# │ ├── lidarr_cleanup.sh # Removes orphaned music files via Lidarr API +# │ ├── sonarr_cleanup.sh # Removes orphaned TV files via Sonarr API +# │ └── radarr_cleanup.sh # Removes orphaned movie files via Radarr API # │ # ├── Transcodes/ -# │ ├── ramdisk_setup.sh # Creates ramdisk and transcode symlink — run at array start -# │ ├── transcode_manager.sh # Monitors ramdisk usage, manages symlink direction +# │ ├── ramdisk_setup.sh # Creates ramdisk and transcode symlink +# │ ├── transcode_manager.sh # Monitors ramdisk usage, manages symlink # │ └── transcode_cleanup.sh # Removes old inactive transcode files # │ # ├── Tools/ @@ -82,34 +90,39 @@ # ├── server_reboot.sh # Graceful server reboot with user warning # ├── system_watchdog.sh # System health monitor — last line of defense # ├── user_scripts_stop.sh # Stops running User Scripts plugin jobs -# ├── webgui_restart.sh # WebGUI watchdog — nginx + emhttp escalating restart -# └── zfs_memory_snapshot.sh # Weekly ZFS health and memory diagnostic report +# └── webgui_restart.sh # WebGUI watchdog — nginx + emhttp restart # # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # ━━━ 🚀 Script Commands — Uncomment the one you want to run ━━━ # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # # ━━━ Failover (README) ━━━ -# Runs continuously as a background task — do NOT schedule with a time interval. -# Set schedule to "At Startup of Array" — background task. -# Both servers must have this script running for mutual failover to work. -# Use --status to check current state without restarting the loop. +# Runs continuously as background task — set "At Startup of Array" as background task. +# Both servers must have this running for mutual failover to work. # -# ━━━ Failover ━━━ #/mnt/user/appdata/unraid_scripts/Failover/failover.sh # +# ━━━ Monitor ━━━ +#/mnt/user/appdata/unraid_scripts/Monitor/backup_verify.sh +#/mnt/user/appdata/unraid_scripts/Monitor/cert_monitor.sh +#/mnt/user/appdata/unraid_scripts/Monitor/emby_session_report.sh +#/mnt/user/appdata/unraid_scripts/Monitor/smart_health.sh +#/mnt/user/appdata/unraid_scripts/Monitor/bandwidth_monitor.sh +#/mnt/user/appdata/unraid_scripts/Monitor/weekly_health_digest.sh +#/mnt/user/appdata/unraid_scripts/Monitor/zfs_memory_snapshot.sh +# # ━━━ Orchestrators ━━━ #/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync.sh #/mnt/user/appdata/unraid_scripts/Orchestrators/media_management.sh # -# ━━━ Rsync — Appdata Profiles (scheduled individually) ━━━ +# ━━━ Rsync — Appdata Profiles ━━━ #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Critical-Data #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Important-Data -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Media_Server/Emby +#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Emby #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Gmer4Lfe # -# ━━━ Rsync — Individual Media Shares (ad hoc use) ━━━ +# ━━━ Rsync — Individual Media Shares (ad hoc) ━━━ #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Movies #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Tv_Shows #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Shows @@ -148,51 +161,68 @@ #/mnt/user/appdata/unraid_scripts/unRAID_Essentials/system_watchdog.sh #/mnt/user/appdata/unraid_scripts/unRAID_Essentials/user_scripts_stop.sh #/mnt/user/appdata/unraid_scripts/unRAID_Essentials/webgui_restart.sh -#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/zfs_memory_snapshot.sh # # ━━━ Git repo (Gitea) ━━━ #/mnt/user/appdata/unraid_scripts/git_pull_execute.sh # # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ━━━ ⚙️ Arguments — Add after the script path ━━━ +# ━━━ ⚙️ Arguments ━━━ # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # # --dry-run Preview what would happen — no changes made # --log Enable verbose logging output -# --no-log Disable logging (overrides Master.conf setting) -# --status Print resolved configuration and current state then exit +# --no-log Disable logging (overrides Master.conf) +# --status Print resolved config and current state then exit # --help Show usage information # KEY=VALUE Override any Master.conf variable for this run only # -# ━━━ Failover specific ━━━ -# failover.sh --status — check current state -# failover.sh --dry-run --log — test logic without touching containers +# ━━━ Special arguments ━━━ +# failover.sh --status — check current state without restarting loop +# failover.sh --dry-run --log — test logic without touching containers +# media_cleaner.sh anime --dry-run — profile required as first argument +# bandwidth_monitor.sh --report — generate weekly summary report +# lidarr/sonarr/radarr --dry-run --log — always test arr cleanup first # # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # ━━━ 📋 Recommended Schedules ━━━ # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # -# failover.sh — At Startup of Array (background task) -# ramdisk_setup.sh — At Startup of Array -# docker_network_connect.sh — At Startup of Array -# docker_syslog_filter.sh — At Startup of Array -# php_fpm_max_children.sh — At Startup of Array +# ━━━ At Startup of Array ━━━ +# failover.sh — background task +# ramdisk_setup.sh +# docker_network_connect.sh +# docker_syslog_filter.sh +# php_fpm_max_children.sh # -# transcode_manager.sh — */3 * * * * (every 3 minutes) -# transcode_cleanup.sh — */5 * * * * (every 5 minutes) -# docker_watchdog.sh — */15 * * * * (every 15 minutes) -# webgui_restart.sh — */10 * * * * (every 10 minutes) -# system_watchdog.sh — */15 * * * * (every 15 minutes) +# ━━━ Frequent (every few minutes) ━━━ +# */3 * * * * transcode_manager.sh +# */5 * * * * transcode_cleanup.sh +# */10 * * * * webgui_restart.sh +# */15 * * * * docker_watchdog.sh +# */15 * * * * system_watchdog.sh # -# daily_sync.sh — 0 1 * * * (daily at 1am) -# docker_daily_restart.sh — 0 3 * * * (daily at 3am) -# media_management.sh — 0 2 * * * (daily at 2am) +# ━━━ Daily ━━━ +# 0 1 * * * daily_sync.sh +# 0 2 * * * media_management.sh +# 0 3 * * * docker_daily_restart.sh +# 0 8 * * * weekly_health_digest.sh (profile controls if it sends) # -# clear_logs.sh — 0 5 * * 0 (weekly Sunday 5am) -# zfs_memory_snapshot.sh — 0 6 * * 0 (weekly Sunday 6am) -# docker_weekly_restart.sh — 0 3 * * 0 (weekly Sunday 3am) +# ━━━ Weekly — Sunday morning block ━━━ +# 0 3 * * 0 docker_weekly_restart.sh +# 0 5 * * 0 clear_logs.sh +# 0 6 * * 0 zfs_memory_snapshot.sh +# 0 7 * * 0 smart_health.sh +# 0 8 * * 0 weekly_health_digest.sh (weekly profile sends today) +# 0 9 * * 0 cert_monitor.sh +# 0 10 * * 0 backup_verify.sh +# 0 11 * * 0 emby_session_report.sh # -# rsync.sh profiles — schedule individually as needed +# ━━━ Rsync profiles ━━━ +# Schedule individually as needed +# +# ━━━ Bandwidth monitor ━━━ +# bandwidth_monitor.sh --log-transfer is called automatically by rsync.sh +# bandwidth_monitor.sh --report runs standalone for weekly summary # # ============================================================================================== \ No newline at end of file