From ec7de648dc7b335c56deeabe2696b873a4b5fc01 Mon Sep 17 00:00:00 2001 From: FailedProxy Date: Sun, 3 May 2026 17:16:49 -0400 Subject: [PATCH] massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2 --- .vscode/settings.json | 10 + Docker_Essentials/README-Docker_Essentials.md | 1414 ++++++++++--- Docker_Essentials/docker_daily_restart.sh | 299 ++- Docker_Essentials/docker_network_connect.sh | 1551 ++------------ Docker_Essentials/docker_watchdog.sh | 650 ++++-- Docker_Essentials/docker_weekly_restart.sh | 258 ++- Docker_Essentials/downloaders_reset.sh | 249 ++- Failover/README-Failover.md | 1488 +++++++++---- Failover/failover.sh | 623 +++--- Failover/failover_test.sh | 334 +-- Media/README-Media.md | 1160 +++++++--- Media/arrs_failed_stalled_recovery.sh | 407 ++-- Media/lidarr_cleanup.sh | 478 +++-- Media/media_cleaner.sh | 178 +- Media/media_shares_permissions.sh | 230 +- Media/radarr_cleanup.sh | 462 ++-- Media/sonarr_cleanup.sh | 453 ++-- Monitors/README-Monitors.md | 1175 ++++++++--- Monitors/backup_verify.sh | 261 ++- Monitors/bandwidth_monitor.sh | 198 +- Monitors/cert_monitor.sh | 190 +- Monitors/continuous_scripts_status.sh | 280 +-- Monitors/emby_session_report.sh | 292 ++- Monitors/smart_health.sh | 290 ++- Monitors/system_tuning_monitor.sh | 205 ++ Monitors/weekly_health_digest.sh | 366 ++-- Monitors/zfs_memory_snapshot.sh | 237 ++- Notes_To-Do.md | 40 + Orchestrators/README-Orchestrators.md | 1166 ++++++++--- Orchestrators/array_start.sh | 208 +- Orchestrators/critical_sync_maintenance.sh | 230 +- Orchestrators/daily_sync_maintenance.sh | 455 ++-- Orchestrators/sunday_morning_coffee_report.sh | 945 ++++----- Orchestrators/transcode_management.sh | 271 ++- Orchestrators/weekly_sync_maintenance.sh | 418 ++-- Partnership/README-Partnership.md | 1201 ++++++++--- Partnership/partnership_manager.sh | 578 ++--- README-User_Script_Plug-in.md | 1862 ++++++++++++++--- README.md | 1325 ++++++------ Rsync/README-Rsync_Setup.md | 1483 ++++++++----- Rsync/rsync.sh | 336 ++- Tools/README-Tools.md | 960 +++++++-- Tools/bulk_permissions_repair.sh | 187 +- Tools/container_data_export.sh | 283 ++- Tools/emby_database_repair.sh | 351 ++-- Tools/failover_state_reset.sh | 234 ++- Tools/recreate_shares.sh | 325 ++- Tools/watchdog_skip_list_manager.sh | 236 ++- Tools/zfs_pool_scrub.sh | 266 ++- Transcodes/README-Transcoding.md | 938 ++++++--- Transcodes/ramdisk_setup.sh | 282 ++- Transcodes/transcode_cleanup.sh | 282 +-- Transcodes/transcode_manager.sh | 488 ++--- common.sh | 1204 +++++++---- git_pull_execute.sh | 204 +- load_config.sh | 97 + Master.conf => master.conf | 1031 ++++----- master_host1.conf | 646 ++++++ master_host2.conf | 545 +++++ safe_master.conf | 1518 -------------- unRAID_Essentials/README-Unraid_Essentials.md | 1012 ++++++--- unRAID_Essentials/clear_logs.sh | 324 ++- unRAID_Essentials/docker_syslog_filter.sh | 267 ++- unRAID_Essentials/inotify_tuning.sh | 231 +- unRAID_Essentials/mover_stop.sh | 212 +- unRAID_Essentials/php_fpm_max_children.sh | 287 ++- unRAID_Essentials/rsync_stop.sh | 459 ++-- unRAID_Essentials/server_reboot.sh | 377 ++-- unRAID_Essentials/system_watchdog.sh | 778 +++++-- unRAID_Essentials/user_scripts_stop.sh | 289 ++- unRAID_Essentials/webgui_restart.sh | 345 +-- user_script_plug-in.sh | 1355 +++++++++--- 72 files changed, 25640 insertions(+), 14629 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 Notes_To-Do.md create mode 100644 load_config.sh rename Master.conf => master.conf (62%) create mode 100644 master_host1.conf create mode 100644 master_host2.conf delete mode 100644 safe_master.conf diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..22d54c7 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,10 @@ +{ + "files.exclude": { + "**/.cache/**": true, + "**/.next/**": true, + "**/build/**": true, + "**/coverage/**": true, + "**/dist/**": true, + "**/node_modules/**": true + } +} \ No newline at end of file diff --git a/Docker_Essentials/README-Docker_Essentials.md b/Docker_Essentials/README-Docker_Essentials.md index 602b01d..cd3a895 100644 --- a/Docker_Essentials/README-Docker_Essentials.md +++ b/Docker_Essentials/README-Docker_Essentials.md @@ -1,349 +1,1211 @@ -# Docker Essentials +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# 🐳 DOCKER ESSENTIALS +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Container lifecycle management — health monitoring, scheduled restarts, and network configuration. These scripts keep your Docker stack healthy, fresh, and correctly connected without manual intervention. +**Self-healing container lifecycle management for a 50+ container unRAID stack.** +Health monitoring that catches problems as they happen. Scheduled restarts that prevent +degradation before it becomes visible. Network configuration that survives reboots and +unRAID updates. Recovery tooling for when something genuinely breaks. + +> **Why this folder exists:** Docker on unRAID does not heal itself. A container that +> crashes stays crashed. A memory leak accumulates silently for days until the system +> starts swapping. A container whose dependency restarted first fails in a loop while +> the dependency comes up fine five seconds later. None of this surfaces clearly — it +> just builds into a system that feels flaky without a clear reason why. These scripts +> are the answer to all of that. + +--- + +## ━━━ THE PROBLEM THAT BUILT THIS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Running a large Docker stack on unRAID is genuinely powerful — but Docker's own tooling +gives you almost nothing between "container is running" and "container has been dead for +three days and you just noticed." + +The built-in restart policies help with outright crashes, but they have zero visibility +into memory leaks, frozen application layers, dependency ordering, restart loops, or the +difference between a container that crashed and one you intentionally stopped. They give +you no way to know *why* something keeps restarting — just that it does. + +These are the specific problems that led to building this, in roughly the order they were +encountered: + +--- + +### 🔴 Memory Leaks Accumulating Silently + +Emby's transcode session handling occasionally leaks memory — a session ends but its +memory allocation doesn't fully release. SABnzbd's Python process slowly expands over +days as it processes downloads. Without hard memory limits and automatic enforcement, +these don't fail dramatically — they just consume more and more RAM until the system +starts swapping and everything slows to a crawl. By then, nothing in the Docker logs +tells you why the system feels slow. `docker stats` shows a container at 22GB and +climbing, but Docker itself does nothing about it. + +The fix: hard per-container memory ceilings in `WATCHDOG_CONTAINERS`. When a container +exceeds its limit the watchdog restarts it immediately — no strikes, no waiting. A +memory leak is not a transient spike. Immediate action is correct. + +--- + +### 🔴 Containers That Look Running But Aren't Responding + +Docker reports a container as `Up 14 days` while its application layer has been silently +frozen for hours. The reverse proxy happily forwards traffic to a service that returns +nothing. Users see a broken page. Docker sees a healthy container. The container process +is technically running — it just isn't doing anything. + +Docker's built-in health checks require a `HEALTHCHECK` instruction in the image itself, +which most self-hosted images don't have. Even those that do often check something too +shallow — a process exists, not whether it's actually serving requests. + +The fix: HTTP health checks on the actual service port. `curl` to the real endpoint on +every watchdog cycle. If the service doesn't respond within `CURL_TIMEOUT` seconds, that's +a strike. Two consecutive failures trigger a restart. The distinction between "process +running" and "service responding" is the distinction that matters. + +--- + +### 🔴 Dependency Failures on Restart + +Authelia connects to MariaDB at startup. If both are down simultaneously — say, after a +power cut — and the watchdog restarts Authelia first, Authelia fails to connect, exits +immediately, and goes into a crash loop. Meanwhile MariaDB is coming up fine in the +background. The watchdog sees Authelia crash three times, adds it to the skip list, +and sends a critical notification. Authelia was never broken. It just came up in the +wrong order, failed at startup, and got punished for it. + +The fix: dependency ordering. If a container's dependency is also down, the dependent is +skipped entirely this cycle. The dependency is restarted first. On the next cycle — once +MariaDB is actually accepting connections — Authelia is restarted and comes up cleanly. +The skip list is never involved. No false alarms. No manual recovery needed. + +--- + +### 🔴 Restart Loops Corrupting State + +Some containers corrupt their internal state if restarted repeatedly in rapid succession. +SQLite databases that don't get a clean shutdown write incomplete transactions. Partially +applied database migrations leave schema in an inconsistent state. A container that +crashes on startup after a bad migration gets restarted immediately, crashes again, +gets restarted again — each restart has a chance of making the database worse, not better. + +A naive watchdog that just keeps hammering a crashed container is actively harmful in +this scenario. More restarts mean more corruption risk. The right response when restarts +aren't working is to stop restarting and alert the operator. + +The fix: restart loop protection. After `WATCHDOG_CONTAINER_RESTART_LIMIT` restarts +within a rolling `WATCHDOG_CONTAINER_RESTART_WINDOW` hour window, the container goes +on the skip list. A critical notification goes out. The watchdog stops touching it. +The operator investigates and clears the skip list once the underlying problem is fixed. + +--- + +### 🔴 Slow Degradation That Never Becomes a Failure + +NginxProxyManager accumulates stale entries in its connection table over weeks of uptime. +Dispatcharr's Live TV scheduler builds up internal scheduling state that makes decisions +progressively slower after months of continuous operation. These containers never crash. +They never throw errors. They just get progressively worse in ways that are hard to +attribute to anything specific — until someone notices that the proxy feels slower than +it used to, or that Live TV channel changes take longer than they should. + +The fix: scheduled restarts. Not because something is broken, but because some containers +simply perform better with a clean start. Daily at 1am for connection-heavy services. +Weekly for less-critical services that run fine for weeks but benefit from a clean slate. +Zero user impact — happens while everyone is asleep. + +--- + +### 🔴 Network Configuration Lost After Updates + +unRAID occasionally wipes custom Docker networks after updates — particularly networks +created by Docker Compose stacks or the NextCloud AIO container. Any container that +depended on those networks for internal communication suddenly can't reach its peers. +memcached can't talk to NextCloud. CrowdSec can't talk to NginxProxyManager. Services +appear to be up but silently fail to communicate with each other. + +The fix: network recreation at every array start. `docker_network_connect.sh` checks +every configured network on startup, creates any that are missing, and connects all +configured containers to them. Idempotent — if everything is already correct, it does +nothing and produces no output. If a network had to be created, it notifies — that only +happens after an update, and you want to know when it does. + +--- + +### 🔴 No Visibility Into What the Watchdog Already Tried + +A container keeps appearing in a broken state. You SSH in and see it's stopped. You +don't know if the watchdog tried to restart it and failed, gave up and skip-listed it, +is mid-attempt right now, or hasn't noticed yet. You have to manually check the skip +list file on `/boot/config/`, check the restart history file, check the state file — +none of which have obvious formats. + +The fix: `watchdog_skip_list_manager.sh`. One command to see exactly what's on the +skip list, which containers are stopped vs running, how many restarts were attempted, +and what the watchdog's current state is. One command to clear a specific container and +its history after you've fixed the problem. No manual file editing required. + +--- + +## ━━━ WHAT THIS FOLDER DOES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ``` -Monitors/ — observes containers, reports issues -Docker_Essentials/ — acts on containers (this folder) -unRAID_Essentials/ — acts on the server itself +Docker_Essentials/ ← acts on containers (this folder) +unRAID_Essentials/ ← acts on the server itself +Monitors/ ← observes, measures, reports +Rsync/ ← moves data between servers +``` + +Four distinct roles, each handled by dedicated scripts: + +--- + +### 🔁 Reactive Healing — `docker_watchdog.sh` + +Continuous two-tier monitoring that catches problems as they happen and acts on them +immediately. Runs as a background process started at array start. Every 15 minutes it +checks the full stack and fixes what it can — silently when everything is fine, visibly +when something needs attention. + +Two tiers because different containers need different monitoring strategies: +- **Tier 1** — explicit per-container configuration with specific thresholds +- **Tier 2** — global scan of everything that's running with catch-all health checks + +--- + +### ♻️ Proactive Freshness — `docker_daily_restart.sh` + `docker_weekly_restart.sh` + +Scheduled restarts that prevent slow degradation before it becomes visible. Not because +something broke — because some containers simply work better after a clean start. + +Called by the maintenance orchestrators (`daily_sync_maintenance.sh` and +`weekly_sync_maintenance.sh`) — not run standalone. They run inside the maintenance +windows so any downtime from restarts is absorbed by the window that's already happening. + +--- + +### 🌐 Network Integrity — `docker_network_connect.sh` + +Ensures custom Docker networks exist and containers are connected to them at every +array start. Silent when everything is correct. Notifies when it has to create something +— which means something was wiped and you should know about it. + +--- + +### 🔧 Recovery Tooling — `watchdog_skip_list_manager.sh` + +Manual tool for the moments when automatic recovery hasn't worked. Gives a clear picture +of what the watchdog has already tried, lets you clear the skip list cleanly after fixing +the root cause, and warns you if clearing might immediately re-add the container. + +--- + +## ━━━ RELATIONSHIP TO SYSTEM WATCHDOG ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Two watchdogs run simultaneously. They are designed to work together, not compete: + +``` +system_watchdog.sh ← watches the server: RAM, CPU, disk, kernel, daemon health +docker_watchdog.sh ← watches the containers: memory, CPU, HTTP response, crashes +``` + +**The coordination problem:** During a RAM emergency, `system_watchdog.sh` stops +non-essential containers to recover free memory. Without coordination, `docker_watchdog.sh` +would see stopped containers on its next cycle and restart them — directly undoing the +RAM recovery. The two watchdogs would fight indefinitely. RAM would never recover. +The system would eventually hit the reboot threshold anyway, having accomplished nothing. + +**The solution:** A shared state file at `SYS_WATCHDOG_STATE_FILE`. When +`system_watchdog.sh` triggers a RAM emergency shutdown it writes +`mem_shutdown_active=true`. `docker_watchdog.sh` reads this flag at the start of every +cycle and defers all container restart logic until it clears. Health URL checks for +excluded containers (DNS, auth, Emby, Dispatcharr — the ones that stayed running) still +run. Everything else stands down. + +**The stale state guard:** `system_watchdog.sh` writes `watchdog_cycle=N` to the state +file on every cycle — this keeps the file's modification time current. `docker_watchdog.sh` +checks how long ago the state file was modified. If it's more than 2 hours old while +`mem_shutdown_active=true` is set, `system_watchdog.sh` has likely stopped running. +`docker_watchdog.sh` logs a warning and resumes normal operation — it won't be silenced +indefinitely by a stale flag from a process that's no longer running. + +--- + +## ━━━ SCRIPTS AT A GLANCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +| Script | Role | When It Runs | +|--------|------|-------------| +| `docker_watchdog.sh` | Two-tier self-healing container monitor | Continuous background loop | +| `docker_daily_restart.sh` | Nightly proactive restart of degradation-prone containers | 1am via `daily_sync_maintenance.sh` | +| `docker_weekly_restart.sh` | Weekly restart of less-critical services | 2:30am Sunday via `weekly_sync_maintenance.sh` | +| `docker_network_connect.sh` | Network existence + container connection enforcement | Every array start | +| `watchdog_skip_list_manager.sh` | Skip list inspection and manual recovery | On demand | + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🐳 docker_watchdog.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +The self-healing heart of the ecosystem. Two tiers of monitoring run every 15 minutes +as a background process. Tier 1 applies specific thresholds to explicitly configured +containers. Tier 2 scans everything else for generic health problems. Together they +catch the full range of container failures — from subtle memory leaks to outright crashes. + +```bash +# Started automatically at array start via array_start.sh +# Runs continuously until array stops (SIGTERM → clean shutdown) +# Interval: DOCKER_WATCHDOG_INTERVAL=900 (15 minutes) ``` --- -## Scripts +### ── Tier 1 — Strict Per-Container Monitoring ──────────────────────────────── -### `docker_watchdog.sh` +Applies **only** to containers you explicitly configure. These are the containers +you care most about — the ones that affect users when they fail. Configure them +once in `master_host*.conf` and they are monitored with specific, appropriate +thresholds forever. -**The self-healing container monitoring system.** Two tiers of monitoring that work together to keep every container in the stack healthy — from strict per-container thresholds down to global health scanning of everything that's running. +--- + +#### 💾 Memory Hard Limits ```bash -# Scheduled as: */15 * * * * (every 15 minutes) -/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_watchdog.sh +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# Memory hard limits in MB — immediate restart when exceeded, no strike system. +# Each entry: "ContainerName:LimitInMB" +# +# These are NOT soft targets — they are hard ceilings. A container that hits +# its limit gets restarted immediately on the current cycle. Memory leaks are +# not brief spikes. Waiting for a second confirmation just allows more leak. +# +# How to size limits: +# Check normal usage: docker stats ContainerName +# Set limit at ~150-200% of normal peak usage +# Emby peaks around 8-12GB during heavy transcoding — 18GB gives headroom +# without triggering on legitimate load spikes +# +HOST1_WATCHDOG_CONTAINERS=( + "Emby:18432" # 18GB — peaks ~12GB under heavy transcode load + "LidaTube:6144" # 6GB — YouTube downloader, can grow with large queues + "Tdarr:6144" # 6GB — video transcoder, memory-intensive by nature + "Code-Server:1024" # 1GB — IDE, should be light; 1GB is generous +) +``` + +A soft warning threshold fires at `SOFT_MEM_THRESHOLD=80` percent of the hard limit — +giving early visibility into a container approaching its ceiling before a restart is +triggered. Useful for catching gradual leaks before they become events. + +--- + +#### 📊 CPU Thresholds + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# CPU thresholds are normalised against the server's total core count. +# This makes the threshold meaningful regardless of hardware. +# +# Why normalised: +# A container using 85% of one core on a 16-core machine = 5.3% normalised +# → a nothing, don't touch it +# A container using 85% normalised on a 16-core machine = 13.6 cores worth +# → a runaway process, restart it +# +# CPU uses a STRIKE SYSTEM — not immediate restart like memory. +# Brief CPU spikes are completely normal (Tdarr encoding, Emby transcoding, +# SABnzbd unpacking). The strike system ignores spikes and acts on sustained usage. +# +# Strike 1: CPU above HARD_CPU_THRESHOLD this cycle → warn, increment strike +# Strike 2: CPU above threshold next cycle → restart, reset strike counter +# Recovery: CPU drops below threshold any cycle → reset strike counter to 0 +# + SOFT_CPU_THRESHOLD=50 # warn at 50% normalised — informational only + HARD_CPU_THRESHOLD=85 # strike at 85% normalised + CPU_FAIL_LIMIT=2 # consecutive strikes before restart ``` --- -#### Tier 1 — Strict Monitoring - -Explicitly configured containers with per-container thresholds. Every container you care about most lives here. - -**Memory hard limits:** +#### 🌐 HTTP Responsiveness ```bash -declare -A WATCHDOG_CONTAINERS=( - ["Emby"]=16384 # 16GB hard limit — immediate restart if exceeded - ["LidaTube"]=6144 # 6GB - ["Tdarr"]=6144 - ["Code-Server"]=1024 +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# HTTP health checks hit the actual service endpoint on every watchdog cycle. +# "Container running" and "service responding" are not the same thing. +# +# A frozen Emby will respond to nothing but show as Up in Docker. +# A curl timeout catches this where Docker's own checks never would. +# +# Also uses a STRIKE SYSTEM — network hiccups and brief restarts happen. +# Two consecutive non-responses before acting prevents false positives +# from momentary connectivity issues. +# +# Format: "ContainerName:http://host:port/optional-path" +# The path can be a lightweight health endpoint or just the root URL +# +HOST1_WATCHDOG_CONTAINER_URLS=( + "Emby:http://localhost:8096" # Emby WebUI root — fast to respond + "NginxProxyManager:http://localhost:81" # NPM admin interface +) +# +# CURL_TIMEOUT=5 # seconds before a non-response counts as a failure +# RESP_FAIL_LIMIT=2 # consecutive failures before restart +``` + +> **Note on Docker HEALTHCHECK:** Docker has its own `HEALTHCHECK` mechanism but it +> requires the image to define a health check command — most self-hosted images don't. +> These HTTP checks work regardless of what the image defines. They check what actually +> matters: does the service respond to a request? + +--- + +#### ✅ Required Containers + +```bash +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# Containers that must always be running. If found stopped, the watchdog +# attempts to restart them every cycle until they are running or hit the +# restart loop limit. +# +# These are the containers whose absence breaks everything else: +# NginxProxyManager — all external traffic routes through this +# Authelia — authentication for every protected service +# Mariadb-Authelia — Authelia's database — Authelia cannot start without it +# Redis-Authelia — Authelia's session store — same dependency +# +# Required containers use the STRIKE SYSTEM — one miss might be mid-restart. +# Persistent failure → skip list → critical notification. +# +HOST1_WATCHDOG_REQUIRED_CONTAINERS=( + "NginxProxyManager" # reverse proxy — external access depends on this + "Authelia" # SSO authentication — all protected services need it + "Mariadb-Authelia" # Authelia database — must be up before Authelia starts + "Redis-Authelia" # Authelia session cache — same startup dependency ) ``` -Memory is checked against the configured limit in MB. If a container exceeds its hard limit it is restarted immediately — no strike system, no waiting. Memory leaks are real and immediate action is right. +--- -A soft threshold (`SOFT_MEM_THRESHOLD=80`) warns when a container reaches 80% of its hard limit — useful for spotting gradual leaks before they become problems. +### ── Tier 2 — Global Health Scan ───────────────────────────────────────────── -**CPU thresholds:** +Scans **every running container** when `WATCHDOG_SCAN_ALL=true`. No per-container +configuration required — this is the catch-all that protects everything not explicitly +configured in Tier 1. -CPU is normalised against total core count automatically. A container using 85% of one core on a 16-core system is ~5.3% normalised — not a problem. 85% normalised on a 16-core system means 13.6 cores worth of CPU — that's a problem. +Containers in `WATCHDOG_SCAN_IGNORE` are excluded from Tier 2 but still covered by +Tier 1 if configured there. -The strike system prevents restarts on brief spikes: -``` -CPU above HARD_CPU_THRESHOLD → strike 1 -CPU above HARD_CPU_THRESHOLD next cycle → strike 2 → restart -CPU recovers → strike count resets -``` +--- -**HTTP responsiveness:** +| Check | What Triggers It | What Happens | Why This Matters | +|-------|-----------------|--------------|-----------------| +| `WATCHDOG_RESTART_UNHEALTHY` | Docker HEALTHCHECK reports `unhealthy` | Restart | Catches containers with built-in health checks that are failing | +| `WATCHDOG_NOTIFY_OOM` | Kernel OOM-killed the container | Restart + notify | OOM kills are silent by default — you'd never know without this | +| `WATCHDOG_NOTIFY_CRASHLOOP` | Docker RestartCount climbing | Notify; above `WATCHDOG_CRASH_LIMIT` → restart → skip list | Distinguishes "just restarted once" from "has crashed 12 times" | +| `WATCHDOG_RESTART_DEAD` | Container in `dead` state | Remove + start | Dead containers can't be restarted — must be removed first | +| `WATCHDOG_RESTART_CRASHED` | Non-zero exit code, exited state | Restart | Catches clean-exit crashes that Docker's restart policy misses | ```bash -declare -A WATCHDOG_CONTAINER_URLS=( - ["Emby"]="http://localhost:8096" +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Tier 2 toggles — disable any check that produces false positives in your +# environment. Each is independent — disabling one doesn't affect the others. +# + WATCHDOG_SCAN_ALL=true # enable Tier 2 global scan + WATCHDOG_RESTART_UNHEALTHY=true # act on Docker HEALTHCHECK failures + WATCHDOG_NOTIFY_OOM=true # detect and notify kernel OOM kills + WATCHDOG_NOTIFY_CRASHLOOP=true # detect escalating restart counts + WATCHDOG_RESTART_DEAD=true # recover containers in dead state + WATCHDOG_RESTART_CRASHED=true # restart containers that exited non-zero + WATCHDOG_CRASH_LIMIT=5 # RestartCount above this → restart + skip list + +# Containers to exclude from Tier 2 global scan entirely. +# Use for containers you intentionally stop/start manually, or containers that +# have benign non-zero exits as part of their normal operation. + WATCHDOG_SCAN_IGNORE=( + "my-one-shot-container" # runs and exits normally — don't treat as crash + ) +``` + +--- + +### ── Cross-Cutting Intelligence ────────────────────────────────────────────── + +These mechanisms apply to **both tiers** on every watchdog cycle. They are what +separates intelligent monitoring from naive restart-on-failure. + +--- + +#### ⏱️ Startup Grace Period + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# After array start, containers take time to initialise — databases run recovery, +# services wait for dependencies, Emby scans its library. During this window, +# a container that isn't responding yet is not broken — it's just starting. +# +# Without a grace period, the watchdog fires false-positive restarts in the +# first minutes after every array start. With it, checks run and log normally +# but restart actions are suppressed until the grace period expires. +# +# The grace period clock starts from when the watchdog process itself starts — +# not from when the array starts — so it's accurate even if array start +# takes a few minutes. +# + WATCHDOG_STARTUP_GRACE=600 # 10 minutes — restarts suppressed, checks still log +``` + +--- + +#### 🔗 Dependency Ordering + +```bash +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# When a container and its dependency are both down, restart the dependency first. +# Skip the dependent container entirely on this cycle. On the next cycle the +# dependency should be healthy, and the dependent can restart cleanly. +# +# Without this: Authelia starts, can't connect to MariaDB (still starting), +# exits immediately, strike 1. Next cycle: same thing, strike 2. Next cycle: +# restart, skip list, critical notification. MariaDB was fine the whole time. +# +# With this: MariaDB restarted first. Authelia skipped this cycle. +# Next cycle: MariaDB healthy → Authelia restarts cleanly. No false alarms. +# +# Format: "DependentContainer:dependency1 dependency2" +# Multiple dependencies space-separated. All must be running before dependent restarts. +# +HOST1_WATCHDOG_DEPENDENCIES=( + "Authelia:Mariadb-Authelia Redis-Authelia" # both db and cache must be up first + "Authelia-Secondary:Mariadb-Authelia Redis-Authelia-Secondary" + "NextCloud:Postgres-NextCloud" # NextCloud needs its postgres first ) ``` -Containers with configured URLs are checked via `curl`. If the endpoint doesn't respond within `CURL_TIMEOUT` seconds that's a strike. Two consecutive failures trigger a restart. A container can be running and appear healthy to Docker while its application layer is frozen — HTTP checks catch this. +--- -**Required containers:** +#### 🔒 Restart Loop Protection + Skip List ```bash -WATCHDOG_REQUIRED_CONTAINERS=( +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# If the watchdog restarts the same container N times within a rolling time +# window and it keeps crashing, something is genuinely broken that restarts +# are not fixing. Continued restart attempts risk making it worse (database +# corruption from incomplete shutdowns, etc.). +# +# When a container hits the limit: +# 1. Added to the persistent skip list on /boot/config/ +# 2. Critical notification sent — this needs human attention +# 3. Watchdog stops touching it — completely hands off +# +# The skip list survives reboots — it lives on /boot/. This is intentional. +# If a container was in a bad enough state to be skip-listed, a reboot +# doesn't fix the underlying problem. It stays on the list until cleared. +# +# AUTO-CLEAR: The watchdog checks the skip list every cycle and removes any +# container it finds running. So if the container recovers on its own (e.g. +# Docker's own restart policy eventually succeeds after a longer backoff), +# the watchdog detects it running and resumes normal monitoring automatically. +# Manual clear only needed when the container is stuck stopped. +# + WATCHDOG_CONTAINER_RESTART_LIMIT=3 # restarts in the window before skip list + WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours + WATCHDOG_CONTAINER_RESTART_LOG="/boot/config/container_restart_history.db" + SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" +``` + +--- + +#### ⚡ Docker Daemon Health Check + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Every cycle begins with a daemon health check before any container operations. +# A hung Docker daemon makes every subsequent docker command hang — which would +# stall the entire watchdog indefinitely, leaving containers unmonitored. +# +# If the daemon doesn't respond within DOCKER_TIMEOUT seconds: +# 1. Attempt daemon restart via /etc/rc.d/rc.docker restart +# 2. Wait 15 seconds for recovery +# 3. Verify daemon is responding again +# 4. If recovered: log and continue the cycle normally +# 5. If still hung: log critical error, skip the rest of this cycle +# system_watchdog.sh will escalate from here (it has its own daemon check +# in Tier 1 Critical — bypass strikes, reboot if daemon stays down) +# +# DOCKER_TIMEOUT=10 # seconds — tight enough to detect hangs, not false positives +``` + +--- + +#### 📬 Notification Batching + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# All events from a single watchdog cycle are collected and sent as ONE +# notification at the end of the cycle. +# +# Why batching matters: +# On a 50+ container system, a single problem (e.g. shared database goes down) +# can cascade into 10+ dependent containers failing simultaneously. Without +# batching: 10 individual "container X failed" notifications arrive in rapid +# succession. With batching: one summary notification lists all affected +# containers. The first format is overwhelming. The second is actionable. +# + WATCHDOG_BATCH_NOTIFY=true +``` + +--- + +#### 🔇 Silent When Healthy + +``` +Runs 96 times per day. If it produced output every run, the logs would be +useless noise. The watchdog produces no output on clean cycles — only when +something needs attention or a periodic heartbeat fires. + +Heartbeat interval: SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1 + → "♥ docker_watchdog alive — HOST1 — ~4hr uptime" once per hour + → proof the watchdog is running without log spam +``` + +--- + +### ── State Files ───────────────────────────────────────────────────────────── + +| File | Configured As | Location | Resets | Purpose | +|------|--------------|----------|--------|---------| +| Strike counts | `WATCHDOG_STATE_FILE` | `/tmp/` | On reboot | Per-container CPU/HTTP strike counters | +| Skip list | `SYS_WATCHDOG_FAILED_FILE` | `/boot/config/` | Never (manual / auto-clear) | Containers that exhausted restart attempts | +| Restart history | `WATCHDOG_CONTAINER_RESTART_LOG` | `/boot/config/` | Auto-purge after window | Restart loop detection data | +| Shared state | `SYS_WATCHDOG_STATE_FILE` | `/tmp/` | On reboot | RAM emergency flag + cycle heartbeat from system_watchdog.sh | + +`/tmp/` files reset on every reboot — correct, because strike counts from before +a reboot are meaningless after it. `/boot/config/` files survive reboots — also +correct, because a container that was skip-listed before a reboot is still broken after it. + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Normal operation — started automatically by array_start.sh, runs until +# array stops. You do not need to run this manually under normal circumstances. +# ───────────────────────────────────────────────────────────────────────────── +docker_watchdog.sh + +# ───────────────────────────────────────────────────────────────────────────── +# Dry run — walk through a full watchdog cycle without restarting anything. +# Shows what the watchdog would do based on current container states. +# Useful for verifying configuration before enabling automatic restarts. +# ───────────────────────────────────────────────────────────────────────────── +docker_watchdog.sh --dry-run + +# ───────────────────────────────────────────────────────────────────────────── +# Status — show current watchdog state at a glance: +# • Strike counts for all monitored containers +# • Current skip list contents + which are running vs stopped +# • Whether grace period is active and how long remains +# • Whether RAM emergency deferral is active +# • Last cycle timing and daemon health +# ───────────────────────────────────────────────────────────────────────────── +docker_watchdog.sh --status + +# ───────────────────────────────────────────────────────────────────────────── +# Verbose — show full detail for every container checked, every decision made. +# Useful for debugging why a container is or isn't being restarted. +# ───────────────────────────────────────────────────────────────────────────── +docker_watchdog.sh --log +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ♻️ docker_daily_restart.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Restarts configured containers every night at 1am. Called by `daily_sync_maintenance.sh` +via `DAILY_MAINTENANCE_SCRIPTS` — not scheduled directly. The maintenance window already +owns this timeslot, so any downtime from restarts is absorbed by a window that's already +happening. + +--- + +### ── Why Daily Restarts ─────────────────────────────────────────────────────── + +Not all container degradation triggers a watchdog response. Memory leaks that grow +slowly over days stay well below the hard limit for weeks. Connection tables that fill +up over a month never cross a clear threshold. Live TV schedulers that accumulate +internal state don't fail — they just get progressively slower. + +Daily restarts at 1am clear all of this with zero user impact. Proactive maintenance +at a time when no one is using the services. The containers in this list were specifically +chosen because they are known to degrade — not because they are unreliable. + +```bash +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# Containers restarted every night at 1am. +# Each entry is just the container name — no configuration needed. +# +# Good candidates for daily restarts: +# - Reverse proxies (connection table management) +# - Authentication services (session cache clearing) +# - Live TV schedulers (accumulated scheduling state) +# - Download managers (connection pool maintenance) +# +HOST1_DAILY_RESTART_CONTAINERS=( + "NginxProxyManager" # connection table fills slowly over weeks + "Authelia" # session cache benefits from periodic clearing + "Dispatcharr" # Live TV scheduler accumulates state + "Dispatcharr-Basic" # secondary Live TV scheduler — same reason + "ErsatzTV-Emby" # channel schedule builder, stale entries accumulate +) +``` + +--- + +### ── Behaviour Rules ────────────────────────────────────────────────────────── + +``` +Running containers → docker restart (graceful stop + start — the correct approach) +Stopped containers → left stopped (was intentionally stopped — state is respected) +Missing containers → logged + skip (not found on this server — not an error) +``` + +The "was running → restart, was stopped → leave stopped" rule is consistent across +the entire ecosystem. Container state is always respected. The restart scripts +never bring back a container that was intentionally stopped. + +--- + +### ── Safeguards ────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# DEPENDENCY ORDERING — restarts happen in dependency-safe order. +# The same WATCHDOG_DEPENDENCIES configuration used by the watchdog applies +# here. If Authelia depends on Mariadb and Redis, those restart first. +# CONTAINER_DELAY seconds wait between dependency restart and dependent restart +# — gives the dependency time to fully initialise before the dependent tries +# to connect. +# +# RESTART VERIFICATION — after each restart, the script waits a settle period +# then checks if the container is still running. A container that starts and +# immediately crashes is marked as failed with a notification sent. The script +# does not silently pass a restart that didn't stick. +# +# DOCKER_TIMEOUT=30 — every docker command is wrapped in a 30 second timeout. +# A hung Docker daemon cannot cause this script to hang indefinitely. +# Timed-out commands are retried per RETRY_COUNT before marking as failed. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Normal — called by daily_sync_maintenance.sh, rarely run directly. +# Safe to run manually for ad hoc restarts when needed. +# ───────────────────────────────────────────────────────────────────────────── +docker_daily_restart.sh + +# ───────────────────────────────────────────────────────────────────────────── +# Dry run — show exactly which containers would be restarted and which would +# be skipped (with reason). Run this before scheduling to verify the list. +# ───────────────────────────────────────────────────────────────────────────── +docker_daily_restart.sh --dry-run + +# ───────────────────────────────────────────────────────────────────────────── +# Status — show configured restart list, current container states, +# and dependency ordering for this server. +# ───────────────────────────────────────────────────────────────────────────── +docker_daily_restart.sh --status + +# Verbose per-container output +docker_daily_restart.sh --log +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ♻️ docker_weekly_restart.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Restarts configured containers once per week. Called by `weekly_sync_maintenance.sh` +via `WEEKLY_MAINTENANCE_SCRIPTS` — runs at 2:30am Sunday after the sync window has +completed and already restarted the critical containers (Emby, auth stack). + +--- + +### ── Context: After the Sync Window ───────────────────────────────────────── + +`weekly_sync_maintenance.sh` stops the critical container set (Emby, auth stack) for +a clean sync, then restarts them. This script runs **after** that restart — targeting a +**different** set of less-critical services that benefit from a weekly restart but do +not need to be stopped for the sync itself. + +These containers are already running when this script executes. The distinction between +daily and weekly is purely about how often each service needs a clean start to maintain +its best performance: + +``` +Daily: Connection-heavy infrastructure (proxy, auth, Live TV) + → degrades faster, benefits from more frequent resets + +Weekly: Productivity and media services (NextCloud, AdGuard, Immich) + → degrades slowly, monthly restart is overkill, daily is unnecessary +``` + +```bash +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# Containers restarted every Sunday at 2:30am (after weekly sync completes). +# +HOST1_WEEKLY_RESTART_CONTAINERS=( + "NextCloud" # file sync — benefits from clean weekly start + "AdGuard-Home" # DNS — cache and stat accumulation + "Immich" # photo library — index/cache maintenance +) +``` + +Behaviour, dependency ordering, restart verification, and DOCKER_TIMEOUT are identical +to `docker_daily_restart.sh`. Same rules apply: running → restart, stopped → leave, +missing → skip. + +```bash +docker_weekly_restart.sh # normal run +docker_weekly_restart.sh --dry-run # preview +docker_weekly_restart.sh --status # show config and current states +docker_weekly_restart.sh --log # verbose +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🌐 docker_network_connect.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Ensures custom Docker networks exist and connects specified containers to them at every +array start. Idempotent — if everything is already correct it produces no output. +Run via `ARRAY_START_SCRIPTS` — starts early in the array start sequence, before the +watchdogs begin their first cycle. + +--- + +### ── What It Does ──────────────────────────────────────────────────────────── + +For each configured network, in order: + +``` +1. Does the network exist? + NO → Create it (bridge driver, Docker assigns subnet automatically) + → Send notification — this should not happen except after an update + → Log the subnet that was assigned + YES → Skip creation silently — correct state, nothing to do + +2. For each configured container: + Already connected? → Skip silently — correct state + Not connected? → Connect it + Container missing? → Warn and skip — container may not be running yet, + not treated as fatal, will succeed on next array start +``` + +--- + +### ── Why Network Recreation Matters ───────────────────────────────────────── + +Docker containers get their networks assigned at creation time via the unRAID template. +Changing the network assignment means deleting and recreating the container — which loses +any state not stored in the appdata volume. For containers created by other containers +(NextCloud AIO spawns its own stack, for example), you can't even touch the network +assignment through the unRAID UI. + +`high-availability` is the main shared network in this setup — most containers join it +so they can communicate internally without going through the reverse proxy. After a +unRAID update wipes custom networks, every container on `high-availability` suddenly +can't reach its peers. This script recreates the network and reconnects everything at +the next array start. + +The notification on creation is intentional. Network creation should only happen after +an update — if it's happening regularly, something is wrong with the network configuration +and you need to know. + +```bash +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# Networks to ensure exist + containers to connect to each network. +# The mapping is many-to-many: every container connects to every network. +# +# Containers listed here do not need to be running — the script handles +# missing containers gracefully (warns + skips). They will be connected on +# the next array start if they come up later. +# +HOST1_NETWORK_CONNECT_NETWORKS=( + "high-availability" # main internal network — most containers should be on this +) + +HOST1_NETWORK_CONNECT_CONTAINERS=( + "memcached" # NextCloud's cache — needs to reach NextCloud AIO network + "Npm-CrowdSec" # CrowdSec bouncer — needs to reach NPM's network +) +``` + +> **Timing dependency:** Networks created by Docker Compose stacks exist only after +> those stacks start. If NextCloud AIO creates the `nextcloud-aio` network at startup +> and this script runs before NextCloud AIO starts, the network won't exist yet and +> the connection fails this run. It will succeed on the next array start once the +> network exists. This is a known limitation — schedule the Compose stacks early in +> the ARRAY_START_SCRIPTS order to minimise the window. + +```bash +docker_network_connect.sh # normal run (at array start) +docker_network_connect.sh --dry-run # show what would be created / connected +docker_network_connect.sh --status # show current network and connection state +docker_network_connect.sh --log # verbose per-network per-container output +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔧 watchdog_skip_list_manager.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Manual recovery tool. Used when automatic healing hasn't worked and a container needs +human intervention. Gives a clear picture of what the watchdog has already tried and +provides a clean path to resume normal monitoring after the problem is fixed. + +--- + +### ── What the Skip List Is ─────────────────────────────────────────────────── + +When `docker_watchdog.sh` restarts the same container `WATCHDOG_CONTAINER_RESTART_LIMIT` +times within `WATCHDOG_CONTAINER_RESTART_WINDOW` hours, it concludes that restarts are +not fixing whatever is wrong. The container is added to the persistent skip list at +`SYS_WATCHDOG_FAILED_FILE` on `/boot/config/`. The watchdog stops touching it entirely. +A critical notification goes out. + +This is the correct response. A watchdog that keeps hammering a broken container is +not helpful — it's potentially destructive (database corruption, incomplete writes). +Stopping automated attempts and alerting the operator is the right escalation path. + +**Auto-clear:** The watchdog checks the skip list every cycle and removes any container +it finds running. If the container recovers on its own — Docker's built-in restart +policy eventually succeeds after a longer backoff, or someone manually starts it — the +watchdog detects it running and resumes normal monitoring without any intervention needed. +Manual clear is only necessary when the container is stuck stopped and cannot self-recover. + +--- + +### ── Standard Recovery Workflow ───────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Step 1 — Understand the situation +# Shows the skip list, which containers are running vs stopped, how many +# restarts were attempted, and the restart history for each. +# ───────────────────────────────────────────────────────────────────────────── +watchdog_skip_list_manager.sh --status + +# ───────────────────────────────────────────────────────────────────────────── +# Step 2 — Fix the underlying problem first +# Check container logs: docker logs ContainerName --tail 100 +# Check for disk issues: df -h /mnt/user +# Check for database issues: docker exec ContainerName sqlite3 /path/to.db ".tables" +# Fix whatever caused the repeated crashes before clearing the skip list. +# ───────────────────────────────────────────────────────────────────────────── + +# ───────────────────────────────────────────────────────────────────────────── +# Step 3 — Clear the container from the skip list + its restart history +# Clearing restart history is important — otherwise the counter carries over +# and the container hits the limit again almost immediately if it has any +# startup trouble. +# ───────────────────────────────────────────────────────────────────────────── +watchdog_skip_list_manager.sh --clear ContainerName + +# ───────────────────────────────────────────────────────────────────────────── +# Step 4 — Start the container manually +# Starting it yourself confirms your fix worked before handing it back +# to the watchdog. If it crashes immediately, you know the fix didn't work. +# ───────────────────────────────────────────────────────────────────────────── +docker start ContainerName + +# ───────────────────────────────────────────────────────────────────────────── +# Step 5 — Normal monitoring resumes automatically +# On the watchdog's next cycle it will see the container running and remove +# it from the skip list (if you cleared it manually, it's already gone). +# Restart history is clean. Back to normal. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Safety Warning ────────────────────────────────────────────────────────── + +``` +⚠️ If docker_watchdog.sh is currently running when you clear the skip list, + it may re-add the container on its very next cycle if the container is + still in a bad state. + + The script detects this and warns you: + "docker_watchdog.sh is currently RUNNING — cleared container may be + re-added on next cycle if still failing" + + Fix the root cause BEFORE clearing. Clearing the skip list without fixing + the underlying problem just resets the counter — the container will exhaust + its restart attempts again and return to the skip list. +``` + +--- + +### ── All Actions ───────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Show status — skip list contents, container states, restart history +# This is the default action — running without arguments shows status +# ───────────────────────────────────────────────────────────────────────────── +watchdog_skip_list_manager.sh +watchdog_skip_list_manager.sh --status # explicit + +# ───────────────────────────────────────────────────────────────────────────── +# Clear specific container — removes from skip list + clears restart history +# Requires confirmation (type YES) unless --force is passed +# ───────────────────────────────────────────────────────────────────────────── +watchdog_skip_list_manager.sh --clear ContainerName +watchdog_skip_list_manager.sh --clear ContainerName --force # no prompt + +# ───────────────────────────────────────────────────────────────────────────── +# Clear everything — full reset of skip list and restart history +# Use when multiple containers are affected or after a systemic problem is fixed +# ───────────────────────────────────────────────────────────────────────────── +watchdog_skip_list_manager.sh --clear-all +watchdog_skip_list_manager.sh --clear-all --force # non-interactive + +# ───────────────────────────────────────────────────────────────────────────── +# Dry run — show what would be cleared without actually clearing +# Works with --clear and --clear-all +# ───────────────────────────────────────────────────────────────────────────── +watchdog_skip_list_manager.sh --clear-all --dry-run +``` + +--- + +## ━━━ HOW THE SCRIPTS FIT TOGETHER ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +``` +Array starts + │ + ▼ +docker_network_connect.sh ─────── run once + │ ensure networks + connections + │ silent if correct, notify if creating + │ + ▼ (continuous background) +docker_watchdog.sh ─────────────── every 15 minutes + │ Tier 1: memory, CPU, HTTP, required + │ Tier 2: global unhealthy/OOM/crash/dead scan + │ reads system_watchdog state (RAM emergency) + │ + │ (on skip list event) + ▼ +watchdog_skip_list_manager.sh ──── manual + inspect state, clear after fixing + +Scheduled maintenance windows: + │ + ▼ (1am daily) +daily_sync_maintenance.sh + └── docker_daily_restart.sh ── restart connection-heavy services + NPM, Authelia, Dispatcharr, ErsatzTV + │ + ▼ (2:30am Sunday) +weekly_sync_maintenance.sh + └── docker_weekly_restart.sh ─ restart less-critical services + NextCloud, AdGuard, Immich +``` + +--- + +## ━━━ CONFIGURATION REFERENCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +All configuration lives in two places. `detect_hosts()` in `common.sh` aliases all +`HOST1_` and `HOST2_` prefixed variables to their unprefixed names so the scripts +always use the right values for the server they're running on. + +--- + +### 📋 master_host*.conf — Per-Host Configuration + +These vary between HOST1 and HOST2 because each server runs different containers +at different resource limits with different network requirements. + +```bash +# master_host1.conf (or master_host2.conf for HOST2) +# ───────────────────────────────────────────────────────────────────────────── + +# Tier 1 — memory limits ("ContainerName:LimitInMB") +HOST1_WATCHDOG_CONTAINERS=( + "Emby:18432" + "LidaTube:6144" +) + +# Tier 1 — HTTP health check endpoints ("ContainerName:http://host:port") +HOST1_WATCHDOG_CONTAINER_URLS=( + "Emby:http://localhost:8096" +) + +# Tier 1 — must always be running +HOST1_WATCHDOG_REQUIRED_CONTAINERS=( "NginxProxyManager" - "Lldap-Gmer4Lfe" "Authelia" "Mariadb-Authelia" "Redis-Authelia" - "Authelia-Secondary" - "Redis-Authelia-Secondary" ) -``` -These must always be running. If any are found stopped, the watchdog attempts to restart them. Strike system applies — persistent failures get added to the skip list. - ---- - -#### Tier 2 — Global Health Scan - -Scans every running container for health issues — catches anything not explicitly configured in Tier 1. - -| Check | Trigger | Action | -|-------|---------|--------| -| `WATCHDOG_RESTART_UNHEALTHY` | Docker HEALTHCHECK reports `unhealthy` | Restart | -| `WATCHDOG_NOTIFY_OOM` | Kernel OOM-killed the container | Restart + notify | -| `WATCHDOG_NOTIFY_CRASHLOOP` | Docker RestartCount climbing | Notify (critical above `WATCHDOG_CRASH_LIMIT`) | -| `WATCHDOG_RESTART_DEAD` | Container in dead state | Remove + restart | -| `WATCHDOG_RESTART_CRASHED` | Non-zero exit code | Restart | - -Each check is independently toggleable — disable checks that cause false positives in your environment. - -> **Note on HEALTHCHECK:** Only containers with a `HEALTHCHECK` instruction defined in their Docker image report health status. Containers without one are invisible to the unhealthy check but still caught by crash, dead, and OOM checks. You can add custom HEALTHCHECKs via unRAID's Extra Parameters field — see the health check guide for your specific containers. - ---- - -#### Cross-cutting Intelligence - -These apply to **both tiers** on every watchdog run: - -**Startup grace period:** -```bash -WATCHDOG_STARTUP_GRACE=600 # seconds after boot -``` -For the first 10 minutes after array start, checks run but restarts are suppressed. Containers need time to come up — false positives during boot are common without this. Checks still run and report so you can see what's happening, but no restarts fire. - -**Dependency ordering:** -```bash -declare -A WATCHDOG_DEPENDENCIES=( - ["Authelia"]="Mariadb-Authelia Redis-Authelia" - ["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary" - ["NextCloud"]="Postgres-NextCloud" +# Tier 1 + 2 — dependency ordering ("Dependent:dep1 dep2") +HOST1_WATCHDOG_DEPENDENCIES=( + "Authelia:Mariadb-Authelia Redis-Authelia" ) -``` -If a container's dependency is also down, the dependent is skipped this cycle. The dependency gets restarted first. On the next cycle — once the database is up and accepting connections — the dependent container gets restarted. This prevents the classic failure mode where Authelia is restarted before its database is ready and fails immediately, triggering another restart attempt. -**Restart loop protection:** -```bash -WATCHDOG_CONTAINER_RESTART_LIMIT=3 -WATCHDOG_CONTAINER_RESTART_WINDOW=1 # hours -WATCHDOG_CONTAINER_RESTART_LOG="/boot/config/container_restart_history.db" -``` -If the watchdog restarts the same container 3 times within 1 hour, that container is added to the persistent skip list. Something is genuinely broken that restarts are not fixing — continued hammering wastes resources and masks the real problem. A critical notification is sent when a container hits the skip list. - -The skip list lives on `/boot/` — it survives reboots. The container stays on the skip list until it is found running again (manually fixed or recovered after a reboot), at which point it's automatically removed and the restart history is cleared. - -**Notification batching:** -```bash -WATCHDOG_BATCH_NOTIFY=true -``` -All events from a single watchdog run are collected and sent as one notification at the end. On a system with 50+ containers, individual per-event notifications during a problem cascade would be unmanageable. One clean summary tells you what happened without flooding your notification channel. - ---- - -#### Skip List Management - -The skip list (`/boot/config/system_watchdog_failed.db`) is the persistent memory of containers that have exhausted restart attempts. - -```bash -# View current skip list -cat /boot/config/system_watchdog_failed.db - -# A container auto-removes itself when found running again -# To manually clear a specific container: -sed -i '/ContainerName/d' /boot/config/system_watchdog_failed.db - -# To clear the entire skip list: -> /boot/config/system_watchdog_failed.db -``` - -Also clear the restart history when manually fixing a container: -```bash -sed -i '/ContainerName|/d' /boot/config/container_restart_history.db -``` - ---- - -#### State Files - -| File | Location | Resets | Purpose | -|------|----------|--------|---------| -| `container_watchdog_state.db` | `/tmp/` | On reboot | Strike counts for all containers | -| `system_watchdog_failed.db` | `/boot/config/` | Never (manual) | Persistent skip list | -| `container_restart_history.db` | `/boot/config/` | Auto-purge after window | Restart loop detection | - ---- - -### `docker_daily_restart.sh` - -Restarts configured containers every day. - -```bash -# Scheduled as: 0 3 * * * (3am daily) -/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_daily_restart.sh -``` - -**Why daily restarts:** - -Some containers accumulate memory over time — connection pools that don't shrink, caches that grow without bound, log buffers that don't rotate. A daily restart clears all of this. It's simpler and more reliable than trying to tune every container's internal memory management. - -Containers that benefit from daily restarts are typically those handling lots of short-lived connections — reverse proxies, auth servers, and live TV schedulers. - -```bash -DAILY_RESTART_CONTAINERS=( - "NginxProxyManager" # connection pool accumulation - "Authelia" # session and token cache - "Dispatcharr" # live TV connection management - "Dispatcharr-Basic" - "Dispatcharr-Iptv-Users" - "ErsatzTV-Emby" # channel scheduling state +# Daily restart list +HOST1_DAILY_RESTART_CONTAINERS=( + "NginxProxyManager" + "Authelia" + "Dispatcharr" ) -``` -**Retry logic:** Uses `RETRY_COUNT` and `SLEEP` from `Master.conf`. If a container fails to restart it retries before marking it as failed and notifying. - ---- - -### `docker_weekly_restart.sh` - -Restarts configured containers once per week. - -```bash -# Scheduled as: 0 3 * * 0 (Sunday 3am weekly) -/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh -``` - -For less critical services that benefit from periodic restarts but don't need daily cycling. Typically productivity and self-hosted application containers that are stable but benefit from a clean weekly slate. - -```bash -WEEKLY_RESTART_CONTAINERS=( +# Weekly restart list +HOST1_WEEKLY_RESTART_CONTAINERS=( "NextCloud" - "Organizrv2-Gmer4Lfe" "AdGuard-Home" - "Immich-Gmer4Lfe" ) -``` -Sunday morning is the natural maintenance window — it runs alongside the weekly log clear, ZFS snapshot, SMART check and backup verify. Everything happens while load is lowest. +# Networks to ensure exist +HOST1_NETWORK_CONNECT_NETWORKS=( + "high-availability" +) ---- - -### `docker_network_connect.sh` - -Connects containers to extra Docker networks on array start. - -```bash -# Scheduled as: At Startup of Array -/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh -``` - -**The problem it solves:** - -Docker containers are assigned networks at creation time via the unRAID template. Sometimes containers need to communicate with containers on a different network that wasn't configured in the original template — for example, `memcached` needing to talk to the `nextcloud-aio` network so NextCloud can use it for caching. - -The correct solution is to add the network in the template. But some containers are created by other containers (like `nextcloud-aio`) and their network assignments can't easily be changed. This script handles those edge cases at array start. - -Every container in `NETWORK_CONNECT_CONTAINERS` is connected to every network in `NETWORK_CONNECT_NETWORKS` — many-to-many. Already-connected containers are skipped cleanly — safe to run multiple times. - -```bash -NETWORK_CONNECT_CONTAINERS=( +# Containers to connect to every configured network +HOST1_NETWORK_CONNECT_CONTAINERS=( "memcached" "Npm-CrowdSec" ) - -NETWORK_CONNECT_NETWORKS=( - "nextcloud-aio" # Docker network name — must exist before array start -) ``` -**Note:** The target network must exist before this script runs. Networks created by Docker Compose or the nextcloud-aio stack are created when their containers start — if those containers start after this script, the connection will fail. The unRAID User Scripts plugin "At Startup of Array" timing usually handles this correctly but be aware of the dependency. - --- -## Relationship Between Scripts +### 📋 master.conf — Shared Configuration +These apply equally to both servers — thresholds, intervals, toggle switches. + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── + +# ── Watchdog Intervals and Timing ────────────────────────────────────────── + DOCKER_WATCHDOG_INTERVAL=900 # seconds between cycles (15 minutes) + WATCHDOG_STARTUP_GRACE=600 # seconds before restarts begin after boot + CONTAINER_DELAY=15 # seconds between dependency + dependent restart + +# ── Memory Thresholds ────────────────────────────────────────────────────── + SOFT_MEM_THRESHOLD=80 # warn at % of hard limit (no restart) + +# ── CPU Thresholds ───────────────────────────────────────────────────────── + SOFT_CPU_THRESHOLD=50 # warn threshold — normalised % of total cores + HARD_CPU_THRESHOLD=85 # strike threshold — normalised % of total cores + CPU_FAIL_LIMIT=2 # consecutive strikes before restart + +# ── HTTP Health Check ────────────────────────────────────────────────────── + CURL_TIMEOUT=5 # seconds before curl times out + RESP_FAIL_LIMIT=2 # consecutive failures before restart + +# ── Restart Loop Protection ──────────────────────────────────────────────── + WATCHDOG_CONTAINER_RESTART_LIMIT=3 # restarts before skip list + WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours + +# ── Tier 2 Global Scan ───────────────────────────────────────────────────── + WATCHDOG_SCAN_ALL=true # enable global scan + WATCHDOG_SCAN_IGNORE=() # containers excluded from Tier 2 + WATCHDOG_RESTART_UNHEALTHY=true + WATCHDOG_NOTIFY_OOM=true + WATCHDOG_NOTIFY_CRASHLOOP=true + WATCHDOG_CRASH_LIMIT=5 + WATCHDOG_RESTART_DEAD=true + WATCHDOG_RESTART_CRASHED=true + +# ── Notifications ────────────────────────────────────────────────────────── + WATCHDOG_BATCH_NOTIFY=true # collect events, send one summary ``` -docker_network_connect.sh — runs once at array start - ↓ -docker_watchdog.sh — runs every 15 minutes - ├── Tier 1: strict per-container monitoring - └── Tier 2: global health scan of everything - -docker_daily_restart.sh — runs at 3am every day -docker_weekly_restart.sh — runs at 3am every Sunday -``` - -The watchdog is the continuous monitor. The restart scripts are the scheduled maintenance. Together they cover both reactive healing (watchdog) and proactive freshness (restarts). --- -## Adding a New Container to the Watchdog +## ━━━ ADDING A CONTAINER ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Adding a new container to monitoring is additive — add the relevant lines to +`master_host1.conf` (and `master_host2.conf` if it runs there too). No changes to +any script needed. `detect_hosts()` picks up the new configuration on the next cycle. -**Tier 1 — memory monitoring:** ```bash -# Add to WATCHDOG_CONTAINERS in Master.conf -declare -A WATCHDOG_CONTAINERS=( - ["Emby"]=16384 - ["MyNewContainer"]=2048 # 2GB hard limit +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# Example: adding "MyApp" to full Tier 1 monitoring + daily restarts + +# 1. Memory hard limit — restart immediately if exceeded +HOST1_WATCHDOG_CONTAINERS=( + "Emby:18432" + "MyApp:2048" # 2GB — check with `docker stats MyApp` to size this correctly ) -``` -**Tier 1 — HTTP check:** -```bash -declare -A WATCHDOG_CONTAINER_URLS=( - ["Emby"]="http://localhost:8096" - ["MyNewContainer"]="http://localhost:9000/health" +# 2. HTTP health check — restart after 2 consecutive failures +HOST1_WATCHDOG_CONTAINER_URLS=( + "Emby:http://localhost:8096" + "MyApp:http://localhost:8080/health" # or just the root if no /health endpoint ) -``` -**Tier 1 — required container:** -```bash -WATCHDOG_REQUIRED_CONTAINERS=( +# 3. Required — must always be running +HOST1_WATCHDOG_REQUIRED_CONTAINERS=( "NginxProxyManager" - "MyNewContainer" # must always be running + "Authelia" + "MyApp" # add here if it should always be running ) -``` -**Tier 2 — dependency:** -```bash -declare -A WATCHDOG_DEPENDENCIES=( - ["Authelia"]="Mariadb-Authelia Redis-Authelia" - ["MyNewContainer"]="its-database-container" # restart db first +# 4. Dependency — if MyApp needs its database up first +HOST1_WATCHDOG_DEPENDENCIES=( + "Authelia:Mariadb-Authelia Redis-Authelia" + "MyApp:MyApp-Database" # database restarts first, then MyApp ) -``` -**Tier 2 — ignore in global scan:** -```bash +# 5. Daily restart — if MyApp degrades over time +HOST1_DAILY_RESTART_CONTAINERS=( + "NginxProxyManager" + "Authelia" + "MyApp" +) + +# ───────────────────────────────────────────────────────────────────────────── +# Tier 2 picks up MyApp automatically — no configuration needed. +# It will be included in the global unhealthy/OOM/crash/dead scan without +# any additional setup. Tier 2 is the catch-all for everything not in Tier 1. +# ───────────────────────────────────────────────────────────────────────────── + +# To EXCLUDE MyApp from Tier 2 (e.g. it intentionally exits between runs): +# master.conf WATCHDOG_SCAN_IGNORE=( - "intentionally-stopped-container" # skip this in global scan + "MyApp" # one-shot container — exits normally, not a crash ) -``` - ---- - -## Scheduled Summary - -```bash -# At Startup of Array -docker_network_connect.sh - -# Every 15 minutes -*/15 * * * * docker_watchdog.sh - -# Daily — 3am -0 3 * * * docker_daily_restart.sh - -# Weekly — Sunday 3am -0 3 * * 0 docker_weekly_restart.sh -``` - ---- - -## --dry-run Support - -All scripts support `--dry-run`. Always test before scheduling: - -```bash -/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_watchdog.sh --dry-run -/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_daily_restart.sh --dry-run -/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh --dry-run -/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh --dry-run -``` - -`docker_watchdog.sh --status` shows current strike counts, skip list contents, and grace period status without running any checks. \ No newline at end of file +``` \ No newline at end of file diff --git a/Docker_Essentials/docker_daily_restart.sh b/Docker_Essentials/docker_daily_restart.sh index 786683f..9a7ec9d 100644 --- a/Docker_Essentials/docker_daily_restart.sh +++ b/Docker_Essentials/docker_daily_restart.sh @@ -1,24 +1,64 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Docker Daily Restart --------------------------------------- -# ----------------------------------------------------------------------------------------------- -# Restarts or starts specified Docker containers with retry logic. -# Containers are configured in Master.conf under DAILY_RESTART_CONTAINERS. -# Uses global RETRY_COUNT and SLEEP from Master.conf for retry behaviour. -# Sends notifications on completion or failure via common.sh notify(). -# Supports --dry-run to preview what would be restarted without taking action. -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= Docker Daily Restart ======================================= +# ============================================================================================== +# Restarts or starts all containers in HOST*_DAILY_RESTART_CONTAINERS. +# Called by daily_sync_maintenance.sh via DAILY_MAINTENANCE_SCRIPTS every night at 1am. +# Can also be run manually for ad hoc restarts. +# +# ── WHY DAILY RESTARTS ──────────────────────────────────────────────────────────────────────── +# Some containers degrade over time without a restart: +# Dispatcharr — Live TV scheduler accumulates state and slows down +# NginxProxyManager — connection table grows, occasional stale proxy entries +# Authelia — session cache benefits from periodic clearing +# Daily restart is intentional maintenance, not just housekeeping. +# +# ── BEHAVIOUR ───────────────────────────────────────────────────────────────────────────────── +# Running containers → docker restart (graceful stop + start) +# Stopped containers → left stopped — was down intentionally, do not bring back up +# Missing containers → logged and skipped — not treated as fatal +# Each action uses RETRY_COUNT + SLEEP from master.conf for retry logic. +# +# The "was running → restart, was stopped → leave stopped" rule is consistent +# across the entire ecosystem — container state is always respected. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# Dependency ordering — containers restart in dependency-safe order using +# HOST*_WATCHDOG_DEPENDENCIES from master_host*.conf. If Authelia depends on +# Mariadb + Redis, those restart first with CONTAINER_DELAY before Authelia starts. +# +# Restart verification — after each restart, container state is checked after a short +# settle period. If the container fails to stay running it is marked as failed and +# a notification is sent rather than silently passing. +# +# Timeout protection — all docker commands are wrapped in a 30 second timeout. +# A hung Docker daemon cannot cause this script to hang indefinitely. +# Timed-out commands are retried per RETRY_COUNT before marking as failed. +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_DAILY_RESTART_CONTAINERS — list of containers to restart daily +# Set by detect_hosts() alias → DAILY_RESTART_CONTAINERS used by this script +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# RETRY_COUNT — retry attempts before giving up on a container +# SLEEP — seconds between retry attempts +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# docker_daily_restart.sh — normal restart +# docker_daily_restart.sh --dry-run — preview without restarting +# docker_daily_restart.sh --log — verbose output +# docker_daily_restart.sh --status — show config and exit +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" @@ -26,7 +66,6 @@ if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi - success "Running as root" acquire_lock @@ -36,30 +75,41 @@ if ! command -v docker &>/dev/null; then notify "Docker daily restart failed — Docker not found on $(hostname)" "Docker Daily Restart" "warning" exit 1 fi - success "Docker found" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +# detect_hosts() sets MY_ID and aliases HOST*_DAILY_RESTART_CONTAINERS → DAILY_RESTART_CONTAINERS +detect_hosts + +if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then + warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to restart" + warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in master_host*.conf" + exit 0 +fi + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_CONTAINERS Containers: ${DAILY_RESTART_CONTAINERS[*]}" - echo "$ICON_RETRY Retries: $RETRY_COUNT" - echo "$ICON_TIME Sleep: ${SLEEP}s between retries" - echo "$ICON_NOTIFY Notifications: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_RETRY Retries: $RETRY_COUNT" + echo "$ICON_TIME Sleep: ${SLEEP}s between retries" + echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)" + echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted" -# ----------------------------------------------------------------------------------------------- -# FUNCTIONS -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ── FUNCTIONS ───────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Retries a docker command up to RETRY_COUNT times with SLEEP seconds between attempts. +# Usage: retry_docker docker restart ContainerName retry_docker() { local attempt=1 @@ -80,9 +130,132 @@ retry_docker() { return 1 } -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_CONTAINERS Daily Restart ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ── FUNCTIONS ───────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# Wraps docker commands with a 30 second timeout. +# Prevents a hung Docker daemon from causing the script to hang indefinitely. +# Usage: docker_cmd docker restart ContainerName +DOCKER_TIMEOUT=30 +docker_cmd() { + timeout "$DOCKER_TIMEOUT" "$@" + local exit_code=$? + if [[ "$exit_code" -eq 124 ]]; then + error "Docker command timed out after ${DOCKER_TIMEOUT}s: $*" + return 1 + fi + return "$exit_code" +} + +# Verifies a container is still running after restart. +# Gives the container a short settle period before checking. +# Returns 0 if running, 1 if crashed or stopped. +RESTART_VERIFY_WAIT=5 # seconds to wait before checking state post-restart +verify_running() { + local container="$1" + sleep "$RESTART_VERIFY_WAIT" + local state + state=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + if [[ "$state" != "true" ]]; then + error "$container failed to stay running after restart — may have crashed" + return 1 + fi + return 0 +} + +# Builds a dependency-safe restart order from DAILY_RESTART_CONTAINERS. +# Containers that are dependencies of others restart first. +# Returns ordered list in ORDERED_RESTART array. +build_restart_order() { + ORDERED_RESTART=() + local remaining=("${DAILY_RESTART_CONTAINERS[@]}") + local placed=() + + # First pass — add dependency containers that appear in our list + for container in "${remaining[@]}"; do + [[ -z "$container" ]] && continue + local is_dependency=false + # Check if this container is a dependency of any other in our list + for dep_string in "${WATCHDOG_DEPENDENCIES[@]:-}"; do + if [[ "$dep_string" == *"$container"* ]]; then + is_dependency=true + break + fi + done + # Also check associative array format + for dependent in "${!WATCHDOG_DEPENDENCIES[@]:-}"; do + if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then + is_dependency=true + break + fi + done + if [[ "$is_dependency" == true ]]; then + # Check not already placed + local already=false + for p in "${placed[@]:-}"; do [[ "$p" == "$container" ]] && already=true && break; done + if [[ "$already" == false ]]; then + ORDERED_RESTART+=("$container") + placed+=("$container") + fi + fi + done + + # Second pass — add remaining containers (dependents and independents) + for container in "${remaining[@]}"; do + [[ -z "$container" ]] && continue + local already=false + for p in "${placed[@]:-}"; do [[ "$p" == "$container" ]] && already=true && break; done + if [[ "$already" == false ]]; then + ORDERED_RESTART+=("$container") + placed+=("$container") + fi + done + + log "Restart order: ${ORDERED_RESTART[*]}" +} + +# Checks if a container is a dependent of the previously restarted container. +# If so, waits CONTAINER_DELAY before restarting to allow dependency to settle. +# Usage: check_dependency_delay "$container" "$last_restarted" +check_dependency_delay() { + local container="$1" + local last="$2" + [[ -z "$last" ]] && return + + local deps="${WATCHDOG_DEPENDENCIES[$container]:-}" + if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then + info "Waiting ${CONTAINER_DELAY}s — $container depends on $last..." + sleep "$CONTAINER_DELAY" + fi +} + +# Retries a docker command up to RETRY_COUNT times with SLEEP seconds between attempts. +# Uses docker_cmd wrapper for timeout protection on each attempt. +# Usage: retry_docker docker restart ContainerName +retry_docker() { + local attempt=1 + + while [[ "$attempt" -le "$RETRY_COUNT" ]]; do + info "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*" + + if docker_cmd "$@"; then + success "Succeeded on attempt $attempt" + return 0 + else + warn "Attempt $attempt failed" + (( attempt++ )) + [[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP" + fi + done + + error "Command failed after $RETRY_COUNT attempts: $*" + return 1 +} + +# ============================================================================================== +# ━━━ Daily Restart ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_CONTAINERS Daily Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_CONTAINERS Containers: ${DAILY_RESTART_CONTAINERS[*]}" @@ -92,30 +265,49 @@ echo "" START=$(date +%s) FAILED=() RESTARTED=() -STARTED=() +SKIPPED=() -for container in "${DAILY_RESTART_CONTAINERS[@]}"; do +# Build dependency-safe restart order +build_restart_order +echo "$ICON_GEAR Restart order: ${ORDERED_RESTART[*]}" +echo "" + +LAST_RESTARTED="" + +for container in "${ORDERED_RESTART[@]}"; do + [[ -z "$container" ]] && continue echo "━━━ $ICON_CONTAINERS $container ━━━" - if ! docker inspect "$container" &>/dev/null; then - error "$container does not exist — skipping" - FAILED+=("$container") + if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then + warn "$container does not exist — skipping" echo "" continue fi - STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) case "$STATUS" in true) echo "$ICON_RUNNING $container is running — restarting..." + # Wait if this container depends on the last one restarted + check_dependency_delay "$container" "$LAST_RESTARTED" + if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would restart $container" + RESTARTED+=("$container") else if retry_docker docker restart "$container"; then - echo "$ICON_STARTED $container restarted" - RESTARTED+=("$container") + # Verify container stayed running after restart + if verify_running "$container"; then + echo "$ICON_STARTED $container restarted and running ✅" + RESTARTED+=("$container") + LAST_RESTARTED="$container" + else + error "$container restarted but crashed immediately" + notify "$container crashed after restart on $(hostname)" "Docker Daily Restart" "warning" + FAILED+=("$container") + fi else error "Failed to restart $container after $RETRY_COUNT attempts" notify "$container failed to restart on $(hostname)" "Docker Daily Restart" "warning" @@ -124,20 +316,10 @@ for container in "${DAILY_RESTART_CONTAINERS[@]}"; do fi ;; false) - echo "$ICON_NOT_RUNNING $container is stopped — starting..." - - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would start $container" - else - if retry_docker docker start "$container"; then - echo "$ICON_STARTED $container started" - STARTED+=("$container") - else - error "Failed to start $container after $RETRY_COUNT attempts" - notify "$container failed to start on $(hostname)" "Docker Daily Restart" "warning" - FAILED+=("$container") - fi - fi + # Container was stopped — leave it stopped + # Intentionally stopped containers are not restarted + echo "$ICON_NOT_RUNNING $container is stopped — skipping (respecting stopped state)" + SKIPPED+=("$container") ;; *) error "Unknown status for $container: $STATUS" @@ -150,20 +332,21 @@ done END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "━━━━━ $ICON_SUMMARY DAILY RESTART SUMMARY ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}" -[[ ${#STARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Started: ${STARTED[*]}" -[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" +[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}" +[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]} (were stopped)" +[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" if [[ "$DRY_RUN" == true ]]; then echo "$ICON_WARN Status: DRY RUN — no changes made" elif [[ ${#FAILED[@]} -eq 0 ]]; then echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE" - notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#STARTED[@]} started on $(hostname)" "Docker Daily Restart" "normal" + notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#SKIPPED[@]} skipped (stopped) on $(hostname)" "Docker Daily Restart" "normal" else echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed" notify "Daily restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Daily Restart" "warning" diff --git a/Docker_Essentials/docker_network_connect.sh b/Docker_Essentials/docker_network_connect.sh index aa37860..f032cfe 100644 --- a/Docker_Essentials/docker_network_connect.sh +++ b/Docker_Essentials/docker_network_connect.sh @@ -1,1365 +1,236 @@ #!/bin/bash # ============================================================================================== -# ================================= MASTER CONFIGURATION ======================================= +# ================================= Docker Network Connect ===================================== # ============================================================================================== -# All user-facing variables for the unRAID script ecosystem. -# Scripts source this file — edit here, changes apply everywhere on next git pull. -# -# ── HOW THIS FILE WORKS ─────────────────────────────────────────────────────────────────────── -# Every script sources Master.conf and common.sh at startup. -# Change a value here and it affects all scripts that use it — no hunting through files. -# To disable something: comment it out with # rather than deleting it. -# To add a new rsync profile: add a key to each PROFILE_* array. -# To add or remove orchestrator jobs: edit the arrays in the ORCHESTRATORS section. -# -# ── INDEX ───────────────────────────────────────────────────────────────────────────────────── -# -# Section Description -# ─────────────────────────────────────────────────────────────────────────────────────────── -# HOST CONFIGURATION Server hostnames, SSH keys, Emby connection details, DATA_DIR -# LOGGING Enable or disable verbose logging -# NOTIFICATIONS unRAID native and Discord webhook settings -# GIT / REPO Gitea repository and SSH settings -# -# ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────── -# ARRAY START Scripts launched at array start (array_start.sh) -# DAILY SYNC MAINTENANCE Job list + media shares (daily_sync_maintenance.sh) -# WEEKLY SYNC MAINTENANCE Job list + sync jobs + sync settings (weekly_sync_maintenance.sh) -# MEDIA MANAGEMENT Job list for media_management.sh -# -# ── RSYNC ────────────────────────────────────────────────────────────────────────────────── -# RSYNC DEFAULTS Global fallback rsync options and limits -# REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort -# RSYNC PROFILE SYSTEM Per-profile overrides for appdata syncs -# -# ── FAILOVER ─────────────────────────────────────────────────────────────────────────────── -# FAILOVER Mutual container failover between two servers -# FAILOVER TEST Simulated outage settings for failover_test.sh -# DDNS Script-controlled DDNS — absolute rules -# INTERNET LOSS Containers to stop when internet is lost -# TIERED CONTAINER LISTS What each server runs for the other per tier -# TIER DELAY SETTINGS How long before each tier activates (minutes) -# RSYNC WRITEBACK JOBS Appdata synced back to primary on handback -# -# ── DOCKER ESSENTIALS ────────────────────────────────────────────────────────────────────── -# DOCKER DAILY RESTART Containers restarted daily -# DOCKER WEEKLY RESTART Containers restarted weekly -# DOCKER WATCHDOG Continuous two-tier self-healing container monitoring -# DOCKER NETWORK CONNECT Connect containers to extra networks on array start -# -# ── UNRAID ESSENTIALS ────────────────────────────────────────────────────────────────────── -# REBOOT User warning delay before scheduled reboot -# MOVER Mover stop timeout -# SYSLOG FILTER Docker veth noise filter file path -# PHP-FPM PHP-FPM max children config -# CLEAR LOGS System log file paths -# WEBGUI WATCHDOG WebGUI nginx + emhttp monitoring and restart -# -# ── MEDIA ────────────────────────────────────────────────────────────────────────────────── -# MEDIA PERMISSIONS Share list, mode and owner for permissions script -# MEDIA CLEANER Anime and media folder lists and file patterns -# ARR CLEANUP Lidarr, Sonarr, Radarr orphan file cleanup -# ARR FAILED/STALLED RECOVERY Auto blocklist + re-search failed imports and stalled downloads -# -# ── TRANSCODES ───────────────────────────────────────────────────────────────────────────── -# TRANSCODE MANAGER Ramdisk and SSD fallback transcode management -# TRANSCODE SERVER ARRAY Multi-server session monitoring (Emby, Jellyfin, Plex) -# -# ── MONITORS ─────────────────────────────────────────────────────────────────────────────── -# CERTIFICATE MONITOR SSL certificate expiry monitoring -# BACKUP VERIFY Random sample checksum verification against remote -# SMART HEALTH Drive SMART attribute monitoring -# ZFS MEMORY SNAPSHOT Weekly ZFS health and memory diagnostic report -# BANDWIDTH MONITOR Daily rsync transfer logging and weekly summary -# HEALTH DIGEST Aggregated system health digest — always/smart/weekly -# EMBY SESSION REPORT Weekly Emby usage statistics via API -# -# ── SYSTEM WATCHDOG ──────────────────────────────────────────────────────────────────────── -# SYSTEM WATCHDOG Continuous system health monitoring — last line of defense -# -# ============================================================================================== - -# ============================================================================================== -# ── HOST CONFIGURATION ──────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ Hosts ━━━ -# Hostnames must match exact Docker/unRAID hostnames — case sensitive. -# Used by detect_hosts() in common.sh to determine which server is local and which is remote. -# Both servers run identical scripts — host detection makes them bidirectional. - HOST1="unRAID-Gmer4Lfe" - HOST2="unRAID-Jayred365" - -# Data directory — persistent script state and statistics files. -# Array share — survives reboots, no flash drive wear. -# Created automatically if it doesn't exist. -# Only truly critical files (failover state, watchdog reboot log) stay on /boot/config. - DATA_DIR="/mnt/user/appdata/unraid_scripts/data" - -# SSH keys for server-to-server rsync and failover container operations. -# Both keys must be in /root/.ssh/ and authorised in the remote server's authorized_keys. - HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" - HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" - -# ━━━ Emby ━━━ -# Defined once here — referenced by transcode_manager.sh, emby_session_report.sh, -# emby_database_repair.sh, weekly_sync_maintenance.sh, and TRANSCODE_SERVERS array. -# API key: Emby Dashboard → API Keys → + New Key - HOST1_EMBY_CONTAINER="Emby" - HOST1_EMBY_URL="http://localhost:8096" - HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829" - - HOST2_EMBY_CONTAINER="Emby-Jayred365" - HOST2_EMBY_URL="http://localhost:8096" # same port — different server, different key - HOST2_EMBY_API_KEY="your-host2-emby-api-key" - -# ============================================================================================== -# ── LOGGING ─────────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# Controls verbose [LOG] output across all scripts. -# true = show detailed [LOG] lines — useful for debugging or first-time setup -# false = show only user-facing output — cleaner for scheduled runs - ENABLE_LOGGING=true - -# ============================================================================================== -# ── NOTIFICATIONS ───────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# unRAID native notification system — integrates with the bell icon in the WebGUI. -# normal = job completed successfully / warning = something failed or needs attention - NOTIFY_UNRAID=true - -# Discord webhook URL — leave blank to disable - DISCORD_WEBHOOK="" - -# ============================================================================================== -# ── GIT / REPO ──────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# Gitea self-hosted repository — used by git_pull_execute.sh. -# Detects Gitea container location at runtime — works through failover automatically. -# Falls back to GITEA_DOMAIN if local and Tailscale both fail. - GITEA_CONTAINER="Gitea" # exact Docker container name - GITEA_REPO_PATH="FailedProxy/Unraid_Scripts.git" # repo path on Gitea server - GITEA_DOMAIN="" # e.g. git.gmer4lfe.com — requires NPM + DNS setup - TARGET_DIR="/mnt/user/appdata/unraid_scripts" # where scripts are cloned to - GITEA_SSH_KEY="/root/.ssh/unraid_gitea" # SSH key for authenticating to Gitea - SSH_PORT=221 # Gitea SSH port (default 22, Gitea often uses 222/221) - -# ============================================================================================== -# ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# All orchestrator job lists live here — edit arrays to add/remove scripts. -# No changes to orchestrator scripts needed when adding or removing jobs. - -# ━━━ Array Start ━━━ -# Scripts launched by array_start.sh when the array comes online. -# Launched in order — each as a background process. -# One-shot scripts (ramdisk, syslog, fpm, network) run and exit naturally. -# Continuous scripts (watchdogs, failover) run until array stops. - -ARRAY_START_SCRIPTS=( - "Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts - "unRAID_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill - "unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning - "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers - "unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop - "Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop -# "Failover/failover.sh" # mutual failover — continuous loop -) - -# ━━━ Daily Sync Maintenance ━━━ -# daily_sync_maintenance.sh runs the media share sync built into the script first, -# then iterates DAILY_MAINTENANCE_SCRIPTS for additional jobs. -# Schedule: 0 1 * * * (1am daily) - -DAILY_MAINTENANCE_SCRIPTS=( - "git_pull_execute.sh" # pull latest scripts — always runs first - "Docker_Essentials/docker_daily_restart.sh" # daily container restarts -) - -# Media shares synced daily by daily_sync_maintenance.sh. -# Each server syncs only the shares it owns (source of truth) — direction is automatic. -# HOST1 pushes its truth shares to HOST2. HOST2 pushes its truth shares to HOST1. -# Never both pushing the same share — one server is always the truth holder. -# These shares use DEFAULT_RSYNC_OPTS — no profile entry needed. -# For shares needing custom options or container stops — create a profile in the RSYNC section. - -HOST1_DAILY_SYNC_SHARES=( - /mnt/user/Books - /mnt/user/Intros - /mnt/user/Kids_Movies - /mnt/user/Kids_Tv_Shows - /mnt/user/Movies - /mnt/user/Music - /mnt/user/Music_Videos - /mnt/user/Nextcloud - /mnt/user/stand-up_comedy - /mnt/user/Sports - /mnt/user/Tv_Shows - /mnt/user/Anime_Shows-Old - /mnt/user/Anime_Movies-Old -) - -HOST2_DAILY_SYNC_SHARES=( - /mnt/user/Anime_Movies - /mnt/user/Anime_Shows -) - -# Personal encrypted shares — synced for offsite backup, independent of media shares. -# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content. -# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting. -HOST1_PERSONAL_SHARES=( - # /mnt/user/Gmer4Lfe-Personal # uncomment after creating encrypted dataset -) - -HOST2_PERSONAL_SHARES=( - # /mnt/user/Jayred365-Personal # uncomment after creating encrypted dataset -) - -# ━━━ Weekly Sync Maintenance ━━━ -# weekly_sync_maintenance.sh handles the critical sync built into the script first: -# stop containers both sides → pull updates → sync Emby + Critical-Data → restart -# Then iterates WEEKLY_MAINTENANCE_SCRIPTS for additional jobs. -# Schedule: 30 2 * * 0 (Sunday 2:30am) - -WEEKLY_MAINTENANCE_SCRIPTS=( - "Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync -) - -# Shares synced by weekly_sync_maintenance.sh during the maintenance window. -# Containers are stopped both sides before these sync — full clean state guaranteed. -# Profiles drive container stops, excludes, and options — configure in RSYNC section. -# Order matters — Emby first, then auth stack. -WEEKLY_SYNC_JOBS=( - "/mnt/user/Media_Server/Emby" # emby profile — full clean mirror - "/mnt/user/appdata-Failover/Critical-Data" # critical-data profile — auth stack -) - -# Container update toggles for the weekly sync window. -# Containers are already stopped for the sync — updates pull at no extra downtime. -# Both false → sync only, no updates. -# Toggle false temporarily to skip updates without changing the schedule. - CRITICAL_SYNC_UPDATES=true # pull container updates locally - CRITICAL_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH - -# ━━━ Media Management ━━━ -# Job list run directly by daily_sync_maintenance.sh after the media share sync. -# Runs sequentially — permissions first, then cleaners, then arr cleanup. -# Comment out any job to disable without removing it. -# Each individual script can still be run manually for one-off maintenance. - -MEDIA_MANAGEMENT_JOBS=( - "Media/media_shares_permissions.sh" # apply permissions — runs first - "Media/media_cleaner.sh anime" # remove junk from anime shares - "Media/media_cleaner.sh media" # remove junk from media shares -# "Media/lidarr_cleanup.sh" # remove orphaned music files -# "Media/sonarr_cleanup.sh" # remove orphaned TV files -# "Media/radarr_cleanup.sh" # remove orphaned movie files -) - -# ============================================================================================== -# ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ Rsync Defaults ━━━ -# Global fallback values used when no profile match is found. -# Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed. -# Appdata shares match profiles by directory basename (lowercased). -# If a profile key exists it overrides the global. If missing the global is used. - - BW_LIMIT=12500 # KB/s — 12500 ≈ 100Mbit — network transfer speed cap - RETRY_COUNT=3 # retry attempts if rsync fails before giving up - SLEEP=300 # seconds between retry attempts - CRITICAL_CONTAINER_NAMES=() # containers to stop on REMOTE before rsync — profiles override - DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override - CONTAINER_DELAY=5 # seconds to wait before starting delayed containers - EXCLUDE_DIRS=() # directories to exclude from transfer — profiles override - -# --delete removes files on remote that no longer exist on source (mirror behaviour) -# --inplace writes directly to destination — better for large files, avoids temp copies -# --no-whole-file forces delta transfer even on fast connections — sends only changed blocks - DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file) - -# ━━━ Remote Health Checks ━━━ -# Pre-flight check — aborts if remote rootfs (/) usage is at or above this percentage. -# When remote array is down, rsync writes land on rootfs — fills fast and crashes the server. - ROOTFS_WARN=75 - -# ━━━ Rsync Profile System ━━━ -# Profiles allow per-share rsync behaviour without touching script logic. -# Profile key matched by basename of directory passed to rsync.sh (lowercased). -# Override with --profile=name flag. -# -# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit DEFAULT_RSYNC_OPTS. -# List ALL desired options explicitly when defining a profile. -# -# Current profiles: -# arrs_stack — arr databases — lower bandwidth, containers stopped for consistency -# critical-data — auth stack — containers stopped both sides, Authelia delayed start -# gmer4lfe — server-specific appdata — no container stops needed -# important-data — NextCloud + Postgres — NextCloud delayed start after Postgres -# emby — weekly clean sync — both Emby stopped, full mirror, minimal excludes -# called by weekly_sync_maintenance.sh only — do NOT schedule separately -# emby-failover — frequent dirty sync — Emby stays running, WAL excluded, critical data only -# also used for failover writeback on handback - -declare -A PROFILE_RSYNC_OPTS=( - [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" - [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete" - [gmer4lfe]="-av --info=progress2 --bwlimit=$BW_LIMIT" - [important-data]="-av --human-readable --bwlimit=$BW_LIMIT" - [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" - [emby-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" -) - -# Per-profile bandwidth limits in KB/s — overrides global BW_LIMIT for that profile only -# Lower for shares running alongside other jobs, higher for time-sensitive critical data -declare -A PROFILE_BW_LIMIT=( - [arrs_stack]=5000 # lower — runs alongside other syncs, avoids saturating link - [critical-data]=9500 # high — small dataset, get it synced fast and clean - [gmer4lfe]=8000 - [important-data]=9500 # high — database sync needs to be fast - [emby]=8000 # medium — large full mirror, steady transfer - [emby-failover]=9500 # high — small critical dataset, sync as fast as possible -) - -# Retry attempts per profile — how many times to retry before giving up on a failed sync -declare -A PROFILE_RETRY_COUNT=( - [arrs_stack]=3 - [critical-data]=3 - [gmer4lfe]=3 - [important-data]=3 - [emby]=3 - [emby-failover]=3 -) - -# Seconds to wait between retry attempts -# emby-failover shorter — frequent sync, faster retry on transient failures -declare -A PROFILE_SLEEP=( - [arrs_stack]=300 - [critical-data]=300 - [gmer4lfe]=300 - [important-data]=300 - [emby]=300 - [emby-failover]=120 # shorter — frequent dirty sync, retry faster -) - -# Containers stopped on BOTH LOCAL and REMOTE servers before rsync. -# Local stops first — flushes databases cleanly before pushing data out. -# Remote stops next — prevents writes to destination while receiving. -# Only containers that were running get restarted — stopped containers stay stopped. -# Same container names on both servers — consistent naming is required by this ecosystem. -# If a container is not found on a server it is skipped gracefully, not errored. -# SPACE-SEPARATED STRINGS — converted to array at runtime -declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( - [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" - [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary" - [gmer4lfe]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe" - [important-data]="Postgres-NextCloud NextCloud" - [emby]="Emby" # weekly clean sync — both Emby instances stopped, WAL checkpointed - [emby-failover]="" # dirty sync — Emby stays running both sides, WAL excluded from sync -) - -# Containers that need a delay before starting after rsync completes. -# Database containers must be accepting connections before dependent apps start. -# Authelia waits for Mariadb + Redis. NextCloud waits for Postgres. -# SPACE-SEPARATED STRINGS — converted to array at runtime -declare -A PROFILE_DELAYED_CONTAINERS=( - [arrs_stack]="" - [critical-data]="Authelia Authelia-Secondary" # wait for Mariadb + Redis to be ready - [gmer4lfe]="" - [important-data]="NextCloud" # wait for Postgres to accept connections - [emby]="" - [emby-failover]="" -) - -# Seconds to wait before starting delayed containers -# 15s gives Mariadb, Redis, and LLDAP time to accept connections before Authelia starts -declare -A PROFILE_CONTAINER_DELAY=( - [arrs_stack]=5 - [critical-data]=15 # Mariadb + Redis need time to accept connections - [gmer4lfe]=5 - [important-data]=10 # Postgres needs time before NextCloud - [emby]=5 - [emby-failover]=5 -) - -# Directories excluded from rsync transfer per profile -# emby-failover excludes WAL files — safe to sync while Emby is running -# emby clean sync only excludes logs, transcodes, cache — full metadata mirror -# SPACE-SEPARATED STRINGS — converted to array at runtime -declare -A PROFILE_EXCLUDE_DIRS=( - [arrs_stack]="logs *.tmp" - [gmer4lfe]="logs *.tmp" - [important-data]="logs *.tmp" - [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt" - [emby]="logs transcodes cache crash*" - # emby-failover: Emby running, WAL excluded — only safe critical data synced - # users.db, library.db, authentication.db, config/ — everything else excluded - [emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root" -) - -# Skip per-disk space check for these profiles — appdata syncs go to cache/appdata -# not to array disks, so disk space check is irrelevant and just slows things down -declare -A PROFILE_SKIP_DISK_CHECK=( - [arrs_stack]=true - [critical-data]=true - [gmer4lfe]=true - [important-data]=true - [emby]=true - [emby-failover]=true -) - -# ============================================================================================== -# ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Mutual container failover between two unRAID servers. -# Each server runs Failover/failover.sh independently via array_start.sh. -# All decisions based on two pings: remote reachable + internet reachable. -# -# States: NORMAL | FAILOVER | NO_INTERNET | DARK -# -# DDNS rules — absolute: -# Internet loss → stop own DDNS immediately -# Failover → start remote DDNS first (Tier 1) -# Handback → stop remote DDNS → rsync → start containers → start local DDNS last -# -# Tiers: -# Tier 1 — Immediate — vital services + Live TV -# Tier 2 — configurable delay — productivity services -# Tier 3 — configurable delay — secondary services -# Tier 4 — configurable delay — arrs + downloaders - - EXTERNAL_IP="8.8.8.8" - FAILOVER_CHECK_INTERVAL=120 - FAILOVER_HANDBACK_STRIKES=2 - FAILOVER_STATE_FILE="/boot/config/failover_state.db" - -# ━━━ Failover Test ━━━ - FAILOVER_TEST_BLOCK_WAIT=150 - FAILOVER_TEST_HANDBACK_WAIT=360 - -# ━━━ DDNS ━━━ -HOST1_DDNS_CONTAINERS=( - "Gmer4Lfe.com" -) - -HOST2_DDNS_CONTAINERS=( - "Gmer4Lfe.us" -) - -# ━━━ Internet Loss ━━━ -FAILOVER_HOST1_STOP_ON_NO_NET=( - "Gmer4Lfe.com" -) - -FAILOVER_HOST2_STOP_ON_NO_NET=( - "Gmer4Lfe.us" -) - -# ━━━ Tiered Container Lists ━━━ - -# HOST1 runs for HOST2 -FAILOVER_HOST1_RUNS_FOR_HOST2_IMMEDIATE=( - "Gmer4Lfe.us" - "VaultWarden-Jayred365" - # "container-placeholder" -) - -FAILOVER_HOST1_RUNS_FOR_HOST2_2HR=( - # "container-placeholder" -) - -FAILOVER_HOST1_RUNS_FOR_HOST2_6HR=( - # "container-placeholder" -) - -FAILOVER_HOST1_RUNS_FOR_HOST2_18HR=( - # "container-placeholder" -) - -# HOST2 runs for HOST1 -FAILOVER_HOST2_RUNS_FOR_HOST1_IMMEDIATE=( - "Gmer4Lfe.com" - "Emby" - "VaultWarden-Gmer4Lfe" - "Dispatcharr" - "Dispatcharr-Basic" - "Dispatcharr-Iptv-Users" - "ErsatzTV-Emby" -) - -FAILOVER_HOST2_RUNS_FOR_HOST1_2HR=( - "Postgres-NextCloud" - "NextCloud" - "PostgreSQL_Immich" - "Immich-Gmer4Lfe" - # "container-placeholder" -) - -FAILOVER_HOST2_RUNS_FOR_HOST1_6HR=( - "Gitea" - # "container-placeholder" -) - -FAILOVER_HOST2_RUNS_FOR_HOST1_18HR=( - "Sonarr" - "Radarr" - "Lidarr" - "Readarr" - "Prowlarr" - "Bazarr" - "SABnzbd-Gmer4Lfe" - "Qbittorrent-Gmer4Lfe" - "LidaTube" - "Pinchflat" - "ChannelTube" - # "container-placeholder" -) - -# ━━━ Tier Delay Settings ━━━ -# How long the primary server must be down before each tier activates — in minutes. -# Tier 1 is always immediate — Live TV and media can't wait. -# Set independently per host — adjust based on hardware and what's worth starting. -# Longer delays = less resource usage on covering server but slower recovery. -# -# HOST1's containers running on HOST2 (HOST1 is down): -HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich — can wait -HOST1_TIER3_DELAY=720 # 12 hours — secondary services — Gitea etc. -HOST1_TIER4_DELAY=1440 # 24 hours — full workflow — arrs and downloaders - -# HOST2's containers running on HOST1 (HOST2 is down): -HOST2_TIER2_DELAY=240 -HOST2_TIER3_DELAY=720 -HOST2_TIER4_DELAY=1440 - -# ━━━ Rsync Writeback Jobs ━━━ -# Syncs critical appdata BACK to primary server during handback after failover. -# Containers are stopped before writeback runs — clean source, no competing writes. -# Purpose: primary comes back online with the state that built up during its outage -# (watch states, auth changes, library updates that happened on HOST2) -# -# HOST*_TIER1_WRITEBACK_DELAY: -# Short outages skip Tier 1 writeback — primary state is more reliable than dirty sync data -# Only writeback if outage lasted longer than this many minutes -# 60 minutes = if HOST1 was down less than 1hr, don't bother writing back Emby -# -# Tier 4 writeback automatically syncs HOST*_DAILY_SYNC_SHARES back — no need to list those here -# Only add paths that are NOT in DAILY_SYNC_SHARES and need writeback after extended outage - -HOST1_TIER1_WRITEBACK_DELAY=60 # minutes — skip Emby writeback if outage under 1hr -HOST2_TIER1_WRITEBACK_DELAY=60 - -# HOST1 writeback — run by HOST2 during HOST1 handback -FAILOVER_HOST1_WRITEBACK_TIER1=( - "/mnt/user/Media_Server/Emby" # watch states, playstates built up during outage -) - -FAILOVER_HOST1_WRITEBACK_TIER2=( - "/mnt/user/appdata-Failover/Important-Data" # NextCloud + Postgres — files added during outage -) - -FAILOVER_HOST1_WRITEBACK_TIER3=( - # "location-placeholder" -) - -FAILOVER_HOST1_WRITEBACK_TIER4=( - # Edge cases outside HOST1_DAILY_SYNC_SHARES - "/mnt/user/appdata-Failover/Arrs_Stack" # arr databases — downloads queued during outage -) - -# HOST2 writeback — run by HOST1 during HOST2 handback -FAILOVER_HOST2_WRITEBACK_TIER1=( - # "/mnt/user/appdata-Failover/Jayred365-Emby" -) - -FAILOVER_HOST2_WRITEBACK_TIER2=( - # "/mnt/user/appdata-Failover/Jayred365-Important" -) - -FAILOVER_HOST2_WRITEBACK_TIER3=( - # "location-placeholder" -) - -FAILOVER_HOST2_WRITEBACK_TIER4=( - # Edge cases outside HOST2_DAILY_SYNC_SHARES - "/mnt/user/appdata-Failover/Arrs_Stack" -) - -# ============================================================================================== -# ── DOCKER ESSENTIALS ───────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ Docker Daily Restart ━━━ -# Containers restarted every day by docker_daily_restart.sh via daily_sync_maintenance.sh. -# These containers run better with a daily restart — not just "keeping things fresh". -# Dispatcharr specifically degrades over time without restart — daily is intentional. -# Schedule is set in daily_sync_maintenance.sh — runs at 1am as part of daily window. -# Case-sensitive — must match exact Docker container names. -DAILY_RESTART_CONTAINERS=( - "NginxProxyManager" - "Authelia" - "Dispatcharr-Iptv-Users" - "Dispatcharr" # Live TV scheduler — degrades without daily restart - "Dispatcharr-Basic" - "ErsatzTV-Emby" -) - -# ━━━ Docker Weekly Restart ━━━ -# Less critical services restarted weekly by docker_weekly_restart.sh. -# Called by weekly_sync_maintenance.sh Sunday 2:30am — containers already stopped -# for the weekly sync window so restart adds zero extra downtime. -# Weekly restarts also catch any pending image updates not applied during weekly sync. -WEEKLY_RESTART_CONTAINERS=( - "NextCloud" - "Organizrv2-Gmer4Lfe" - "AdGuard-Home" - "Immich-Gmer4Lfe" -) - -# ━━━ Docker Watchdog ━━━ -# Continuous two-tier self-healing container monitoring. -# Started by array_start.sh — runs until array stops. -# Re-sources Master.conf each cycle — add/remove containers without restarting watchdog. -# Silent when all healthy — only logs when something needs attention. -# Heartbeat fires periodically as proof of life even when everything is healthy. -# -# Tier 1 — strict monitoring of explicitly configured containers: -# Memory hard limits — immediate restart if container exceeds limit -# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT sustained strikes -# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT failed checks -# Required containers — must always be running, strike + skip list with auto-clear -# -# Tier 2 — global health scan of ALL running containers: -# Unhealthy status — Docker HEALTHCHECK unhealthy → restart -# OOM killed — kernel killed container → restart + notify -# Crash loop detection — RestartCount climbing → notify, critical above limit -# Dead containers — remove and restart -# Unexpected exits — non-zero exit code → restart - -# Memory hard limits in MB — immediate restart if exceeded -# Container restarted the moment it crosses this line — no strike system -# 20GB=20480 16GB=16384 12GB=12288 10GB=10240 -# 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024 -declare -A WATCHDOG_CONTAINERS=( - ["Emby"]=16384 - ["LidaTube"]=6144 - ["Tdarr"]=6144 - ["Code-Server"]=1024 -) - -# HTTP health check URLs — checked every cycle, strike system before restart -# Container must respond with HTTP 200 within CURL_TIMEOUT seconds -# Per-host — HOST1 and HOST2 may run different containers on different ports -declare -A HOST1_WATCHDOG_CONTAINER_URLS=( - ["Emby"]="http://localhost:8096" -) - -declare -A HOST2_WATCHDOG_CONTAINER_URLS=( - ["Emby"]="http://localhost:8096" -) - -# Required containers — must always be running -# Strike system: SYS_WATCHDOG_STRIKE_LIMIT strikes before restart attempt -# Persistent skip list: added after WATCHDOG_CONTAINER_RESTART_LIMIT restarts in window -# Skip list auto-clears when container recovers — no manual intervention needed -# Per-host — each server has different critical containers -HOST1_WATCHDOG_REQUIRED_CONTAINERS=( - "NginxProxyManager" - "Lldap-Gmer4Lfe" - "Authelia" - "Mariadb-Authelia" - "Redis-Authelia" - "Authelia-Secondary" - "Redis-Authelia-Secondary" -) - -HOST2_WATCHDOG_REQUIRED_CONTAINERS=( - "NginxProxyManager" - # add HOST2 required containers here -) - -# Strike state file — /tmp resets on reboot which is correct -# Fresh start after reboot means no stale strikes carrying over - WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db" - -# CPU thresholds — normalised against total core count automatically at runtime -# SOFT = warn only, HARD = strike toward restart -# CPU_FAIL_LIMIT = consecutive HARD strikes before restart - SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU - HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU - CPU_FAIL_LIMIT=2 # consecutive hard CPU strikes before container restart - -# Memory soft threshold — warn when container reaches this % of its WATCHDOG_CONTAINERS hard limit -# Does not trigger restart — informational only - SOFT_MEM_THRESHOLD=80 - -# HTTP responsiveness — consecutive failed checks before restart -# CURL_TIMEOUT = seconds before curl gives up on a single check - RESP_FAIL_LIMIT=2 # consecutive failed checks before restart - CURL_TIMEOUT=5 # seconds per check before timeout - -# How often the watchdog runs its checks -# 900 = 15 minutes — long enough to not be noisy, short enough to catch issues quickly -# Containers have this long to recover before next check - DOCKER_WATCHDOG_INTERVAL=900 # seconds between watchdog cycles - -# Heartbeat — proof of life logged periodically even when everything is healthy -# Useful to confirm the watchdog is still running without flooding logs - DOCKER_WATCHDOG_HEARTBEAT=true # true = log heartbeat / false = completely silent - DOCKER_WATCHDOG_HEARTBEAT_HOURS=1 # how often to log heartbeat (hours) - -# Tier 2 master toggle — set false to disable global container scanning entirely -# When false only WATCHDOG_CONTAINERS and required containers are monitored - WATCHDOG_SCAN_ALL=true - -# Containers to skip in Tier 2 scan entirely -# Useful for containers that legitimately exit/restart frequently -WATCHDOG_SCAN_IGNORE=( - "DashGate" - "PIA-WG-Config-Generator" - "Aperture" - "Aperture-Kids" - "pgvector-18-Apeture-Kids" - "Pgvector18-Aperture" -) - -# Individual Tier 2 check toggles — disable specific checks without disabling Tier 2 - WATCHDOG_RESTART_UNHEALTHY=true # restart containers with Docker HEALTHCHECK = unhealthy - WATCHDOG_RESTART_DEAD=true # restart containers in dead state - WATCHDOG_RESTART_CRASHED=true # restart containers that exited with non-zero code - WATCHDOG_NOTIFY_OOM=true # notify + restart OOM killed containers - WATCHDOG_NOTIFY_CRASHLOOP=true # notify when Docker RestartCount keeps climbing - -# Crash loop threshold — notify critical if Docker has restarted this many times total -# Above this number the notification escalates to critical — manual intervention needed - WATCHDOG_CRASH_LIMIT=5 - -# Startup grace period — skip restarts while system is still booting after array start -# Prevents watchdog from restarting containers that are legitimately still initializing - WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures - -# Restart loop protection — stops hammering a broken container -# If watchdog restarts a container more than LIMIT times in WINDOW hours → skip list -# Skip list auto-clears when container recovers healthy - WATCHDOG_CONTAINER_RESTART_LIMIT=3 # max watchdog restarts allowed in window - WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours - WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db" - # rolling restart history for loop detection - -# Notification batching — one clean summary per cycle instead of one ping per event -# true = batch all events into one notification at end of cycle -# false = send one notification per event (noisy on busy systems) - WATCHDOG_BATCH_NOTIFY=true - -# Dependency ordering — skip restarting a container if its dependency is also down -# Prevents restarting Authelia before its database is ready -# Space-separated list of dependencies per container -declare -A WATCHDOG_DEPENDENCIES=( - ["Authelia"]="Mariadb-Authelia Redis-Authelia" - ["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary" - ["NextCloud"]="Postgres-NextCloud" -) - -# ━━━ Docker Network Connect ━━━ # Ensures custom Docker networks exist then connects specified containers to them. -# Two operations in one pass — no separate ensure script needed. +# Run once at array start via ARRAY_START_SCRIPTS — idempotent, safe to re-run anytime. # -# Step 1 — network ensure: -# Checks each network in NETWORK_CONNECT_NETWORKS -# Missing → creates it (bridge driver, Docker assigns subnet automatically) -# Exists → skips creation -# unRAID updates occasionally wipe custom networks — this self-heals on next array start +# ── WHAT IT DOES ────────────────────────────────────────────────────────────────────────────── +# For each network in NETWORK_CONNECT_NETWORKS: +# 1. Check if network exists +# → missing → create it (bridge driver, Docker assigns subnet automatically) +# notifies on creation — unexpected, usually means unRAID wiped networks +# → exists → skip creation silently +# 2. Connect each container in NETWORK_CONNECT_CONTAINERS to the network +# → already connected → skip cleanly +# → not connected → connect it +# → container not found → warn and skip (not an error — may not be running yet) # -# Step 2 — container connect: -# Connects each container in NETWORK_CONNECT_CONTAINERS to each network -# Container not found → warn and skip (not an error — may not be running on this host) -# Already connected → skip cleanly +# ── USE CASE ────────────────────────────────────────────────────────────────────────────────── +# high-availability is the main custom network — shared by most containers. +# After a unRAID update wipes custom networks → recreated automatically at next array start. +# Containers on their own networks (NextCloud AIO etc.) can be added to +# NETWORK_CONNECT_CONTAINERS so they also join high-availability without touching their +# primary network configuration. # -# Use case: -# high-availability is the main custom bridge — most containers run here -# Containers with their own networks (NextCloud AIO, etc.) can be listed in -# NETWORK_CONNECT_CONTAINERS to gain access to high-availability -# without touching their primary network — they simply join both -NETWORK_CONNECT_CONTAINERS=( -# "memcached" -# "Npm-CrowdSec" -) - -NETWORK_CONNECT_NETWORKS=( - "high-availability" # must exist before array start — create in Docker settings -) - -# ============================================================================================== -# ── UNRAID ESSENTIALS ───────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ Reboot ━━━ -# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots. -# Gives users time to save work — 300s = 5 minutes - REBOOT_SLEEP=300 - -# ━━━ Mover ━━━ -# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process. -# Gives mover time to finish current file transfer before being interrupted. - MOVER_STOP_TIMEOUT=300 - -# ━━━ Syslog Filter ━━━ -# Path for the rsyslog filter file that suppresses Docker veth interface noise. -# Docker creates a new veth interface for each container — generates hundreds of -# log lines per hour that have no diagnostic value. Filter removes them at source. - FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" - -# ━━━ PHP-FPM ━━━ -# Higher max_children allows more concurrent PHP requests to the unRAID WebGUI. -# Default is very low — increasing it prevents WebGUI slowdowns under load. -# 250 is safe for servers with 32GB+ RAM. - PHP_CONF="/etc/php-fpm.d/www.conf" - PHP_MAX_CHILDREN=250 - -# ━━━ Clear Logs ━━━ -# System log files cleared weekly to prevent rootfs fill over time. -# These grow continuously — without clearing they eventually consume all rootfs space. - LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) - -# ━━━ WebGUI Watchdog ━━━ -# Monitors unRAID WebGUI responsiveness — escalates through nginx restart → emhttp restart. -# Separate from docker_watchdog — this monitors the unRAID UI itself, not containers. -# WEBGUI_NGINX_WAIT = seconds after nginx restart before rechecking -# WEBGUI_EMHTTP_WAIT = seconds after emhttp restart before rechecking - WEBGUI_URL="http://localhost" - WEBGUI_TIMEOUT=5 # seconds before curl gives up on WebGUI check - WEBGUI_NGINX_WAIT=15 # seconds after nginx restart before rechecking - WEBGUI_EMHTTP_WAIT=30 # seconds after emhttp restart before rechecking - -# ============================================================================================== -# ── MEDIA ───────────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ Media Permissions ━━━ -# Applied recursively to all shares in MEDIA_PERMISSION_SHARES by media_shares_permissions.sh. -# Runs first in MEDIA_MANAGEMENT_JOBS — arr cleanup scripts depend on correct ownership. -# 777 mode = read/write/execute for all users — standard for unRAID media shares -# nobody:users = standard unRAID media share ownership - PERMISSIONS_MODE="777" - PERMISSIONS_OWNER="nobody:users" - -MEDIA_PERMISSION_SHARES=( - /mnt/user/Anime_Movies - /mnt/user/Anime_Movies-Old - /mnt/user/Anime_Shows - /mnt/user/Anime_Shows-Old - /mnt/user/appcache - /mnt/user/Books - /mnt/user/Downloads - /mnt/user/Games - /mnt/user/Intros - /mnt/user/Kids_Movies - /mnt/user/Kids_Tv_Shows - /mnt/user/Movie_Recordings - /mnt/user/Movies - /mnt/user/Music - /mnt/user/Music_Videos - /mnt/user/Photo - /mnt/user/Sports - /mnt/user/stand-up_comedy - /mnt/user/Temp_Storage - /mnt/user/Tv_Recordings - /mnt/user/Tv_Shows - /mnt/user/YouTube -) - -# ━━━ Media Cleaner ━━━ -# Removes junk files from media shares — two profiles: anime and media. -# Called via MEDIA_MANAGEMENT_JOBS. Run manually: Media/media_cleaner.sh anime|media - -ANIME_CLEAN_FOLDERS=( - /mnt/user/Anime_Movies - /mnt/user/Anime_Movies-Old - /mnt/user/Anime_Shows - /mnt/user/Anime_Shows-Old -) - -MEDIA_CLEAN_FOLDERS=( - /mnt/user/Kids_Movies - /mnt/user/Kids_Tv_Shows - /mnt/user/Movies - /mnt/user/Music - /mnt/user/Sports - /mnt/user/stand-up_comedy - /mnt/user/Tv_Shows -) - -ANIME_FILE_PATTERNS=( - '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' - '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' - '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp' - '*.log' '*.json' -) - -MEDIA_FILE_PATTERNS=( - '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' - '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' - '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp' - '*.log' '*.json' '*.iso' '*.lrc' -) - -# ━━━ Arr Cleanup ━━━ -# Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs. -# Compares tracked file paths from API against disk — deletes untracked files older than ORPHAN_AGE. -# detect_hosts() selects correct URL, API key, and root path at runtime. +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# Docker daemon check — verifies daemon is responsive before any network operations +# Command validation — validates unRAID notify script before use +# Timeout protection — all docker commands wrapped in timeout — daemon hangs cannot stall +# Empty array guards — warns and exits cleanly if arrays are unconfigured +# Silent by default — only warnings and errors produce output (v3.4 standard) +# network creation always warns — unexpected, means networks were wiped +# Idempotent — safe to run multiple times, skips what is already correct # -# Protected patterns are NEVER deleted — cover art, metadata, subtitles generated by the arr -# are not included in the tracked file API response but must not be deleted. +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_NETWORK_CONNECT_NETWORKS — networks to ensure exist +# HOST*_NETWORK_CONNECT_CONTAINERS — containers to connect to each network +# Aliased by detect_hosts() — script uses unprefixed names # -# API versions and endpoint patterns: -# Sonarr v4 → /api/v3/series (get IDs) → /api/v3/episodefile?seriesId=X per series -# Radarr v5 → /api/v3/movie (get IDs) → /api/v3/moviefile?movieId=X per movie -# Lidarr v3 → /api/v1/artist (get IDs) → /api/v1/trackFile?artistId=X per artist -# All require per-ID loops — bulk endpoints removed in newer versions -# -# Version checking — scripts verify the arr major version matches before running -# If the arr updates and breaks the API the script exits safely before touching files -# Update the MAJOR version here when the script is updated to support a new version -# MINOR = 0 means any minor version within that major is accepted - - SONARR_VERSION_MAJOR=4 # tested major version — script exits if major differs - RADARR_VERSION_MAJOR=6 # tested major version — script exits if major differs - LIDARR_VERSION_MAJOR=3 # tested major version — script exits if major differs - -# ── Lidarr ──────────────────────────────────────────────────────────────────────────────────── -HOST1_LIDARR_URL="http://192.168.50.2:8686" -HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc" -HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New" -LIDARR_LOCK_WARN_AGE=3600 # 1hr — large libraries take time, not stuck - -# Container path → host path translation -# Lidarr stores file paths using container paths — script scans host paths -# Add one entry per root folder configured in Lidarr Settings → Media Management → Root Folders -declare -A HOST1_LIDARR_PATH_MAP=( - ["/ext-music"]="/mnt/user/Music-New" -) -declare -A HOST2_LIDARR_PATH_MAP=( - # HOST2 does not run Lidarr — fill in if that changes - # ["/ext-music"]="/mnt/user/Music-New" -) - -LIDARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion - # protects files that may still be mid-import or recently downloaded -LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") -LIDARR_PROTECTED_PATTERNS=( - # Metadata - "*.nfo" "*.tbn" - # Images — album art, artist images, Emby artwork - "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" - "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" - "banner.*" "thumb.*" "landscape.*" - "folder.*" "cover.*" "album.*" "artist.*" "disc.*" - # Lyrics - "*.lrc" -) - # NEVER deleted — cover art, metadata, lyrics - # Lidarr generates these but doesn't include them in trackFile API - # Without this protection cleanup would delete all your artwork -LIDARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this -LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run - # protects against API returning partial data on a bad day -LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count" - # persists last known tracked count for percentage comparison - -# ── Sonarr ──────────────────────────────────────────────────────────────────────────────────── -HOST1_SONARR_URL="http://192.168.50.2:8989" -HOST1_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f" -HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows" - -# Container path → host path translation -# Add one entry per root folder configured in Sonarr Settings → Media Management → Root Folders -# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder -declare -A HOST1_SONARR_PATH_MAP=( - ["/tv"]="/mnt/user/Tv_Shows" - ["/ext-standup-comedy"]="/mnt/user/stand-up_comedy" - ["/kids tv"]="/mnt/user/Kids_Tv_Shows" - ["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old" -) - -HOST2_SONARR_URL="http://localhost:8989" -HOST2_SONARR_API_KEY="your-host2-sonarr-api-key" -HOST2_SONARR_TV_ROOT="/mnt/user/Anime_Shows" - -declare -A HOST2_SONARR_PATH_MAP=( - # Fill in when HOST2 is back online - # ["/tv"]="/mnt/user/Anime_Shows" -) - -SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion -SONARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this -SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") -SONARR_PROTECTED_PATTERNS=( - # Subtitles - "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" - # Metadata - "*.nfo" "*.tbn" - # Images — cover art, posters, fanart, Emby artwork - "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" - "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" - "banner.*" "thumb.*" "landscape.*" - # Kodi/Emby extras — not tracked by Sonarr API - "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" - "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" - "*-clip.*" "*-other.*" - # Theme songs — stored in show folder, not tracked - "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" -) - -# ── Radarr ──────────────────────────────────────────────────────────────────────────────────── -HOST1_RADARR_URL="http://192.168.50.2:7878" -HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" -HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies" - -# Container path → host path translation -# Add one entry per root folder configured in Radarr Settings → Media Management → Root Folders -# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder -declare -A HOST1_RADARR_PATH_MAP=( - ["/movies"]="/mnt/user/Movies" - ["/kids movies"]="/mnt/user/Kids_Movies" - ["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy" - ["/anime-movies"]="/mnt/user/Anime_Movies-Old" -) - -HOST2_RADARR_URL="http://localhost:7878" -HOST2_RADARR_API_KEY="your-host2-radarr-api-key" -HOST2_RADARR_MOVIES_ROOT="/mnt/user/Anime_Movies" - -declare -A HOST2_RADARR_PATH_MAP=( - # Fill in when HOST2 is back online - # ["/anime-movies"]="/mnt/user/Anime_Movies" -) - -RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion -RADARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this -RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") -RADARR_PROTECTED_PATTERNS=( - # Subtitles - "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" - # Metadata - "*.nfo" "*.tbn" - # Images — cover art, posters, fanart, Emby artwork - "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" - "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" - "banner.*" "thumb.*" "landscape.*" - # Kodi/Emby extras — not tracked by Radarr API - "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" - "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" - "*-clip.*" "*-other.*" - # Theme songs — stored in movie folder, not tracked - "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" -) - -# ━━━ Arr Failed/Stalled Recovery ━━━ -# Auto blocklist + re-search failed imports and stalled downloads. -# Runs every 6 hours — schedule: 0 */6 * * * -# -# Targets four problem types: -# importFailed — downloaded but arr couldn't import -# importPending — downloaded, stuck waiting to import (won't self-resolve) -# error status — serious failure not covered above -# stalled — download stuck with no connections or progress -# -# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first. -# API versions: Sonarr /api/v3/ — Radarr /api/v3/ — Lidarr /api/v1/ -# Lidarr runs on HOST1 only — exits cleanly on HOST2. - -ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this - # gives the arr time to retry on its own before we intervene - # matches cron interval — items are eligible after one missed cycle - -# Per-arr enable/disable toggles — set false to temporarily disable without removing from cron -# Useful if an arr is having issues and you want to skip it for a few runs -HOST1_SONARR_RECOVERY=true # Tv_Shows import recovery -HOST1_RADARR_RECOVERY=true # Movies import recovery -HOST1_LIDARR_RECOVERY=true # Music import recovery — HOST1 only, exits cleanly on HOST2 -HOST2_SONARR_RECOVERY=true # Anime_Shows import recovery -HOST2_RADARR_RECOVERY=true # Anime_Movies import recovery - -# ============================================================================================== -# ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Session-based storage allocator using filesystem symlink indirection. -# ffmpeg resolves the symlink ONCE at session start — existing sessions are never affected. -# -# How it works: -# ramdisk_setup.sh — creates tmpfs and symlink at array start via array_start.sh -# transcode_management.sh — every 3min, runs cleanup then manager in correct order -# transcode_cleanup.sh — removes old inactive files -# transcode_manager.sh — manages symlink direction based on usage thresholds -# -# ⚠️ Docker mount — must use shared propagation: -# --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared -# Standard rprivate bind mounts lock the inode — sessions drift to SSD permanently. - -# ━━━ Transcode Manager ━━━ -# tmpfs mount point — created at array start by ramdisk_setup.sh -# Must exist before Emby starts so the symlink resolves correctly - RAMDISK_PATH="/mnt/ramdisk_transcodes" - -# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront -# Set this to a comfortable limit based on your typical concurrent stream count -# Real-world: 9 streams peaked at ~5.5GB — 8G gives comfortable headroom - RAMDISK_SIZE="8G" - -# Symlink that Emby points at — this path NEVER changes regardless of ramdisk/SSD state -# Emby resolves the symlink once per session at start — symlink flips are transparent -# Must match the container path configured in Emby's Extra Parameters - TRANSCODE_LINK="/mnt/ram-transcode" - -# SSD fallback location — where transcodes land when ramdisk is too full -# Must have enough free space to handle peak session load - TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" - -# Usage thresholds in GB — hysteresis gap between WARN and LOW prevents flip-flop -# RAMDISK_WARN_GB: flip symlink to SSD when ramdisk usage reaches this -# RAMDISK_LOW_GB: flip symlink back to ramdisk when usage drops to this -# Gap (6.8 - 5.5 = 1.3GB) means ramdisk must drop 1.3GB before flipping back -# Without hysteresis a session right at the threshold causes rapid flipping - RAMDISK_WARN_GB=6.8 - RAMDISK_LOW_GB=5.5 - -# Minimum free GB on SSD before allowing a flip to SSD -# Prevents flipping to SSD when it's almost full — that would be worse than a full ramdisk - RAMDISK_SSD_MIN_GB=20 - -# File age thresholds in minutes before cleanup eligibility -# TRANSCODE_MAX_AGE: HLS segment files older than this with no active session = clean up -# TRANSCODE_ORPHAN_AGE: files with no matching session at all = clean up - TRANSCODE_MAX_AGE=20 - TRANSCODE_ORPHAN_AGE=30 - -# Notify if symlink flips this many times in one hour -# Frequent flips indicate the ramdisk is too small or thresholds need adjustment - TRANSCODE_FLIP_WARN=3 - -# Permissions applied to ramdisk and SSD transcode directories - TRANSCODE_OWNER="nobody:users" - TRANSCODE_CHMOD="755" - -# Operating mode — controls symlink direction behaviour -# smart — auto-flips between ramdisk and SSD based on RAMDISK_WARN_GB / RAMDISK_LOW_GB -# hysteresis gap prevents flip-flop — default for production -# ramdisk — always uses ramdisk, never flips to SSD -# warns if RAMDISK_WARN_GB exceeded but holds position -# use during SSD maintenance or when SSD space is low -# ssd — always uses SSD, never flips to ramdisk -# use during ramdisk maintenance or after a ramdisk issue - TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd - -# Daily statistics log — read by weekly_health_digest.sh for transcode summary -# Tracks peak usage, flip count, session ratio, files cleaned per day -# Bounded to TRANSCODE_LOG_RETENTION days — auto-purges old entries on each write - TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db" - TRANSCODE_LOG_RETENTION=90 # days before old entries are purged - -# ━━━ Transcode Server Array ━━━ -# All media servers sharing the ramdisk transcode space. -# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex -# Entries with placeholder API keys are skipped automatically. -# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD. - -TRANSCODE_SERVERS=( - "${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby" - # "${HOST2_EMBY_CONTAINER}|${HOST2_EMBY_URL}|${HOST2_EMBY_API_KEY}|emby" - # "Jellyfin|http://localhost:8097|jellyfin-api-key|jellyfin" - # "Plex|http://localhost:32400|plex-token|plex" -) - - TRANSCODE_CHECK_EMBY=true - -# ============================================================================================== -# ── MONITORS ────────────────────────────────────────────────────────────────────────────────── +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# docker_network_connect.sh — normal run +# docker_network_connect.sh --dry-run — preview without making changes +# docker_network_connect.sh --log — verbose output +# docker_network_connect.sh --status — show current network state and exit # ============================================================================================== -# ━━━ Certificate Monitor ━━━ -# Checks SSL certificate expiry via direct openssl connection — no NPM dependency. -# Checks the actual certificate served by each domain, not what NPM thinks it has. -# CERT_WARN_DAYS = notify this many days before expiry -# CERT_CRIT_DAYS = escalate to critical this many days before expiry -# CERT_TIMEOUT = seconds before giving up on the openssl connection -CERT_MONITOR_DOMAINS=( - "Gmer4Lfe.com" - "Gmer4Lfe.us" -) - CERT_WARN_DAYS=30 # warn when cert expires within this many days - CERT_CRIT_DAYS=7 # critical alert within this many days - CERT_TIMEOUT=10 # seconds per domain check +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# ━━━ Backup Verify ━━━ -# Verifies rsync mirror health by comparing random file checksums between servers. -# Catches silent corruption or incomplete syncs that rsync itself wouldn't detect. -# Leave BACKUP_VERIFY_SHARES empty to use HOST*_DAILY_SYNC_SHARES automatically. -# BACKUP_VERIFY_SAMPLE = number of random files to checksum per share -# BACKUP_VERIFY_MIN_SIZE = skip files smaller than this (small files are rarely corrupted) -BACKUP_VERIFY_SHARES=( - # leave empty to use HOST*_DAILY_SYNC_SHARES automatically -) - BACKUP_VERIFY_SAMPLE=10 # random files to check per share - BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample +source "$SCRIPT_DIR/../load_config.sh" -# ━━━ SMART Health ━━━ -# Monitors drive SMART attributes — discovers all drives automatically via /dev/sd* and /dev/nvme*. -# Reads live SMART data — no persistent writes. -# SMART_IGNORE_DRIVES = drives to skip (boot USB, drives without meaningful SMART data) - SMART_TEMP_WARN=45 # Celsius — warn above this temperature - SMART_TEMP_CRIT=55 # Celsius — critical above this temperature -SMART_IGNORE_DRIVES=( - "sda" # boot USB — SMART not meaningful on flash drives -) - -# ━━━ ZFS Memory Snapshot ━━━ -# Weekly ZFS pool health and memory diagnostic report — informational only, no action taken. -# ZFS_REPORT_ARC_WARN_PCT = warn if ARC is using more than this % of its max -# ZFS_REPORT_FREE_WARN_GB = warn if less than this GB free RAM -# ZFS_REPORT_AVAIL_WARN_GB = warn if less than this GB available on ZFS pool -# ZFS_REPORT_DOCKER_TOP = how many top Docker containers to show by memory usage -# ZFS_REPORT_IGNORE_POOLS = individual disk pools to skip (unRAID array disks as ZFS) - ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log" - ZFS_REPORT_ARC_WARN_PCT=90 - ZFS_REPORT_FREE_WARN_GB=10 - ZFS_REPORT_AVAIL_WARN_GB=20 - ZFS_REPORT_DOCKER_TOP=10 -ZFS_REPORT_IGNORE_POOLS=( - "disk10" - "disk9" - "disk8" - "disk6" - "disk5" -) - -# ━━━ Bandwidth Monitor ━━━ -# Called automatically by rsync.sh after each sync — one bounded write per run. -# Tracks transfer size, duration and profile per sync for weekly summary reporting. -# BANDWIDTH_LOG_RETENTION = days to keep entries before auto-purging old records -# BANDWIDTH_WARN_GB = flag in weekly summary if a single sync exceeded this size - BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db" - BANDWIDTH_LOG_RETENTION=90 # days before old entries are purged - BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report - -# Stats files — written by cleanup and recovery scripts, read by coffee report -# All in DATA_DIR — array always running when these are written - ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats - ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats - -# ━━━ Health Digest ━━━ -# Aggregated system health summary — reads existing state files, no new writes. -# Three profiles control when the digest email is sent: -# always — sends every run regardless of findings -# smart — sends only when DIGEST_SMART_ON_* conditions are found -# weekly — sends once per week on DIGEST_DAY only -# Smart profile triggers — set true to send digest when finding is detected: - DIGEST_PROFILE="weekly" # always | smart | weekly - DIGEST_DAY="Sunday" # day of week for weekly profile - DIGEST_SMART_ON_WATCHDOG=true # send if any watchdog strikes are active - DIGEST_SMART_ON_FAILOVER=true # send if failover state is not NORMAL - DIGEST_SMART_ON_CERT_WARN=true # send if any cert is under CERT_WARN_DAYS - DIGEST_SMART_ON_BANDWIDTH=true # send if any transfer exceeded BANDWIDTH_WARN_GB - -# ━━━ Emby Session Report ━━━ -# Weekly Emby usage statistics via API — no persistent writes, queries fresh each run. -# Shows top content, most active users, session counts over the report period. -# URL and API key pulled from HOST1/HOST2_EMBY_URL and HOST1/HOST2_EMBY_API_KEY -# defined in Host Configuration at the top of this file — no duplication needed. - EMBY_REPORT_DAYS=7 # days to include in the report period - EMBY_REPORT_TOP_N=10 # number of top content items to show +parse_args "$@" # ============================================================================================== -# ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── +# ━━━ Setup ━━━ # ============================================================================================== -# Continuous system health monitoring — last line of defense before a crash. -# Started by array_start.sh — runs until array stops. -# Re-sources Master.conf each cycle — config changes take effect on next cycle. -# Strike system: sustained threshold hits trigger reboot — single spikes ignored. -# Reboot loop protection: shuts down instead if reboot limit hit in rolling window. -# Silent when healthy — logs only when a threshold is triggered. +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi -# ━━━ State Files ━━━ - SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" # /tmp resets on reboot ✅ - SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" # survives reboots - SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db" # reboot loop detection +acquire_lock -# ━━━ Strike and Reboot Loop Settings ━━━ -# Strike system: a check must fail this many consecutive cycles before action is taken -# Single spikes (one bad reading) are ignored — sustained problems trigger reboot - SYS_WATCHDOG_STRIKE_LIMIT=2 # consecutive failures before reboot trigger +if ! command -v docker &>/dev/null; then + error "Docker not found — cannot manage networks" + exit 1 +fi -# How often checks run — 300s = 5 minutes -# At STRIKE_LIMIT=2 and INTERVAL=300: problem must persist 10min before reboot - SYSTEM_WATCHDOG_INTERVAL=300 # seconds between watchdog cycles +# detect_hosts() sets MY_ID and aliases HOST*_NETWORK_CONNECT_* arrays +detect_hosts -# Reboot loop protection — if system keeps rebooting something is seriously wrong -# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS hours → shutdown instead of reboot -# Prevents infinite reboot loops when the underlying problem can't be fixed by rebooting - SYS_WATCHDOG_REBOOT_LIMIT=3 # max reboots before shutdown instead - SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 # rolling window in hours +# Validate unRAID notify script — used for network creation alerts +validate_unraid_cmd "/usr/local/emhttp/plugins/dynamix/scripts/notify" "" "" "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" -# Heartbeat — proof of life logged periodically even when everything is healthy - SYSTEM_WATCHDOG_HEARTBEAT=true # true = log heartbeat / false = completely silent - SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1 # how often to log heartbeat (hours) +# Docker daemon check — network operations are useless if daemon is hung +DOCKER_TIMEOUT=15 +if ! timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then + error "Docker daemon not responding — cannot manage networks" + notify "docker_network_connect failed on $(hostname) — Docker daemon not responding" "Network Connect" "warning" + exit 1 +fi -# ━━━ Thresholds ━━━ -# Set at "about to become unstable" levels — not "things are a bit high" -# These should be high enough that normal operation never triggers them +# Empty array guards +if [[ ${#NETWORK_CONNECT_NETWORKS[@]} -eq 0 ]]; then + warn "NETWORK_CONNECT_NETWORKS is empty for $MY_ID — nothing to do" + warn "Check HOST*_NETWORK_CONNECT_NETWORKS in master_host*.conf" + exit 0 +fi -# rootfs (/) usage percentage — when array is down rsync writes land on rootfs -# fills rapidly and can crash the server — 95% is almost too late, act fast - SYS_WATCHDOG_ROOTFS_PCT=95 - -# /var/log usage percentage — log spam can fill rootfs, indicates something broken - SYS_WATCHDOG_LOG_PCT=95 - -# Free RAM in GB — below this is critically low, OOM or swap imminent -# Your server has 128GB — 4GB free means something is consuming everything - SYS_WATCHDOG_MEM_GB=4 - -# ZFS ARC pinned percentage — ARC not releasing after reclaim = memory stuck -# SYS_WATCHDOG_ARC_RELEASE_PCT = after reclaim attempt, if still above this → trigger - SYS_WATCHDOG_ARC_PINNED_PCT=98 - SYS_WATCHDOG_ARC_RELEASE_PCT=95 - -# Load average multiplier — threshold = MULTIPLIER × CPU core count -# MULTIPLIER=3 on 16-core = load average of 48 before triggering -# Set high — transcoding causes legitimate high load spikes - SYS_WATCHDOG_LOAD_MULTIPLIER=3 - -# Zombie process count — large numbers indicate serious process management failure -# A few zombies are normal — 50 means something is very wrong - SYS_WATCHDOG_ZOMBIE_LIMIT=50 - -# CPU temperature in Celsius — sustained high temp causes throttling or kernel panic -# 95°C is close to tjmax on most CPUs — triggers before thermal shutdown - SYS_WATCHDOG_CPU_TEMP_MAX=95 - -# ━━━ Check Toggles ━━━ -# Disable individual checks without disabling the whole watchdog -# All enabled by default except load — transcoding causes legitimate load spikes - SYS_WATCHDOG_CHECK_ROOTFS=true - SYS_WATCHDOG_CHECK_LOG=true - SYS_WATCHDOG_CHECK_RAM=true - SYS_WATCHDOG_CHECK_ARC=true - SYS_WATCHDOG_CHECK_CPU_TEMP=true - SYS_WATCHDOG_CHECK_LOAD=false # disabled — load spikes during transcoding are normal - SYS_WATCHDOG_CHECK_ZOMBIES=true - SYS_WATCHDOG_CHECK_CONTAINERS=true # checks docker_watchdog persistent skip list - SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true # checks if Docker daemon is responding - -# ━━━ Abort Toggles ━━━ -# Conditions that prevent reboot even when a threshold is hit -# true = abort reboot if this condition is active (conservative — avoid data loss) -# false = reboot anyway (aggressive — a clean reboot beats a hard crash) -# Philosophy: aborting is safer for data, rebooting is safer for stability - SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # rebooting with bad pool risks data loss - SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity is better than crashing mid-check - SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting move is better than crashing mid-move +if [[ ${#NETWORK_CONNECT_CONTAINERS[@]} -eq 0 ]]; then + warn "NETWORK_CONNECT_CONTAINERS is empty for $MY_ID — no containers to connect" + warn "Check HOST*_NETWORK_CONNECT_CONTAINERS in master_host*.conf" + exit 0 +fi # ============================================================================================== -# ──────────────────────── End Of User Variables ─────────────────────────────────────────────── -# ============================================================================================== \ No newline at end of file +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_DOCKER_NET Networks: ${NETWORK_CONNECT_NETWORKS[*]}" + echo "$ICON_CONTAINERS Containers: ${NETWORK_CONNECT_CONTAINERS[*]}" + echo "" + echo "━━━ Network State ━━━" + for network in "${NETWORK_CONNECT_NETWORKS[@]}"; do + [[ -z "$network" ]] && continue + if timeout "$DOCKER_TIMEOUT" docker network inspect "$network" &>/dev/null; then + echo " $ICON_SUCCESS $network — exists" + timeout "$DOCKER_TIMEOUT" docker network inspect "$network" \ + --format ' Subnet: {{range .IPAM.Config}}{{.Subnet}}{{end}}' 2>/dev/null + else + echo " $ICON_ERROR $network — missing (will be created on next run)" + fi + done + echo "" + echo "━━━ Container Connections ━━━" + for container in "${NETWORK_CONNECT_CONTAINERS[@]}"; do + [[ -z "$container" ]] && continue + echo "$ICON_CONTAINERS $container:" + timeout "$DOCKER_TIMEOUT" docker inspect "$container" \ + --format '{{range $k, $v := .NetworkSettings.Networks}} {{$k}}{{"\\n"}}{{end}}' \ + 2>/dev/null || echo " not found" + done + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" + +# ============================================================================================== +# ━━━ Network Ensure + Connect ━━━ +# ============================================================================================== +log "Networks: ${NETWORK_CONNECT_NETWORKS[*]}" +log "Containers: ${NETWORK_CONNECT_CONTAINERS[*]}" + +START=$(date +%s) +NETWORKS_CREATED=() +CONNECTED=() +SKIPPED=() +FAILED=() + +for network in "${NETWORK_CONNECT_NETWORKS[@]}"; do + [[ -z "$network" ]] && continue + + log "Processing network: $network" + + # ── Step 1 — ensure network exists ─────────────────────────────────────────────────────── + if timeout "$DOCKER_TIMEOUT" docker network inspect "$network" &>/dev/null; then + log "$network exists ✅" + else + warn "$ICON_DOCKER_NET $network not found — creating (unRAID update may have wiped networks)" + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would create: $network (bridge, auto subnet)" + else + if timeout "$DOCKER_TIMEOUT" docker network create \ + --driver bridge "$network" >/dev/null 2>&1; then + warn "$ICON_DOCKER_NET $network created" + NETWORKS_CREATED+=("$network") + # Network creation is unexpected — notify so user is aware + notify "Docker network created on $(hostname) — $network (unRAID update likely wiped it)" \ + "Network Connect" "normal" + else + error "$network — failed to create" + FAILED+=("$network:create") + continue + fi + fi + fi + + # ── Step 2 — connect containers to this network ────────────────────────────────────────── + for container in "${NETWORK_CONNECT_CONTAINERS[@]}"; do + [[ -z "$container" ]] && continue + + # Container not found — warn and skip, not an error + if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then + warn "$container not found — skipping (may not be running yet)" + continue + fi + + # Already connected — skip cleanly and silently + if timeout "$DOCKER_TIMEOUT" docker network inspect "$network" \ + --format '{{range .Containers}}{{.Name}} {{end}}' 2>/dev/null \ + | grep -qw "$container"; then + log "$container already on $network — skipping" + SKIPPED+=("$container→$network") + continue + fi + + # Connect + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would connect $container to $network" + continue + fi + + log "Connecting $container to $network..." + if timeout "$DOCKER_TIMEOUT" docker network connect "$network" "$container" 2>/dev/null; then + log "$container connected to $network" + CONNECTED+=("$container→$network") + else + error "Failed to connect $container to $network" + FAILED+=("$container:$network") + fi + done + +done + +END=$(date +%s) + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== +# Always show summary — this runs at array start and output is useful for diagnostics +echo "" +echo "━━━━━ $ICON_SUMMARY NETWORK CONNECT SUMMARY ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +[[ ${#NETWORKS_CREATED[@]} -gt 0 ]] && warn "$ICON_DOCKER_NET Created: ${NETWORKS_CREATED[*]} (networks were missing)" +[[ ${#CONNECTED[@]} -gt 0 ]] && log "Connected: ${CONNECTED[*]}" +[[ ${#SKIPPED[@]} -gt 0 ]] && log "Already connected: ${#SKIPPED[@]} skipped" +[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — no changes made" +elif [[ ${#FAILED[@]} -gt 0 ]]; then + echo "$ICON_ERROR Status: SOME OPERATIONS FAILED" + notify "Docker network connect failed on $(hostname) — ${FAILED[*]}" "Network Connect" "warning" +elif [[ ${#NETWORKS_CREATED[@]} -gt 0 ]]; then + warn "Networks recreated — ${NETWORKS_CREATED[*]} — unRAID update likely wiped them" +else + log "All networks healthy — no action needed" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ ${#FAILED[@]} -gt 0 ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/Docker_Essentials/docker_watchdog.sh b/Docker_Essentials/docker_watchdog.sh index f529a1e..d6e06b3 100644 --- a/Docker_Essentials/docker_watchdog.sh +++ b/Docker_Essentials/docker_watchdog.sh @@ -1,56 +1,93 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Docker Watchdog -------------------------------------------- -# ----------------------------------------------------------------------------------------------- -# Two-tier self-healing container monitoring system — runs continuously as a background process. -# Started by array_start.sh at array start — runs until array stops or SIGTERM received. +# ============================================================================================== +# ================================= Docker Watchdog ============================================ +# ============================================================================================== +# Two-tier self-healing container monitoring system. +# Runs continuously as a background process — started by array_start.sh at array start. +# Shuts down cleanly on SIGTERM/SIGINT when array stops. # -# Tier 1 — Strict monitoring (configured containers only) -# Memory hard limits — immediate restart if exceeded -# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT strikes -# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT strikes -# Required containers — must always be running, strike system with skip list +# ── TIER 1 — STRICT MONITORING ──────────────────────────────────────────────────────────────── +# Applies only to explicitly configured containers (HOST*_WATCHDOG_CONTAINERS etc.) # -# Tier 2 — Global health scan (all running containers) -# Unhealthy status — Docker HEALTHCHECK unhealthy → restart -# OOM killed — kernel killed container → restart + notify -# Crash loop detection — RestartCount climbing → notify, critical above limit -# Dead containers — remove and restart -# Unexpected exits — non-zero exit code → restart +# Memory hard limits — immediate restart if container exceeds configured MB ceiling +# Memory soft threshold — warn at SOFT_MEM_THRESHOLD % of hard limit (no restart) +# CPU thresholds — strike system: warn at SOFT_CPU_THRESHOLD, restart after +# CPU_FAIL_LIMIT consecutive strikes at HARD_CPU_THRESHOLD +# HTTP responsiveness — strike system: restart after RESP_FAIL_LIMIT consecutive failures +# Required containers — must always be running; strike system before restart; +# skip list after WATCHDOG_CONTAINER_RESTART_LIMIT in window; +# auto-clears when container recovers # -# Cross-cutting intelligence: -# Startup grace period — skip restarts while system is still booting -# Dependency ordering — restart database before app -# Restart loop protect — stop restarting after X restarts in X hours → skip list -# Skip list auto-clear — clears when container recovers -# Notification batching — one clean summary per cycle, not one ping per event -# Quiet when healthy — only logs when something needs attention -# Parity awareness — skips restarts during parity check +# ── TIER 2 — GLOBAL HEALTH SCAN ─────────────────────────────────────────────────────────────── +# Scans ALL running containers when WATCHDOG_SCAN_ALL=true. +# Containers in WATCHDOG_SCAN_IGNORE are excluded from Tier 2. # -# Continuous loop: -# Checks run every DOCKER_WATCHDOG_INTERVAL seconds (default 900 = 15min) -# Clean shutdown on SIGTERM/SIGINT — sent by array stop -# Variables scoped per-cycle — no state accumulation between cycles +# Unhealthy status — Docker HEALTHCHECK unhealthy → safe_restart() +# OOM killed — kernel OOM killed → safe_restart() + notify +# OOM state tracked per-session to prevent restart loop +# Crash loop detection — RestartCount climbing → notify; above WATCHDOG_CRASH_LIMIT +# → safe_restart() → skip list if restart limit hit +# Dead containers — safe_restart() via remove + start +# Unexpected exits — non-zero exit code → safe_restart() # -# State files: -# WATCHDOG_STATE_FILE — strike counts (/tmp — resets on reboot) -# SYS_WATCHDOG_FAILED_FILE — persistent skip list (/boot — survives reboots) -# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection +# ── CROSS-CUTTING INTELLIGENCE ──────────────────────────────────────────────────────────────── +# Startup grace period — no restarts for WATCHDOG_STARTUP_GRACE seconds after boot +# Dependency ordering — waits for dependencies before restarting a dependent container +# Restart loop protect — skip list after WATCHDOG_CONTAINER_RESTART_LIMIT in rolling window +# Skip list auto-clear — clears when container is seen running again +# Notification batching — one summary per cycle, not one ping per event +# Parity awareness — skips restart actions during parity check +# Timeout protection — all docker commands wrapped in timeout — daemon hangs cannot +# stall the watchdog and leave containers unmonitored +# Docker daemon check — first check every cycle; hung daemon → strike system → +# restart daemon via /etc/rc.d/rc.docker → verify recovery +# system_watchdog.sh handles escalation if restart fails +# Quiet when healthy — only logs when something needs attention (plus heartbeat) # -# All configuration in Master.conf under Docker Watchdog section. -# Supports --dry-run and --status. -# ----------------------------------------------------------------------------------------------- +# ── STATE FILES ─────────────────────────────────────────────────────────────────────────────── +# WATCHDOG_STATE_FILE — strike counts (/tmp — resets on reboot, correct) +# SYS_WATCHDOG_FAILED_FILE — skip list (/boot — survives reboots, intentional) +# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_WATCHDOG_CONTAINERS — memory hard limits per container +# HOST*_WATCHDOG_CONTAINER_URLS — HTTP health check URLs +# HOST*_WATCHDOG_REQUIRED_CONTAINERS — must always be running +# HOST*_WATCHDOG_SCAN_IGNORE — skip in Tier 2 scan +# HOST*_WATCHDOG_DEPENDENCIES — dependency ordering for restart decisions +# All aliased by detect_hosts() — script uses unprefixed names +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# SOFT_CPU_THRESHOLD / HARD_CPU_THRESHOLD / CPU_FAIL_LIMIT +# SOFT_MEM_THRESHOLD +# RESP_FAIL_LIMIT / CURL_TIMEOUT +# DOCKER_WATCHDOG_INTERVAL +# DOCKER_WATCHDOG_HEARTBEAT / DOCKER_WATCHDOG_HEARTBEAT_HOURS +# WATCHDOG_SCAN_ALL +# WATCHDOG_RESTART_UNHEALTHY / WATCHDOG_RESTART_DEAD / WATCHDOG_RESTART_CRASHED +# WATCHDOG_NOTIFY_OOM / WATCHDOG_NOTIFY_CRASHLOOP +# WATCHDOG_CRASH_LIMIT +# WATCHDOG_STARTUP_GRACE +# WATCHDOG_CONTAINER_RESTART_LIMIT / WATCHDOG_CONTAINER_RESTART_WINDOW +# WATCHDOG_BATCH_NOTIFY +# WATCHDOG_STATE_FILE / SYS_WATCHDOG_FAILED_FILE / WATCHDOG_CONTAINER_RESTART_LOG +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# docker_watchdog.sh — normal start (continuous loop) +# docker_watchdog.sh --dry-run — preview without restarting +# docker_watchdog.sh --status — show config and exit +# docker_watchdog.sh --log — verbose output +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup — runs once at start ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Setup — runs once at start ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" @@ -58,74 +95,70 @@ if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi - success "Running as root" +# Continuous mode — skip gracefully if healthy instance already running acquire_lock "continuous" -# Select correct per-host watchdog lists — done once at startup +# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays detect_hosts -if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - WATCHDOG_REQUIRED_CONTAINERS=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}") - declare -A WATCHDOG_CONTAINER_URLS - for key in "${!HOST1_WATCHDOG_CONTAINER_URLS[@]}"; do - WATCHDOG_CONTAINER_URLS["$key"]="${HOST1_WATCHDOG_CONTAINER_URLS[$key]}" - done -else - WATCHDOG_REQUIRED_CONTAINERS=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}") - declare -A WATCHDOG_CONTAINER_URLS - for key in "${!HOST2_WATCHDOG_CONTAINER_URLS[@]}"; do - WATCHDOG_CONTAINER_URLS["$key"]="${HOST2_WATCHDOG_CONTAINER_URLS[$key]}" - done -fi - -info "Watchdog running as: $LOCAL_SERVER_NAME" -info "Check interval: ${DOCKER_WATCHDOG_INTERVAL}s" - if ! command -v docker >/dev/null 2>&1; then - error "Docker not found" + error "Docker not found — cannot start watchdog" exit 1 fi - success "Docker found" + [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted" +# Validate unRAID-specific commands used by this script +# If rc.docker is missing or changed, daemon restart will fail — better to know now +validate_unraid_cmd "/etc/rc.d/rc.docker" "" "" "Docker rc.d script" || warn "rc.docker not found — daemon restart unavailable if needed" + +validate_unraid_cmd "/usr/local/emhttp/plugins/dynamix/scripts/notify" "" "" "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + # Ensure state files exist touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \ "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +# Timeout for all docker commands — prevents hung daemon from stalling the watchdog +DOCKER_TIMEOUT=15 + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_CONTAINERS Watched: ${!WATCHDOG_CONTAINERS[@]}" - echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]}" - echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL" - echo "$ICON_WATCHDOG Interval: ${DOCKER_WATCHDOG_INTERVAL}s" - echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s" - echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" - echo "$ICON_WATCHDOG Batch notify: $WATCHDOG_BATCH_NOTIFY" - echo "$ICON_WATCHDOG Ignore list: ${WATCHDOG_SCAN_IGNORE[*]:-none}" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_CONTAINERS Watched: ${!WATCHDOG_CONTAINERS[*]:-none}" + echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}" + echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL" + echo "$ICON_WATCHDOG Ignore: ${WATCHDOG_SCAN_IGNORE[*]:-none}" + echo "$ICON_WATCHDOG Interval: ${DOCKER_WATCHDOG_INTERVAL}s" + echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s" + echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" + echo "$ICON_WATCHDOG Batch notify: $WATCHDOG_BATCH_NOTIFY" + echo "$ICON_WATCHDOG Docker timeout: ${DOCKER_TIMEOUT}s" + echo "$ICON_GEAR Dry Run: $DRY_RUN" UPTIME_S=$(awk '{print int($1)}' /proc/uptime) - echo "$ICON_TIME System uptime: $(format_duration $UPTIME_S)" + echo "$ICON_TIME System uptime: $(format_duration $UPTIME_S)" [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]] && \ warn "Within startup grace period — restarts suppressed" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -# ----------------------------------------------------------------------------------------------- -# HELPERS — defined once, used every cycle -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Get strike count for a container from state file get_strikes() { grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0" } +# Set strike count for a container in state file set_strikes() { local container="$1" count="$2" file="$3" if grep -q "^${container}:" "$file" 2>/dev/null; then @@ -135,10 +168,12 @@ set_strikes() { fi } +# Check if container is on the persistent skip list is_skipped() { grep -q "^${1}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null } +# Add container to persistent skip list — manual intervention required to recover add_to_skip_list() { local container="$1" reason="$2" if ! is_skipped "$container"; then @@ -148,11 +183,13 @@ add_to_skip_list() { fi } +# Remove container from skip list — called when container is seen running again remove_from_skip_list() { sed -i "/^${1}$/d" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null - success "$1 removed from skip list — recovered" + warn "$1 recovered — removed from skip list ✅" } +# Log a restart event to the rolling restart history file log_restart() { local container="$1" local now @@ -160,12 +197,14 @@ log_restart() { local cutoff cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S') echo "${container}|${now}" >> "$WATCHDOG_CONTAINER_RESTART_LOG" + # Trim entries older than the rolling window local tmp="${WATCHDOG_CONTAINER_RESTART_LOG}.tmp" awk -F'|' -v cutoff="$cutoff" '$2 >= cutoff' \ "$WATCHDOG_CONTAINER_RESTART_LOG" > "$tmp" && \ mv "$tmp" "$WATCHDOG_CONTAINER_RESTART_LOG" } +# Get number of times a container was restarted within the rolling window get_restart_count() { local container="$1" local cutoff @@ -174,13 +213,15 @@ get_restart_count() { '$1==c && $2>=cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l } +# Check if all dependencies of a container are currently running. +# Returns 0 if all deps running (or no deps), 1 if any dep is down. dependencies_satisfied() { local container="$1" local deps="${WATCHDOG_DEPENDENCIES[$container]:-}" [[ -z "$deps" ]] && return 0 for dep in $deps; do local status - status=$(docker inspect -f '{{.State.Running}}' "$dep" 2>/dev/null) + status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$dep" 2>/dev/null) if [[ "$status" != "true" ]]; then warn "$container — dependency $dep is not running — skipping restart this cycle" return 1 @@ -189,8 +230,18 @@ dependencies_satisfied() { return 0 } +# Safe restart with all guards: +# - Restart loop protection (skip list after limit) +# - Dependency check (don't restart if deps down) +# - Startup grace period (no restarts while booting) +# - Dry run support +# - Timeout protection on docker restart +# +# Returns: 0=restarted, 1=skipped, 2=added to skip list safe_restart() { local container="$1" reason="$2" + + # Restart loop protection — skip list if over limit local restart_count restart_count=$(get_restart_count "$container") if [[ "$restart_count" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then @@ -198,34 +249,42 @@ safe_restart() { "restarted $restart_count times in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" return 2 fi + + # Dependency check dependencies_satisfied "$container" || return 1 + + # Startup grace period local uptime_s uptime_s=$(awk '{print int($1)}' /proc/uptime) if [[ "$uptime_s" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then - warn "$container — within startup grace period — skipping restart" + warn "$container — within startup grace period (${uptime_s}s < ${WATCHDOG_STARTUP_GRACE}s) — skipping" return 1 fi + if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would restart $container ($reason)" return 0 fi - info "Restarting $container ($reason) [restart $((restart_count + 1))/$WATCHDOG_CONTAINER_RESTART_LIMIT in window]..." - if docker restart "$container" >/dev/null 2>&1; then + + log "Restarting $container ($reason) [restart $((restart_count + 1))/$WATCHDOG_CONTAINER_RESTART_LIMIT in window]" + if timeout "$DOCKER_TIMEOUT" docker restart "$container" >/dev/null 2>&1; then success "$ICON_STARTED $container restarted" log_restart "$container" return 0 else - error "Failed to restart $container" + error "Failed to restart $container (timeout or error)" return 1 fi } +# Queue a notification event for batch sending at end of cycle queue_notify() { local message="$1" severity="${2:-warning}" NOTIFY_EVENTS+=("${severity}|${message}") log "Queued: $message" } +# Send all queued notifications — one batched summary or individual per event flush_notify() { [[ ${#NOTIFY_EVENTS[@]} -eq 0 ]] && return if [[ "$WATCHDOG_BATCH_NOTIFY" == "true" ]]; then @@ -251,28 +310,153 @@ flush_notify() { NOTIFY_EVENTS=() } +# Returns 0 if parity check is currently running is_parity_running() { grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null } -# ----------------------------------------------------------------------------------------------- -# CLEAN SHUTDOWN — trap SIGTERM/SIGINT from array stop -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ── DOCKER DAEMON HEALTH CHECK ──────────────────────────────────────────────────────────────── +# ============================================================================================== +# Checks Docker daemon responsiveness at the start of every cycle. +# A hung daemon makes all container operations useless — check first, short-circuit if down. +# +# Strike system: +# Each consecutive failed check adds a strike +# At WATCHDOG_DAEMON_STRIKE_LIMIT → attempt daemon restart via rc.docker +# After restart → wait WATCHDOG_DAEMON_RESTART_WAIT seconds → verify +# If verified → clear strikes, continue cycle ✅ +# If still hung → notify critical, skip cycle → system_watchdog.sh escalates from here +# +# Returns: 0 = daemon healthy | 1 = daemon down, skip this cycle + +WATCHDOG_DAEMON_STRIKE_LIMIT=3 # consecutive failed checks before restart attempt +WATCHDOG_DAEMON_RESTART_WAIT=30 # seconds to wait after restart before verifying +WATCHDOG_DAEMON_STRIKES=0 # persists across cycles — reset when daemon recovers +WATCHDOG_DAEMON_RESTARTED=false # tracks if we already attempted restart this session + +# ============================================================================================== +# ── SYSTEM WATCHDOG COORDINATION ────────────────────────────────────────────────────────────── +# ============================================================================================== +# Reads system_watchdog.sh state file to check if a RAM emergency shutdown is active. +# During RAM emergency: system_watchdog.sh has stopped non-essential containers to free RAM. +# docker_watchdog.sh must not restart them — that would undo the emergency shutdown and +# prevent RAM from recovering, creating an infinite restart/shutdown loop. +# +# Returns: +# 0 = normal — run all checks +# 1 = RAM emergency active — defer container management this cycle + +check_system_watchdog_state() { + # Returns 0 = normal operation | 1 = defer, RAM emergency active + local state_file="$SYS_WATCHDOG_STATE_FILE" + + # No state file = system_watchdog not running or not yet written — assume normal + [[ ! -f "$state_file" ]] && return 0 + + local mem_shutdown + mem_shutdown=$(grep "^mem_shutdown_active=" "$state_file" 2>/dev/null | cut -d= -f2) + + [[ "$mem_shutdown" != "true" ]] && return 0 + + # ── Stale state guard ───────────────────────────────────────────────────────────────────── + # If mem_shutdown_active=true but state file hasn't been updated in > 2 hours, + # system_watchdog.sh may have died — don't be silenced forever by a stale flag. + local state_mtime now age_seconds stale_limit=7200 # 2 hours + state_mtime=$(stat -c %Y "$state_file" 2>/dev/null || echo 0) + now=$(date +%s) + age_seconds=$(( now - state_mtime )) + + if [[ "$age_seconds" -gt "$stale_limit" ]]; then + warn "mem_shutdown_active=true but state file is ${age_seconds}s old — may be stale" + warn "system_watchdog.sh may not be running — resuming normal container management" + warn "If RAM is still low this will be caught on next system_watchdog.sh cycle" + return 0 # Resume normal — don't defer indefinitely on stale state + fi + + return 1 # Defer — RAM emergency confirmed and state is fresh +} + +check_docker_daemon() { + # docker info is more definitive than docker ps for daemon health + if timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then + if [[ "$WATCHDOG_DAEMON_STRIKES" -gt 0 ]]; then + info "$ICON_STARTED Docker daemon recovered — clearing strikes" + queue_notify "Docker daemon recovered on $(hostname)" "normal" + WATCHDOG_DAEMON_STRIKES=0 + WATCHDOG_DAEMON_RESTARTED=false + fi + return 0 + fi + + WATCHDOG_DAEMON_STRIKES=$(( WATCHDOG_DAEMON_STRIKES + 1 )) + warn "$ICON_WATCHDOG Docker daemon not responding (strike $WATCHDOG_DAEMON_STRIKES/$WATCHDOG_DAEMON_STRIKE_LIMIT)" + + if [[ "$WATCHDOG_DAEMON_STRIKES" -lt "$WATCHDOG_DAEMON_STRIKE_LIMIT" ]]; then + warn "Skipping monitoring cycle — waiting for daemon to recover" + return 1 + fi + + if [[ "$WATCHDOG_DAEMON_RESTARTED" == true ]]; then + error "Docker daemon still unresponsive after restart attempt" + error "system_watchdog.sh will handle further escalation" + queue_notify "Docker daemon hung on $(hostname) — restart failed — manual intervention needed" "critical" + flush_notify + return 1 + fi + + error "Docker daemon unresponsive — attempting restart" + notify "Docker daemon hung on $(hostname) — attempting restart" "Docker Watchdog" "warning" + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would restart Docker daemon via /etc/rc.d/rc.docker restart" + return 1 + fi + + # Restart daemon — unRAID uses rc.d scripts, not systemd + WATCHDOG_DAEMON_RESTARTED=true + if /etc/rc.d/rc.docker restart >/dev/null 2>&1; then + info "Docker daemon restart issued — waiting ${WATCHDOG_DAEMON_RESTART_WAIT}s..." + sleep "$WATCHDOG_DAEMON_RESTART_WAIT" + + if timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then + success "Docker daemon restarted successfully ✅" + notify "Docker daemon restarted successfully on $(hostname)" "Docker Watchdog" "normal" + WATCHDOG_DAEMON_STRIKES=0 + WATCHDOG_DAEMON_RESTARTED=false + return 0 + else + error "Docker daemon did not recover after restart" + queue_notify "Docker daemon restart failed on $(hostname) — system_watchdog.sh escalating" "critical" + flush_notify + return 1 + fi + else + error "Failed to issue Docker daemon restart — /etc/rc.d/rc.docker not found or failed" + queue_notify "Docker daemon restart command failed on $(hostname) — manual intervention needed" "critical" + flush_notify + return 1 + fi +} + +# ============================================================================================== +# ── CLEAN SHUTDOWN ──────────────────────────────────────────────────────────────────────────── +# ============================================================================================== WATCHDOG_RUNNING=true cleanup() { echo "" - info "Docker watchdog received shutdown signal — stopping cleanly" + warn "Docker watchdog received shutdown signal — stopping cleanly" WATCHDOG_RUNNING=false exit 0 } trap cleanup SIGTERM SIGINT -# ----------------------------------------------------------------------------------------------- -# ━━━ CONTINUOUS MONITORING LOOP ━━━ -# ----------------------------------------------------------------------------------------------- -info "Docker watchdog started — checking every ${DOCKER_WATCHDOG_INTERVAL}s" +# ============================================================================================== +# ━━━ Continuous Monitoring Loop ━━━ +# ============================================================================================== +info "$ICON_WATCHDOG Docker watchdog started — $MY_ID — checking every ${DOCKER_WATCHDOG_INTERVAL}s" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" CYCLE=0 @@ -281,59 +465,79 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do ((CYCLE++)) CYCLE_START=$(date +%s) - # Re-source Master.conf each cycle — picks up any config changes without restart - source "$SCRIPT_DIR/../Master.conf" + # ── Re-source config each cycle ────────────────────────────────────────────────────────── + # Picks up config changes (new containers, threshold adjustments) without restart. + # detect_hosts() re-aliases all HOST*_WATCHDOG_* arrays after re-source. + source "$SCRIPT_DIR/../load_config.sh" + detect_hosts - # Rebuild per-host lists after re-source in case they changed - if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - WATCHDOG_REQUIRED_CONTAINERS=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}") - for key in "${!HOST1_WATCHDOG_CONTAINER_URLS[@]}"; do - WATCHDOG_CONTAINER_URLS["$key"]="${HOST1_WATCHDOG_CONTAINER_URLS[$key]}" - done - else - WATCHDOG_REQUIRED_CONTAINERS=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}") - for key in "${!HOST2_WATCHDOG_CONTAINER_URLS[@]}"; do - WATCHDOG_CONTAINER_URLS["$key"]="${HOST2_WATCHDOG_CONTAINER_URLS[$key]}" - done - fi - - # Per-cycle variables — cleared each iteration, no accumulation + # ── Per-cycle state — cleared each iteration ────────────────────────────────────────────── NOTIFY_EVENTS=() T1_RESTARTS=0 T1_WARNINGS=0 T2_RESTARTS=0 T2_WARNINGS=0 - # Rebuild ignore map each cycle in case config was updated + # OOM handled set — tracks containers already handled for OOM this cycle + # Prevents restart loop from OOMKilled flag persisting after restart + declare -A OOM_HANDLED + + # Rebuild ignore map each cycle (config may have changed) declare -A IGNORE_MAP - for c in "${WATCHDOG_SCAN_IGNORE[@]}"; do + for c in "${WATCHDOG_SCAN_IGNORE[@]:-}"; do [[ -n "$c" ]] && IGNORE_MAP["$c"]=1 done - # Startup grace check - UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime) - IN_GRACE_PERIOD=false - if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then - IN_GRACE_PERIOD=true + # ── Docker daemon health check — first check every cycle ──────────────────────────────── + # If daemon is hung all container operations will fail — check first, skip cycle if down + if ! check_docker_daemon; then + sleep "$DOCKER_WATCHDOG_INTERVAL" + continue fi - # Parity check — skip restarts during parity to avoid I/O interference + # ── Parity check — skip restarts during parity ─────────────────────────────────────────── if is_parity_running; then log "Parity check in progress — skipping restart actions this cycle" sleep "$DOCKER_WATCHDOG_INTERVAL" continue fi - # ── TIER 1 — Strict Monitoring ────────────────────────────────────────────────────────── + # ── RAM emergency check — system_watchdog.sh managing containers ─────────────────────────── + # If system_watchdog.sh has triggered an emergency RAM shutdown, defer all container + # management this cycle. Docker daemon health checks continue — system still needs + # monitoring even during RAM crisis. Restarts deferred to prevent undoing shutdown. + if ! check_system_watchdog_state; then + MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) + MEM_GB=$(( MEM_KB / 1024 / 1024 )) + warn "RAM emergency active (${MEM_GB}GB free) — system_watchdog.sh managing containers" + warn "Deferring all container restart logic this cycle" + log "Waiting for RAM to recover above ${SYS_WATCHDOG_MEM_RECOVER_GB}GB before resuming" + sleep "$DOCKER_WATCHDOG_INTERVAL" & + wait $! + continue + fi - # Required containers + # ── Startup grace period ────────────────────────────────────────────────────────────────── + UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime) + IN_GRACE_PERIOD=false + [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]] && IN_GRACE_PERIOD=true + + # ========================================================================================== + # ── TIER 1 — Strict Monitoring ──────────────────────────────────────────────────────────── + # ========================================================================================== + + # ── Required containers ─────────────────────────────────────────────────────────────────── + # Must always be running — strike system before restart, skip list after limit if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do [[ -z "$container" ]] && continue - STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + + STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \ + "$container" 2>/dev/null) if is_skipped "$container"; then if [[ "$STATUS" == "true" ]]; then + # Container recovered — remove from skip list and clear its history remove_from_skip_list "$container" set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" sed -i "/^${container}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null @@ -345,6 +549,7 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do fi if [[ "$STATUS" == "true" ]]; then + # Running — clear any strikes set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" else STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE") @@ -360,7 +565,7 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do 0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" ((T1_RESTARTS++)) queue_notify "$container was down and restarted on $(hostname)" "warning" ;; - 2) : ;; + 2) : ;; # Added to skip list — already notified *) queue_notify "$container failed to restart on $(hostname)" "warning" ;; esac fi @@ -368,9 +573,9 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do done fi - # Memory and CPU monitoring + # ── Memory and CPU monitoring ───────────────────────────────────────────────────────────── if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then - STATS=$(docker stats --no-stream \ + STATS=$(timeout "$DOCKER_TIMEOUT" docker stats --no-stream \ --format "{{.Name}}|{{.MemUsage}}|{{.CPUPerc}}" 2>/dev/null) TOTAL_CORES=$(nproc 2>/dev/null || echo 1) @@ -379,6 +584,7 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do CONTAINER_STATS=$(echo "$STATS" | grep "^${container}|" | head -1) [[ -z "$CONTAINER_STATS" ]] && continue + # ── Memory ──────────────────────────────────────────────────────────────────────── MEM_USAGE=$(echo "$CONTAINER_STATS" | cut -d'|' -f2 | awk '{print $1}') MEM_UNIT=$(echo "$MEM_USAGE" | grep -oE '[A-Za-z]+') MEM_VALUE=$(echo "$MEM_USAGE" | grep -oE '[0-9.]+') @@ -390,34 +596,50 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do *) MEM_MB=0 ;; esac + # Soft memory threshold — warn when approaching hard limit + SOFT_MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_LIMIT_MB * $SOFT_MEM_THRESHOLD / 100}") + if [[ "$MEM_MB" -ge "$MEM_LIMIT_MB" ]]; then + # Hard limit exceeded — immediate restart + error "$container — memory ${MEM_MB}MB exceeded hard limit ${MEM_LIMIT_MB}MB" + safe_restart "$container" "memory hard limit exceeded" + ((T1_RESTARTS++)) + queue_notify "$container exceeded memory hard limit on $(hostname) — restarted" "warning" + elif [[ "$MEM_MB" -ge "$SOFT_MEM_MB" ]]; then + # Soft threshold — warn only, no restart + warn "$container — memory ${MEM_MB}MB approaching limit (${SOFT_MEM_THRESHOLD}% of ${MEM_LIMIT_MB}MB)" + ((T1_WARNINGS++)) + fi + + # ── CPU ─────────────────────────────────────────────────────────────────────────── CPU_RAW=$(echo "$CONTAINER_STATS" | cut -d'|' -f3 | tr -d '%') CPU_NORM=$(awk "BEGIN {printf \"%.1f\", $CPU_RAW / $TOTAL_CORES}") CPU_INT=$(printf "%.0f" "$CPU_NORM") - if [[ "$MEM_MB" -ge "$MEM_LIMIT_MB" ]]; then - error "$container — memory exceeded hard limit ${MEM_LIMIT_MB}MB" - safe_restart "$container" "memory hard limit exceeded" - ((T1_RESTARTS++)) - queue_notify "$container exceeded memory limit on $(hostname) — restarted" "warning" - fi - if [[ "$CPU_INT" -ge "$HARD_CPU_THRESHOLD" ]]; then + # Hard CPU threshold — strike system → restart CPU_STRIKES=$(get_strikes "${container}_cpu" "$WATCHDOG_STATE_FILE") CPU_STRIKES=$(( CPU_STRIKES + 1 )) set_strikes "${container}_cpu" "$CPU_STRIKES" "$WATCHDOG_STATE_FILE" + warn "$container — CPU ${CPU_NORM}% (strike $CPU_STRIKES/$CPU_FAIL_LIMIT)" if [[ "$CPU_STRIKES" -ge "$CPU_FAIL_LIMIT" ]]; then - safe_restart "$container" "CPU threshold exceeded" + safe_restart "$container" "CPU hard threshold exceeded ${CPU_NORM}%" set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE" ((T1_RESTARTS++)) queue_notify "$container CPU ${CPU_NORM}% on $(hostname) — restarted" "warning" fi + elif [[ "$CPU_INT" -ge "$SOFT_CPU_THRESHOLD" ]]; then + # Soft CPU threshold — warn only, no restart, clear strikes + warn "$container — CPU ${CPU_NORM}% (above soft threshold ${SOFT_CPU_THRESHOLD}%)" + ((T1_WARNINGS++)) + set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE" else + # Normal — clear CPU strikes set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE" fi done fi - # HTTP responsiveness + # ── HTTP responsiveness ─────────────────────────────────────────────────────────────────── if [[ ${#WATCHDOG_CONTAINER_URLS[@]} -gt 0 ]]; then for container in "${!WATCHDOG_CONTAINER_URLS[@]}"; do URL="${WATCHDOG_CONTAINER_URLS[$container]}" @@ -430,102 +652,133 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do warn "$container — not responding at $URL (strike $HTTP_STRIKES/$RESP_FAIL_LIMIT)" ((T1_WARNINGS++)) if [[ "$HTTP_STRIKES" -ge "$RESP_FAIL_LIMIT" ]]; then - safe_restart "$container" "HTTP unresponsive" - set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE" - ((T1_RESTARTS++)) - queue_notify "$container unresponsive at $URL on $(hostname) — restarted" "warning" + result=0 + safe_restart "$container" "HTTP unresponsive at $URL" || result=$? + if [[ $result -eq 0 ]]; then + set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE" + ((T1_RESTARTS++)) + queue_notify "$container unresponsive at $URL on $(hostname) — restarted" "warning" + fi fi fi done fi - # ── TIER 2 — Global Health Scan ───────────────────────────────────────────────────────── + # ========================================================================================== + # ── TIER 2 — Global Health Scan ─────────────────────────────────────────────────────────── + # ========================================================================================== if [[ "$WATCHDOG_SCAN_ALL" == "true" ]]; then - ALL_CONTAINERS=$(docker ps --format "{{.Names}}" 2>/dev/null) - # Unhealthy containers + ALL_CONTAINERS=$(timeout "$DOCKER_TIMEOUT" docker ps --format "{{.Names}}" 2>/dev/null) + + # ── Unhealthy containers ────────────────────────────────────────────────────────────── if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then - UNHEALTHY=$(docker ps --filter health=unhealthy --format "{{.Names}}" 2>/dev/null) + UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \ + --filter health=unhealthy --format "{{.Names}}" 2>/dev/null) while IFS= read -r container; do [[ -z "$container" ]] && continue [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue is_skipped "$container" && continue - error "$container — unhealthy" + error "$container — Docker HEALTHCHECK unhealthy" ((T2_WARNINGS++)) result=0 safe_restart "$container" "unhealthy health status" || result=$? - [[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \ + if [[ $result -eq 0 ]]; then + ((T2_RESTARTS++)) queue_notify "$container unhealthy on $(hostname) — restarted" "warning" + fi done <<< "$UNHEALTHY" fi - # OOM killed + # ── OOM killed ──────────────────────────────────────────────────────────────────────── + # OOMKilled flag persists after restart — track handled containers per-cycle + # to prevent the same container triggering a restart loop every cycle if [[ "$WATCHDOG_NOTIFY_OOM" == "true" ]]; then while IFS= read -r container; do [[ -z "$container" ]] && continue [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue is_skipped "$container" && continue - OOM=$(docker inspect -f '{{.State.OOMKilled}}' "$container" 2>/dev/null) + [[ -n "${OOM_HANDLED[$container]:-}" ]] && continue # already handled this cycle + OOM=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ + '{{.State.OOMKilled}}' "$container" 2>/dev/null) if [[ "$OOM" == "true" ]]; then - error "$container — OOM killed" + error "$container — OOM killed by kernel" ((T2_WARNINGS++)) + OOM_HANDLED["$container"]=1 result=0 safe_restart "$container" "OOM killed" || result=$? - [[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \ + if [[ $result -eq 0 ]]; then + ((T2_RESTARTS++)) queue_notify "$container OOM killed on $(hostname) — restarted" "warning" - fi - done <<< "$ALL_CONTAINERS" - fi - - # Crash loop detection - if [[ "$WATCHDOG_NOTIFY_CRASHLOOP" == "true" ]]; then - while IFS= read -r container; do - [[ -z "$container" ]] && continue - [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue - is_skipped "$container" && continue - RESTART_COUNT=$(docker inspect -f '{{.RestartCount}}' "$container" 2>/dev/null || echo 0) - PREV_COUNT=$(grep "^${container}_docker:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d: -f2 || echo 0) - set_strikes "${container}_docker" "$RESTART_COUNT" "$WATCHDOG_STATE_FILE" - if [[ "$RESTART_COUNT" -gt "$PREV_COUNT" && "$RESTART_COUNT" -gt 0 ]]; then - ((T2_WARNINGS++)) - if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CRASH_LIMIT" ]]; then - error "$container — crash loop CRITICAL: $RESTART_COUNT restarts" - queue_notify "$container crash loop CRITICAL on $(hostname) — manual intervention needed" "critical" - else - warn "$container — restarted since last check (total: $RESTART_COUNT)" - queue_notify "$container restarted on $(hostname) — count: $RESTART_COUNT" "warning" fi fi done <<< "$ALL_CONTAINERS" fi - # Dead containers + # ── Crash loop detection ────────────────────────────────────────────────────────────── + # Tracks Docker's own RestartCount climbing between cycles. + # Below WATCHDOG_CRASH_LIMIT: notify only — Docker's restart policy is handling it. + # At or above WATCHDOG_CRASH_LIMIT: safe_restart() which will add to skip list + # if WATCHDOG_CONTAINER_RESTART_LIMIT is also hit — ensures eventual quarantine. + if [[ "$WATCHDOG_NOTIFY_CRASHLOOP" == "true" ]]; then + while IFS= read -r container; do + [[ -z "$container" ]] && continue + [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue + is_skipped "$container" && continue + RESTART_COUNT=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ + '{{.RestartCount}}' "$container" 2>/dev/null || echo 0) + PREV_COUNT=$(grep "^${container}_docker:" "$WATCHDOG_STATE_FILE" \ + 2>/dev/null | cut -d: -f2 || echo 0) + set_strikes "${container}_docker" "$RESTART_COUNT" "$WATCHDOG_STATE_FILE" + if [[ "$RESTART_COUNT" -gt "$PREV_COUNT" && "$RESTART_COUNT" -gt 0 ]]; then + ((T2_WARNINGS++)) + if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CRASH_LIMIT" ]]; then + error "$container — crash loop CRITICAL: $RESTART_COUNT restarts" + # Attempt restart via safe_restart — will add to skip list if over limit + result=0 + safe_restart "$container" "crash loop — $RESTART_COUNT restarts" || result=$? + case $result in + 0) ((T2_RESTARTS++)) + queue_notify "$container crash loop on $(hostname) — restarted" "critical" ;; + 2) : ;; # Added to skip list + *) queue_notify "$container crash loop CRITICAL on $(hostname) — manual intervention needed" "critical" ;; + esac + else + warn "$container — restarted since last check (Docker count: $RESTART_COUNT)" + queue_notify "$container restarted on $(hostname) — Docker count: $RESTART_COUNT" "warning" + fi + fi + done <<< "$ALL_CONTAINERS" + fi + + # ── Dead containers ─────────────────────────────────────────────────────────────────── + # Routes through safe_restart() — ensures restart loop protection applies if [[ "$WATCHDOG_RESTART_DEAD" == "true" ]]; then - DEAD=$(docker ps -a --filter status=dead --format "{{.Names}}" 2>/dev/null) + DEAD=$(timeout "$DOCKER_TIMEOUT" docker ps -a \ + --filter status=dead --format "{{.Names}}" 2>/dev/null) while IFS= read -r container; do [[ -z "$container" ]] && continue [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue is_skipped "$container" && continue error "$container — dead" ((T2_WARNINGS++)) - RESTART_COUNT=$(get_restart_count "$container") - if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then - add_to_skip_list "$container" "dead — restarted $RESTART_COUNT times" - elif [[ "$DRY_RUN" == false ]]; then - docker rm "$container" >/dev/null 2>&1 - if docker start "$container" >/dev/null 2>&1; then - success "$container removed from dead state and restarted" - log_restart "$container" - ((T2_RESTARTS++)) - queue_notify "$container was dead on $(hostname) — restarted" "warning" - fi + if [[ "$DRY_RUN" == false ]]; then + timeout "$DOCKER_TIMEOUT" docker rm "$container" >/dev/null 2>&1 + fi + result=0 + safe_restart "$container" "dead container" || result=$? + if [[ $result -eq 0 ]]; then + ((T2_RESTARTS++)) + queue_notify "$container was dead on $(hostname) — removed and restarted" "warning" fi done <<< "$DEAD" fi - # Unexpected exits + # ── Unexpected exits ────────────────────────────────────────────────────────────────── + # Only non-zero exit codes — exit 0 is a clean stop, not a crash + # Skips containers already covered by WATCHDOG_REQUIRED_CONTAINERS (handled in Tier 1) if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then - CRASHED=$(docker ps -a \ + CRASHED=$(timeout "$DOCKER_TIMEOUT" docker ps -a \ --filter status=exited \ --format "{{.Names}}|{{.Status}}" 2>/dev/null | \ grep -v "Exited (0)") @@ -533,30 +786,34 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do [[ -z "$container" ]] && continue [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue is_skipped "$container" && continue - SKIP=false - for req in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do - [[ "$container" == "$req" ]] && SKIP=true && break + # Skip containers already monitored by required containers (Tier 1) + local already_required=false + for req in "${WATCHDOG_REQUIRED_CONTAINERS[@]:-}"; do + [[ "$container" == "$req" ]] && already_required=true && break done - [[ "$SKIP" == true ]] && continue + [[ "$already_required" == true ]] && continue error "$container — $status (unexpected exit)" ((T2_WARNINGS++)) result=0 - safe_restart "$container" "unexpected exit" || result=$? - [[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \ + safe_restart "$container" "unexpected exit: $status" || result=$? + if [[ $result -eq 0 ]]; then + ((T2_RESTARTS++)) queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning" + fi done <<< "$CRASHED" fi - fi - # Send notifications if any events this cycle + fi # WATCHDOG_SCAN_ALL + + # ── Send notifications ──────────────────────────────────────────────────────────────────── flush_notify - # Only log summary if something happened — quiet when all healthy + # ── Cycle summary — quiet when healthy ─────────────────────────────────────────────────── TOTAL_RESTARTS=$(( T1_RESTARTS + T2_RESTARTS )) TOTAL_WARNINGS=$(( T1_WARNINGS + T2_WARNINGS )) + CYCLE_END=$(date +%s) if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then - CYCLE_END=$(date +%s) echo "" echo "━━━ $ICON_WATCHDOG Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings" @@ -565,13 +822,16 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" else log "Cycle $CYCLE — all healthy ($(date '+%H:%M:%S'))" - # Heartbeat — periodic proof of life + + # Heartbeat — periodic proof of life even when everything is healthy if [[ "${DOCKER_WATCHDOG_HEARTBEAT:-true}" == true ]]; then HB_SECONDS=$(( ${DOCKER_WATCHDOG_HEARTBEAT_HOURS:-1} * 3600 )) - UPTIME_SECONDS=$(( CYCLE * DOCKER_WATCHDOG_INTERVAL )) - if [[ "$HB_SECONDS" -gt 0 ]] && (( UPTIME_SECONDS % HB_SECONDS < DOCKER_WATCHDOG_INTERVAL )) && [[ "$UPTIME_SECONDS" -gt 0 ]]; then - HB_UPTIME_HR=$(( UPTIME_SECONDS / 3600 )) - info "♥ docker_watchdog alive — ~${HB_UPTIME_HR}hr uptime ($(date '+%H:%M:%S'))" + UPTIME_APPROX=$(( CYCLE * DOCKER_WATCHDOG_INTERVAL )) + if [[ "$HB_SECONDS" -gt 0 ]] && \ + (( UPTIME_APPROX % HB_SECONDS < DOCKER_WATCHDOG_INTERVAL )) && \ + [[ "$UPTIME_APPROX" -gt 0 ]]; then + HB_UPTIME_HR=$(( UPTIME_APPROX / 3600 )) + info "♥ docker_watchdog alive — $MY_ID — ~${HB_UPTIME_HR}hr uptime ($(date '+%H:%M:%S'))" fi fi fi diff --git a/Docker_Essentials/docker_weekly_restart.sh b/Docker_Essentials/docker_weekly_restart.sh index aa4f881..e77065f 100644 --- a/Docker_Essentials/docker_weekly_restart.sh +++ b/Docker_Essentials/docker_weekly_restart.sh @@ -1,24 +1,64 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Docker Weekly Restart -------------------------------------- -# ----------------------------------------------------------------------------------------------- -# Restarts or starts specified Docker containers with retry logic. -# Containers are configured in Master.conf under WEEKLY_RESTART_CONTAINERS. -# Uses global RETRY_COUNT and SLEEP from Master.conf for retry behaviour. -# Sends notifications on completion or failure via common.sh notify(). -# Supports --dry-run to preview what would be restarted without taking action. -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= Docker Weekly Restart ====================================== +# ============================================================================================== +# Restarts all running containers in HOST*_WEEKLY_RESTART_CONTAINERS. +# Called by weekly_sync_maintenance.sh via WEEKLY_MAINTENANCE_SCRIPTS every Sunday at 2:30am. +# Can also be run manually for ad hoc weekly restarts. +# +# ── CONTEXT ─────────────────────────────────────────────────────────────────────────────────── +# weekly_sync_maintenance.sh stops containers before syncing and restarts them after. +# This script runs AFTER that restart — targeting a different set of less critical services +# that benefit from a weekly restart but don't need to be stopped for the sync itself. +# These containers are typically already running when this script executes. +# +# ── BEHAVIOUR ───────────────────────────────────────────────────────────────────────────────── +# Running containers → docker restart (graceful stop + start) +# Stopped containers → left stopped — was down intentionally, do not bring back up +# Missing containers → logged and skipped — not treated as fatal +# Each action uses RETRY_COUNT + SLEEP from master.conf for retry logic. +# +# The "was running → restart, was stopped → leave stopped" rule is consistent +# across the entire ecosystem — container state is always respected. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# Dependency ordering — containers restart in dependency-safe order using +# HOST*_WATCHDOG_DEPENDENCIES from master_host*.conf. Dependencies restart +# first with CONTAINER_DELAY before their dependents. +# +# Restart verification — after each restart, container state is checked after a short +# settle period. If the container fails to stay running it is marked as failed and +# a notification is sent rather than silently passing. +# +# Timeout protection — all docker commands are wrapped in a 30 second timeout. +# A hung Docker daemon cannot cause this script to hang indefinitely. +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_WEEKLY_RESTART_CONTAINERS — list of containers to restart weekly +# HOST*_WATCHDOG_DEPENDENCIES — dependency ordering for restart sequence +# Set by detect_hosts() alias → WEEKLY_RESTART_CONTAINERS used by this script +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# RETRY_COUNT — retry attempts before giving up on a container +# SLEEP — seconds between retry attempts +# CONTAINER_DELAY — seconds to wait between dependency and dependent restart +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# docker_weekly_restart.sh — normal restart +# docker_weekly_restart.sh --dry-run — preview without restarting +# docker_weekly_restart.sh --log — verbose output +# docker_weekly_restart.sh --status — show config and exit +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" @@ -26,7 +66,6 @@ if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi - success "Running as root" acquire_lock @@ -36,19 +75,28 @@ if ! command -v docker &>/dev/null; then notify "Docker weekly restart failed — Docker not found on $(hostname)" "Docker Weekly Restart" "warning" exit 1 fi - success "Docker found" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +# detect_hosts() sets MY_ID and aliases HOST*_WEEKLY_RESTART_CONTAINERS → WEEKLY_RESTART_CONTAINERS +detect_hosts + +if [[ ${#WEEKLY_RESTART_CONTAINERS[@]} -eq 0 ]]; then + warn "WEEKLY_RESTART_CONTAINERS is empty for $MY_ID — nothing to restart" + warn "Check HOST*_WEEKLY_RESTART_CONTAINERS in master_host*.conf" + exit 0 +fi + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}" echo "$ICON_RETRY Retries: $RETRY_COUNT" echo "$ICON_TIME Sleep: ${SLEEP}s between retries" - echo "$ICON_NOTIFY Notifications: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)" + echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)" echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 @@ -56,19 +104,32 @@ fi [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted" -# ----------------------------------------------------------------------------------------------- -# FUNCTIONS -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ── FUNCTIONS ───────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== -# Attempts a docker command up to RETRY_COUNT times with SLEEP seconds between attempts. -# Returns 0 on success, 1 if all attempts fail. +# Wraps docker commands with a 30 second timeout. +# Prevents a hung Docker daemon from causing the script to hang indefinitely. +DOCKER_TIMEOUT=30 +docker_cmd() { + timeout "$DOCKER_TIMEOUT" "$@" + local exit_code=$? + if [[ "$exit_code" -eq 124 ]]; then + error "Docker command timed out after ${DOCKER_TIMEOUT}s: $*" + return 1 + fi + return "$exit_code" +} + +# Retries a docker command up to RETRY_COUNT times with SLEEP seconds between attempts. +# Uses docker_cmd wrapper for timeout protection on each attempt. retry_docker() { local attempt=1 while [[ "$attempt" -le "$RETRY_COUNT" ]]; do info "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*" - if "$@"; then + if docker_cmd "$@"; then success "Succeeded on attempt $attempt" return 0 else @@ -82,9 +143,77 @@ retry_docker() { return 1 } -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_CONTAINERS Weekly Restart ━━━ -# ----------------------------------------------------------------------------------------------- +# Verifies a container is still running after restart. +# Gives the container a short settle period before checking. +RESTART_VERIFY_WAIT=5 +verify_running() { + local container="$1" + sleep "$RESTART_VERIFY_WAIT" + local state + state=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + if [[ "$state" != "true" ]]; then + error "$container failed to stay running after restart — may have crashed" + return 1 + fi + return 0 +} + +# Builds a dependency-safe restart order from WEEKLY_RESTART_CONTAINERS. +# Containers that are dependencies of others restart first. +build_restart_order() { + ORDERED_RESTART=() + local remaining=("${WEEKLY_RESTART_CONTAINERS[@]}") + local placed=() + + # First pass — add dependency containers that appear in our list + for container in "${remaining[@]}"; do + [[ -z "$container" ]] && continue + local is_dependency=false + for dependent in "${!WATCHDOG_DEPENDENCIES[@]:-}"; do + if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then + is_dependency=true + break + fi + done + if [[ "$is_dependency" == true ]]; then + local already=false + for p in "${placed[@]:-}"; do [[ "$p" == "$container" ]] && already=true && break; done + if [[ "$already" == false ]]; then + ORDERED_RESTART+=("$container") + placed+=("$container") + fi + fi + done + + # Second pass — add remaining containers (dependents and independents) + for container in "${remaining[@]}"; do + [[ -z "$container" ]] && continue + local already=false + for p in "${placed[@]:-}"; do [[ "$p" == "$container" ]] && already=true && break; done + if [[ "$already" == false ]]; then + ORDERED_RESTART+=("$container") + placed+=("$container") + fi + done + + log "Restart order: ${ORDERED_RESTART[*]}" +} + +# Waits CONTAINER_DELAY if this container depends on the last restarted one. +check_dependency_delay() { + local container="$1" + local last="$2" + [[ -z "$last" ]] && return + local deps="${WATCHDOG_DEPENDENCIES[$container]:-}" + if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then + info "Waiting ${CONTAINER_DELAY}s — $container depends on $last..." + sleep "$CONTAINER_DELAY" + fi +} + +# ============================================================================================== +# ━━━ Weekly Restart ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_CONTAINERS Weekly Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}" @@ -94,30 +223,48 @@ echo "" START=$(date +%s) FAILED=() RESTARTED=() -STARTED=() +SKIPPED=() -for container in "${WEEKLY_RESTART_CONTAINERS[@]}"; do +# Build dependency-safe restart order +build_restart_order +echo "$ICON_GEAR Restart order: ${ORDERED_RESTART[*]}" +echo "" + +LAST_RESTARTED="" + +for container in "${ORDERED_RESTART[@]}"; do + [[ -z "$container" ]] && continue echo "━━━ $ICON_CONTAINERS $container ━━━" - if ! docker inspect "$container" &>/dev/null; then - error "$container does not exist — skipping" - FAILED+=("$container") + if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then + warn "$container does not exist — skipping" echo "" continue fi - STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) case "$STATUS" in true) echo "$ICON_RUNNING $container is running — restarting..." + # Wait if this container depends on the last one restarted + check_dependency_delay "$container" "$LAST_RESTARTED" + if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would restart $container" + RESTARTED+=("$container") else if retry_docker docker restart "$container"; then - echo "$ICON_STARTED $container restarted" - RESTARTED+=("$container") + if verify_running "$container"; then + echo "$ICON_STARTED $container restarted and running ✅" + RESTARTED+=("$container") + LAST_RESTARTED="$container" + else + error "$container restarted but crashed immediately" + notify "$container crashed after restart on $(hostname)" "Docker Weekly Restart" "warning" + FAILED+=("$container") + fi else error "Failed to restart $container after $RETRY_COUNT attempts" notify "$container failed to restart on $(hostname)" "Docker Weekly Restart" "warning" @@ -126,20 +273,10 @@ for container in "${WEEKLY_RESTART_CONTAINERS[@]}"; do fi ;; false) - echo "$ICON_NOT_RUNNING $container is stopped — starting..." - - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would start $container" - else - if retry_docker docker start "$container"; then - echo "$ICON_STARTED $container started" - STARTED+=("$container") - else - error "Failed to start $container after $RETRY_COUNT attempts" - notify "$container failed to start on $(hostname)" "Docker Weekly Restart" "warning" - FAILED+=("$container") - fi - fi + # Container was stopped — leave it stopped + # Intentionally stopped containers are not restarted + echo "$ICON_NOT_RUNNING $container is stopped — skipping (respecting stopped state)" + SKIPPED+=("$container") ;; *) error "Unknown status for $container: $STATUS" @@ -152,25 +289,26 @@ done END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "━━━━━ $ICON_SUMMARY WEEKLY RESTART SUMMARY ━━━━━" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}" -[[ ${#STARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Started: ${STARTED[*]}" -[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}" +[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]} (were stopped)" +[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" if [[ "$DRY_RUN" == true ]]; then echo "$ICON_WARN Status: DRY RUN — no changes made" elif [[ ${#FAILED[@]} -eq 0 ]]; then echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE" - notify "Weekly restart complete — ${#RESTARTED[@]} restarted, ${#STARTED[@]} started on $(hostname)" "Docker Weekly Restart" "normal" + notify "Weekly restart complete — ${#RESTARTED[@]} restarted, ${#SKIPPED[@]} skipped (stopped) on $(hostname)" "Docker Weekly Restart" "normal" else echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed" notify "Weekly restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Weekly Restart" "warning" fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" [[ ${#FAILED[@]} -gt 0 ]] && exit 1 exit 0 \ No newline at end of file diff --git a/Docker_Essentials/downloaders_reset.sh b/Docker_Essentials/downloaders_reset.sh index 32469de..b2f24c2 100644 --- a/Docker_Essentials/downloaders_reset.sh +++ b/Docker_Essentials/downloaders_reset.sh @@ -1,40 +1,70 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Downloaders Reset ------------------------------------------ -# ----------------------------------------------------------------------------------------------- -# Daily maintenance reset for all download clients. -# Clears stuck states, purges old history, and prepares downloaders for a clean daily cycle. -# Run before container restarts in daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS. +# ============================================================================================== +# ================================= Downloaders Reset ========================================== +# ============================================================================================== +# Maintenance reset for all download clients on this server. +# Called every 15 minutes by critical_sync_maintenance.sh via CRITICAL_MAINTENANCE_SCRIPTS. +# Can also be run manually for ad hoc cleanup. # -# Downloaders covered: -# slskd — clears stuck/errored searches, dead transfer records, -# purges expired failed imports (albums Lidarr rejected) -# SABnzbd — clears completed and failed history older than retention period, -# removes stalled/paused queue items -# qBittorrent — last chance failsafe delete for torrents older than -# QBIT_FAILSAFE_MIN_DAYS regardless of ratio +# ── DOWNLOADERS COVERED ─────────────────────────────────────────────────────────────────────── +# slskd +# Stuck searches — clears Completed/Errored searches left by Soularr crashes +# prevents 409 Conflict on next Soularr startup +# Dead transfers — removes completed/errored/aborted transfer records per user +# prevents Soularr 404 loop when polling a user whose transfer is gone +# NEVER removes InProgress or Queued transfers +# Failed imports — purges albums Soularr downloaded but Lidarr rejected +# Soularr moves these to failed_imports/ and never cleans them up # -# Safety: -# Always --dry-run first before scheduling -# slskd transfer cleanup skips any user with InProgress or Queued transfers -# qBittorrent only deletes if torrent age exceeds QBIT_FAILSAFE_MIN_DAYS -# SABnzbd only deletes history older than DOWNLOADER_RETENTION_DAYS -# qBittorrent deleteFiles=false — removes from qBit, leaves files for arrs to manage +# SABnzbd +# Completed history — removes completed download records older than DOWNLOADER_RETENTION_DAYS +# Failed history — removes failed download records older than DOWNLOADER_RETENTION_DAYS +# Stalled queue — removes Paused or Stuck queue items no longer progressing +# active downloading items are never touched # -# Configuration in Master.conf under Docker Essentials — Downloaders Reset. -# Supports --dry-run to preview without making changes. -# ----------------------------------------------------------------------------------------------- +# qBittorrent +# Age failsafe — removes torrents older than QBIT_FAILSAFE_MIN_DAYS +# deleteFiles=false — removes from qBit, leaves files for arrs to manage +# optional ratio requirement via QBIT_FAILSAFE_MIN_RATIO +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID and aliases all HOST*_SLSKD_*, HOST*_SABNZBD_*, HOST*_QBIT_* vars. +# If a downloader URL is empty for this host — that section is skipped with a clear message. +# HOST2 currently has no downloaders configured — all sections skip cleanly on HOST2. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# slskd — skips users with InProgress or Queued transfers — never interrupts active downloads +# SABnzbd — age check before deletion — only removes items past retention threshold +# qBittorrent — age + optional ratio check — failsafe only removes old completed torrents +# All sections — skip gracefully if downloader is unreachable, no fatal exit +# acquire_lock "wait" — if previous run still active, waits briefly then exits cleanly +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_SLSKD_URL / HOST*_SLSKD_API_KEY / HOST*_SLSKD_FAILED_IMPORTS_DIR +# HOST*_SABNZBD_URL / HOST*_SABNZBD_API_KEY +# HOST*_QBIT_URL / HOST*_QBIT_USERNAME / HOST*_QBIT_PASSWORD +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# DOWNLOADER_RETENTION_DAYS — days before history entries are purged +# QBIT_FAILSAFE_MIN_DAYS — minimum torrent age before failsafe deletion +# QBIT_FAILSAFE_MIN_RATIO — minimum ratio requirement (0 = age only) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# downloaders_reset.sh — normal reset +# downloaders_reset.sh --dry-run — preview without making changes +# downloaders_reset.sh --log — verbose output +# downloaders_reset.sh --status — show config and exit +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_GEAR Downloaders Reset ━━━" @@ -43,45 +73,54 @@ if [[ "$EUID" -ne 0 ]]; then exit 1 fi -detect_hosts - +# Lock first — wait mode since this runs every 15min and previous may still be finishing acquire_lock "wait" +# detect_hosts() sets MY_ID and aliases all HOST*_SLSKD_*, HOST*_SABNZBD_*, HOST*_QBIT_* vars +detect_hosts + START_TIME=$(date +%s) - -# Select host-specific config -if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - SLSKD_URL="$HOST1_SLSKD_URL" - SLSKD_API_KEY="$HOST1_SLSKD_API_KEY" - SLSKD_FAILED_IMPORTS_DIR="$HOST1_SLSKD_FAILED_IMPORTS_DIR" - SABNZBD_URL="$HOST1_SABNZBD_URL" - SABNZBD_API_KEY="$HOST1_SABNZBD_API_KEY" - QBIT_URL="$HOST1_QBIT_URL" - QBIT_USERNAME="$HOST1_QBIT_USERNAME" - QBIT_PASSWORD="$HOST1_QBIT_PASSWORD" -else - # HOST2 placeholders — fill in when HOST2 is back online - SLSKD_URL="${HOST2_SLSKD_URL:-}" - SLSKD_API_KEY="${HOST2_SLSKD_API_KEY:-}" - SLSKD_FAILED_IMPORTS_DIR="${HOST2_SLSKD_FAILED_IMPORTS_DIR:-}" - SABNZBD_URL="${HOST2_SABNZBD_URL:-}" - SABNZBD_API_KEY="${HOST2_SABNZBD_API_KEY:-}" - QBIT_URL="${HOST2_QBIT_URL:-}" - QBIT_USERNAME="${HOST2_QBIT_USERNAME:-}" - QBIT_PASSWORD="${HOST2_QBIT_PASSWORD:-}" -fi - -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" - CUTOFF=$(( $(date +%s) - (DOWNLOADER_RETENTION_DAYS * 86400) )) TOTAL_PASS=0 TOTAL_FAIL=0 -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_GEAR slskd: ${SLSKD_URL:-not configured}" + echo "$ICON_GEAR SABnzbd: ${SABNZBD_URL:-not configured}" + echo "$ICON_GEAR qBittorrent: ${QBIT_URL:-not configured}" + echo "$ICON_TIME Retention: ${DOWNLOADER_RETENTION_DAYS} days" + echo "$ICON_GEAR qBit age: ${QBIT_FAILSAFE_MIN_DAYS} days" + echo "$ICON_GEAR qBit ratio: ${QBIT_FAILSAFE_MIN_RATIO} (0=age only)" + echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" + +# Log which downloaders are active on this host +if [[ -z "$SLSKD_URL" ]] && [[ -z "$SABNZBD_URL" ]] && [[ -z "$QBIT_URL" ]]; then + warn "No downloaders configured for $MY_ID — nothing to reset" + exit 0 +fi +[[ -n "$SLSKD_URL" ]] && log "slskd active on $MY_ID" +[[ -n "$SABNZBD_URL" ]] && log "SABnzbd active on $MY_ID" +[[ -n "$QBIT_URL" ]] && log "qBittorrent active on $MY_ID" + +# ============================================================================================== # ━━━ slskd — Stuck Searches ━━━ -# Clears searches in Completed/Errored state left by Soularr crashes -# Prevents 409 Conflict error on next Soularr startup -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# Clears searches in Completed/Errored state left by Soularr crashes. +# Prevents 409 Conflict error on next Soularr startup when it tries to +# create a search with the same ID that already exists in a terminal state. + if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then echo "" echo "━━━ 🔍 slskd — Stuck Searches ━━━" @@ -127,12 +166,13 @@ if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then fi fi -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== # ━━━ slskd — Dead Transfer Records ━━━ -# Removes completed/errored/aborted transfer records per user -# Prevents Soularr 404 loop when polling a user whose transfer no longer exists -# NEVER removes transfers that are InProgress or Queued -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# Removes completed/errored/aborted transfer records per user. +# Prevents Soularr 404 loop when polling a user whose transfer no longer exists. +# Safety: NEVER removes transfers that are InProgress or Queued — active downloads protected. + if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then echo "" echo "━━━ 🔍 slskd — Dead Transfer Records ━━━" @@ -152,6 +192,7 @@ if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then SUCCESS=0; SKIPPED=0; FAIL=0 while IFS= read -r USER; do [[ -z "$USER" ]] && continue + # Skip users with any active or queued transfers — never interrupt downloads ACTIVE=$(echo "$TRANSFERS" | grep -o "\"username\":\"$USER\"[^}]*\"state\":\"[^\"]*\"" | \ grep -c "InProgress\|Queued") if [[ "$ACTIVE" -gt 0 ]]; then @@ -182,11 +223,13 @@ if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then fi fi -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== # ━━━ slskd — Purge Expired Failed Imports ━━━ -# Removes albums Soularr downloaded but Lidarr rejected -# Soularr moves these to failed_imports/ and never cleans them up -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# Removes albums Soularr downloaded but Lidarr rejected. +# Soularr moves rejected albums to failed_imports/ and never cleans them up. +# Purges directories older than DOWNLOADER_RETENTION_DAYS to prevent unbounded growth. + if [[ -n "$SLSKD_FAILED_IMPORTS_DIR" ]]; then echo "" echo "━━━ 🔍 slskd — Failed Imports (older than ${DOWNLOADER_RETENTION_DAYS} days) ━━━" @@ -217,10 +260,12 @@ if [[ -n "$SLSKD_FAILED_IMPORTS_DIR" ]]; then fi fi -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== # ━━━ SABnzbd — Clear Completed History ━━━ -# Removes completed download history older than DOWNLOADER_RETENTION_DAYS -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# Removes completed download history older than DOWNLOADER_RETENTION_DAYS. +# Keeps recent history for reference — only purges what's past the retention window. + if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then echo "" echo "━━━ 🔍 SABnzbd — Completed History (older than ${DOWNLOADER_RETENTION_DAYS} days) ━━━" @@ -261,10 +306,12 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then fi fi -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== # ━━━ SABnzbd — Clear Failed History ━━━ -# Removes failed download history older than DOWNLOADER_RETENTION_DAYS -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# Removes failed download history older than DOWNLOADER_RETENTION_DAYS. +# Failed history is kept briefly for diagnosis but purged after the retention window. + if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then echo "" echo "━━━ 🔍 SABnzbd — Failed History (older than ${DOWNLOADER_RETENTION_DAYS} days) ━━━" @@ -305,11 +352,14 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then fi fi -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== # ━━━ SABnzbd — Remove Stalled Queue Items ━━━ -# Removes paused queue items no longer progressing -# Active downloading items are never touched -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# Removes queue items in Paused or Stuck state that are no longer progressing. +# Active downloading items (Downloading, Grabbing) are never touched. +# Paused items may be intentional pauses — but in an automated environment +# a Paused item sitting in the queue indefinitely is effectively stalled. + if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then echo "" echo "━━━ 🔍 SABnzbd — Stalled Queue Items ━━━" @@ -331,15 +381,19 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then [[ -z "$NZO_ID" ]] && continue STATUS=$(echo "$QUEUE" | grep -A10 "$NZO_ID" | \ grep -o '"status":"[^"]*"' | sed 's/"status":"//;s/"//') - [[ "$STATUS" != "Paused" ]] && ((SKIPPED++)) && continue + # Only remove Paused or Stuck items — Downloading/Grabbing are active + if [[ "$STATUS" != "Paused" ]] && [[ "$STATUS" != "Stuck" ]]; then + ((SKIPPED++)) + continue + fi if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would remove stalled item: $NZO_ID" + warn "DRY RUN — would remove stalled item: $NZO_ID ($STATUS)" ((DELETED++)) else curl -sf --max-time 10 \ "$SABNZBD_URL/api?mode=queue&name=delete&value=$NZO_ID&apikey=$SABNZBD_API_KEY" \ >/dev/null - info "$ICON_TRASH Removed stalled: $NZO_ID" + info "$ICON_TRASH Removed stalled ($STATUS): $NZO_ID" ((DELETED++)) fi done <<< "$STALLED_IDS" @@ -349,11 +403,17 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then fi fi -# ----------------------------------------------------------------------------------------------- -# ━━━ qBittorrent — Last Chance Failsafe Cleanup ━━━ -# Deletes torrents older than QBIT_FAILSAFE_MIN_DAYS -# deleteFiles=false — removes from qBit, leaves files for arrs to manage -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ qBittorrent — Age Failsafe Cleanup ━━━ +# ============================================================================================== +# Last-chance cleanup for torrents that have been sitting in qBit past their useful life. +# deleteFiles=false — removes the torrent record from qBit but leaves files on disk. +# Radarr/Sonarr manage actual files independently — this only cleans up the qBit entry. +# +# Safety checks before deletion: +# Age must exceed QBIT_FAILSAFE_MIN_DAYS +# Ratio must meet QBIT_FAILSAFE_MIN_RATIO (0 = age only, no ratio requirement) + if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then echo "" echo "━━━ 🔍 qBittorrent — Failsafe (older than ${QBIT_FAILSAFE_MIN_DAYS} days) ━━━" @@ -366,7 +426,8 @@ if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then grep SID | awk '{print "SID="$NF}') if [[ -z "$QBIT_COOKIE" ]]; then - error "Failed to authenticate with qBittorrent" + error "Failed to authenticate with qBittorrent — check QBIT_USERNAME/PASSWORD" + notify "qBittorrent auth failed on $(hostname) — check credentials in master_host*.conf" "Downloaders Reset" "warning" ((TOTAL_FAIL++)) else TORRENTS=$(curl -sf --max-time 15 \ @@ -385,8 +446,11 @@ if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then [[ -z "$HASH" || -z "$ADDED" ]] && continue AGE_DAYS=$(( (NOW - ADDED) / 86400 )) + + # Age check — must be old enough [[ "$AGE_DAYS" -lt "$QBIT_FAILSAFE_MIN_DAYS" ]] && ((SKIPPED++)) && continue + # Ratio check — if configured if [[ "$QBIT_FAILSAFE_MIN_RATIO" != "0" ]]; then RATIO_INT="${RATIO%.*}" MIN_RATIO_INT="${QBIT_FAILSAFE_MIN_RATIO%.*}" @@ -411,14 +475,15 @@ if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then fi fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY DOWNLOADERS RESET SUMMARY ━━━━━" -echo "$ICON_TIME Duration: $(format_duration $(( $(date +%s) - START_TIME )))" -echo "$ICON_SUCCESS Actions: $TOTAL_PASS" -echo "$ICON_ERROR Failures: $TOTAL_FAIL" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_TIME Duration: $(format_duration $(( $(date +%s) - START_TIME )))" +echo "$ICON_SUCCESS Actions: $TOTAL_PASS" +echo "$ICON_ERROR Failures: $TOTAL_FAIL" echo "" if [[ "$DRY_RUN" == true ]]; then diff --git a/Failover/README-Failover.md b/Failover/README-Failover.md index c9a8483..a3381d0 100644 --- a/Failover/README-Failover.md +++ b/Failover/README-Failover.md @@ -1,551 +1,1165 @@ -# Failover +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# 🔀 FAILOVER +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Mutual container failover between two unRAID servers. When one server goes down, the other automatically starts its containers. When it comes back, everything hands back cleanly with data synced and DNS cutting over at exactly the right moment. +**Mutual automatic failover between two fully independent unRAID servers.** +When one server goes down the other starts its containers, cuts over DNS, and keeps +users online. When it comes back everything hands back in the correct sequence — DDNS +first off, rsync, containers, DDNS last on — so users only hit the returning server +after it is actually ready to serve them. -> This system was built from scratch and refined through a year of production testing before being standardised into this ecosystem. The DDNS sequencing and handback order were the hardest parts to get right — the logic is documented here so it's never lost. +> **Built from scratch. Refined through a year of production testing.** The DDNS +> sequencing and handback order were the hardest parts to get right. Both directions +> are exercised regularly with `failover_test.sh`. The logic is documented here in +> full so it is never lost and never has to be rediscovered. --- -## The Setup +## ━━━ THE PROBLEM THAT BUILT THIS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Two fully independent unRAID servers connected via Tailscale: +Running a self-hosted media and productivity stack means being the operator. When +something goes down — power cut, ISP failure, hardware fault — it is your problem. +Cloud services have entire infrastructure teams and redundant data centers. A home +unRAID server has one power outlet, one ISP, and one point of failure. + +The specific problems that led to building this: + +--- + +### 🔴 A Single Point of Failure for an Entire Household + +HOST1 runs Emby, NginxProxyManager, Authelia, NextCloud, VaultWarden, and every +other service the household uses daily. When HOST1 goes down — even briefly — all of +those services go down with it. Everyone in the household loses access simultaneously. +The one place everything lives is the one thing that can fail. + +The fix: a second server with mirrored critical data and the ability to start any of +HOST1's containers. The second server covers the first automatically. From a user's +perspective, a brief interruption and then everything is back — not hours of downtime. + +--- + +### 🔴 DNS Cutting Over Before the Server Was Ready + +Early attempts at failover started the containers on the covering server and then +updated DNS. The problem: DNS propagation happens in under a minute. Users hit the +new IP before Emby had finished starting, before Authelia had loaded its sessions, +before NPM had loaded its proxy configurations. Emby reconnection attempts failed. +Users who were mid-stream got dropped and couldn't reconnect. The auth wall was up +before the service behind it was. + +The fix: warm standby for the auth stack. NPM, LLDAP, and Authelia run actively on +both servers at all times — not cold-started at failover. When DNS cuts over, auth +is already running and ready. The 30-60 second dead zone after failover disappeared. + +--- + +### 🔴 Split Brain DNS During Handback + +When HOST1 returned after being down, the obvious thing to do was: start HOST1's +containers, then switch DNS back. The problem: between "start containers" and "DNS +switches" there is a window where HOST1's DDNS container and HOST2's DDNS container +are both running, both updating the same domain record with different IPs. Users +during that window get routed randomly between two servers — some to the primary that +has fresh data, some to the covering server that might be mid-cleanup. Authentication +sessions don't transfer between servers. This is split brain and it causes exactly +the intermittent failures that are hardest to diagnose. + +The fix: stop DDNS on the covering server first, before anything else moves. Nothing +starts on HOST1 until HOST2's DDNS is confirmed stopped. DNS is always owned by +exactly one server. There is no window where two DDNS containers are updating the +same record. + +--- + +### 🔴 Rsync Running Into Active Container I/O + +Early versions synced data back to HOST1 while containers were still running on HOST2 +— the logic being "minimise downtime, sync while active." The problem: rsync competing +with active container I/O is slower. Files changing mid-transfer can cause +inconsistency. Database writes during sync risk dirty state on the returned server. +The shorter-downtime approach paradoxically created a higher risk of problems on +handback. + +The fix: stop containers before syncing. The window where containers are down is only +the rsync duration — typically a few minutes. Clean static source at full bandwidth. +Predictable, consistent handback state every time. + +--- + +### 🔴 No Way to Validate the System Before Needing It + +A failover system that has never been tested is not a failover system — it is a hope. +The only honest test is to actually pull the plug and see what happens. But doing that +on a live production server means real users experience real downtime, and if something +is misconfigured the downtime is extended while you debug it. + +The fix: `failover_test.sh`. A controlled simulation that uses an iptables rule to +make the remote server appear unreachable — triggering the full failover sequence +without actually taking anything offline. The test validates every phase: detection, +container start, handback, container stop, DNS return. A safety trap removes the +iptables rule on any exit — crash, error, ctrl-c, or clean completion. The remote is +always reachable after the test regardless of what happened during it. + +--- + +## ━━━ THE SETUP ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ``` -HOST1 — unRAID-Gmer4Lfe (Primary) - Hardware: Threadripper 1950X, 128GB RAM - Owns: Gmer4Lfe.com DDNS - Runs: Full service stack +HOST1 — unRAID-Gmer4Lfe + Hardware: Threadripper 1950X, 128GB RAM, ZFS cache pools + Location: Primary site + DDNS: Gmer4Lfe.com + Role: Primary — full service stack + source of truth for Movies/Shows/Music -HOST2 — unRAID-Jayred365 (Secondary / Buddy server) - Hardware: Intel i5 10th gen — completely different hardware - RAM: 64GB +HOST2 — unRAID-Jayred365 + Hardware: Intel i5 10th gen, 64GB RAM — completely different hardware Location: Remote — 50 miles away - Owns: Gmer4Lfe.us DDNS - Runs: Its own service stack + mirrors HOST1 critical data + DDNS: Gmer4Lfe.us + Role: Secondary — own stack + covers HOST1 + mirrors critical data ``` -**Hardware doesn't need to match.** Everything is accessed through `/mnt/user/` — unRAID's fused share layer. HOST1 has a Threadripper with multiple ZFS pools. HOST2 has a completely different CPU, fewer drives, different layout. The failover containers on HOST2 mount `/mnt/user/Movies` and see the same data because the share names match and the data is mirrored. The hardware underneath is irrelevant. +**Hardware does not need to match.** Everything is accessed through `/mnt/user/` — +unRAID's fused share layer. HOST1 has a Threadripper with ZFS pools. HOST2 has a +completely different CPU, fewer drives, different layout. Failover containers on HOST2 +mount `/mnt/user/Movies` and see mirrored data because the share names match. The +hardware underneath is irrelevant. -**What needs to match between servers:** -- Share names (`/mnt/user/Movies` = `/mnt/user/Movies`) -- Container names for shared failover services -- Docker custom network names (so NPM can reach containers by name, not IP) - -**Split source of truth — each server owns different shares:** - -Both servers run arrs simultaneously — no conflict because they manage completely different shares: +**What must match between servers:** ``` -HOST1 arrs — source of truth for: HOST2 arrs — source of truth for: - Movies (Radarr) Anime_Movies (his Radarr) - Tv_Shows (Sonarr) Anime_Shows (his Sonarr) - Music (Lidarr) - -Each mirrors the other's shares in the opposite direction. -HOST1 mirrors anime FROM HOST2. -HOST2 mirrors movies/shows/music FROM HOST1. +Share names /mnt/user/Movies must exist on both servers (mirrored data) +Container names "Emby" on HOST2 must be the container HOST2 starts for HOST1 +Network names Docker custom networks must have matching names for NPM routing ``` -Scheduling keeps them clean even within the same share window: -``` -HOST2 arrs: midnight → noon managing and downloading anime -HOST1 Tdarr: 12:30 → 23:00 transcoding anime, syncs as source of truth -``` - -**The rule:** never run two instances of the same arr against the same share simultaneously. Different arrs managing different shares is perfectly fine. - -At Tier 4 failover (18hr+ outage) each server's arr copies activate to cover the other's shares — only when the truth holder has genuinely been down long enough. - -**Auth stack — runs warm on both servers simultaneously:** - -NPM, LLDAP, and Authelia run actively on both servers at all times. HOST2 needs them running to serve his users through his domain every day. HOST1 is source of truth — all changes mirror to HOST2 every 15 minutes. Certs, proxy rules, user accounts, Authelia policies — all current on both servers at all times. - -Early testing ran the auth stack cold on failover. Results: 30-60 seconds of broken authentication after DNS cut over. Emby clients hit HOST2 before auth was ready — reconnects failed, streams died. Running warm eliminates this window entirely. - -**What actually starts from stopped — both directions:** - -``` -Tier 1 — immediate: - Emby ← starts from stopped on covering server - DDNS updater ← remote domain updater starts on covering server - - Already running — verified healthy, NOT cold started: - NPM — serving both domains continuously - LLDAP — authenticating all users continuously - Authelia — protecting all services continuously - Certs — mirrored, valid, already loaded on both servers - -Tier 4 — 18hr+ outage: - Arrs_Stack ← starts from stopped on covering server - mirrored Arrs_Stack appdata — starts in known good state - not running in normal operation on covering server -``` - -DNS cuts over in 1 minute. Auth is already ready. Library transcode users reconnect seamlessly through buffer. Live TV and direct play users notice and need to resume — the known, accepted tradeoff. Watch history worst case 15-30min behind — in reality a show or two, movies may be unaffected given their length. - -**Independence — always one rsync stop away:** - -If HOST2 ever wants to fully separate: stop HOST1 pushing. Changes he makes stick permanently. His server becomes fully independent immediately. No script changes, no migration — just stop the rsync job. The ecosystem supports this by design. - -Both servers run `failover.sh` as a background task continuously. Neither server knows what the other is doing — they only know what they can ping from their own network perspective. - --- -## How It Works +### ── Split Source of Truth — No Conflicts ──────────────────────────────────── -Every `FAILOVER_CHECK_INTERVAL` seconds (default: 120s) each server makes two pings: +Both servers run arr instances simultaneously with zero conflict — because they manage +completely different shares: ``` -1. Ping remote server (Tailscale IP) -2. Ping internet (8.8.8.8) +HOST1 owns source of truth for: HOST2 owns source of truth for: + /mnt/user/Movies (Radarr) /mnt/user/Anime_Movies (his Radarr) + /mnt/user/Tv_Shows (Sonarr) /mnt/user/Anime_Shows (his Sonarr) + /mnt/user/Music (Lidarr) + +Each server mirrors the other's shares: + HOST1 ← pulls Anime from HOST2 + HOST2 ← pulls Movies/Shows/Music from HOST1 ``` -The combination of those two results determines the current state and what action to take. That's it. No SSH signaling between servers, no shared state files, no coordination — pure autonomous decision making based on observable facts. +**The rule:** never run two instances of the same arr against the same share +simultaneously. Different arrs managing different shares is fine. Scheduling keeps +them clean even within the same window: + +```bash +# master.conf scheduling notes +# ───────────────────────────────────────────────────────────────────────────── +# HOST2 arrs: midnight → noon managing and downloading anime +# HOST1 Tdarr: 12:30 → 23:00 transcoding anime from HOST2's share +# +# They operate in non-overlapping windows even on the same share. +# HOST1 is processing content that HOST2 already downloaded. +``` --- -## States +### ── Auth Stack — Warm on Both Servers ─────────────────────────────────────── -### NORMAL +NPM, LLDAP, and Authelia run actively on both servers at all times. HOST2 needs them +running to serve his users through his domain every day — this is not a failover-only +configuration. HOST1 is source of truth: all changes (proxy rules, user accounts, +certs, Authelia policies) mirror to HOST2 every 15 minutes via critical sync. ``` -Remote: reachable Internet: reachable +Running warm on both servers always: + NginxProxyManager ← serving both domains continuously + LLDAP ← authenticating all users continuously + Authelia ← protecting all services continuously + Certs ← mirrored, valid, already loaded + +Started from stopped only at failover: + Emby ← media server (Tier 1 — immediate) + Remote DDNS ← DNS updater for the covered domain (Tier 1 — immediate) + Dispatcharr x3 ← Live TV schedulers (Tier 1 — people are watching) + ...and tiered services beyond Tier 1 ``` -Both servers running normally. Each server runs its own containers. DDNS on — pointing DNS at this server's IP. Silent operation. - -### FAILOVER - -``` -Remote: unreachable Internet: reachable -``` - -The remote server is down but this server has internet. Start the remote server's containers locally. The remote's DDNS container is started first — DNS starts pointing at this server immediately. Failover is **additive** — your own containers keep running, remote containers are added on top. - -### NO_INTERNET - -``` -Internet: unreachable (remote state unknown) -``` - -This server has lost internet. Stop own DDNS immediately — no point updating DNS records when you can't reach the outside world and it would send conflicting updates. Do not start remote containers — there's no internet to serve them on. Wait for recovery. - -### DARK - -``` -Remote: unreachable Internet: unreachable -``` - -Both pings fail. Same actions as NO_INTERNET — can't determine if remote is truly down or just unreachable through the same outage affecting your internet. Conservative approach: stop DDNS, wait. +Running warm eliminates the 30-60 second dead zone where auth is coming up after DNS +has already cut over. By the time users hit the covering server, auth is already ready. --- -## DDNS — The Critical Part +## ━━━ HOW IT WORKS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -**This took a year to get right. Do not change the sequencing.** +Both servers run `failover.sh` independently as a continuous background process. They +do not coordinate with each other. There is no SSH signaling between servers, no shared +state file, no election algorithm. Each server makes all of its decisions based entirely +on two pings from its own network perspective: -Each server owns one DDNS container. The script controls when each DDNS runs — the network state never auto-starts DDNS. +```bash +# Every FAILOVER_CHECK_INTERVAL seconds (default 120s): +ping -c 1 -W 3 $REMOTE_TAILSCALE_IP # Is the other server reachable? +ping -c 1 -W 3 $EXTERNAL_IP # Do I have internet? (default: 8.8.8.8) +``` + +The combination of those two answers determines the current state. That is the entire +input to the decision logic. Pure, observable facts from each server's own vantage point. + +--- + +## ━━━ STATES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +### ✅ NORMAL ``` -NORMAL: - HOST1 DDNS (Gmer4Lfe.com) → ON — always on while HOST1 has internet - HOST2 DDNS (Gmer4Lfe.us) → ON — always on while HOST2 has internet +Remote ping: PASS Internet ping: PASS +``` + +Both servers running normally. Each server runs its own containers. Own DDNS ON — +pointing DNS at this server's IP. Silent operation — no output on clean cycles. + +--- + +### 🔴 FAILOVER + +``` +Remote ping: FAIL Internet ping: PASS +``` + +The remote server is unreachable but this server has internet. The remote's containers +need to be started locally — users are hitting DNS records that point to a server that's +down. Failover is **additive** — own containers keep running, remote containers are +added on top. DDNS for the remote domain starts immediately (Tier 1) so DNS cuts over +within TTL window (1 minute). + +--- + +### 🟡 NO_INTERNET + +``` +Internet ping: FAIL (remote state irrelevant) +``` + +This server has lost internet connectivity. Stop own DDNS immediately — updating DNS +records without internet access sends conflicting updates and could route users to an +unreachable server. Do not start remote containers — there is no internet to serve +them on. Wait for recovery. + +--- + +### ⚫ DARK + +``` +Remote ping: FAIL Internet ping: FAIL +``` + +Both pings fail. Cannot determine if the remote is truly down or if the same outage +affecting internet connectivity is making it unreachable. Conservative approach: same +actions as NO_INTERNET — stop own DDNS, wait. Do not start remote containers. + +--- + +## ━━━ DDNS — THE CRITICAL PART ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +> **This took a year to get right. Do not change the sequencing.** + +The DDNS rules are absolute. The script is the sole authority over when any DDNS +container starts. Network state returning is not permission to start DDNS. Only the +completion of the full handback sequence grants that permission. + +--- + +### ── The Rules ──────────────────────────────────────────────────────────────── + +``` +ONE DOMAIN → ONE DDNS ACTIVE → AT ALL TIMES + +Gmer4Lfe.com → HOST1's DDNS normally → HOST2's DDNS during HOST1 outage +Gmer4Lfe.us → HOST2's DDNS normally → HOST1's DDNS during HOST2 outage + +Own DDNS: ON when this server has internet. OFF when internet is lost. +Remote DDNS: ON as Tier 1 failover action. OFF as first handback action. +Auto-start: NEVER — DDNS never starts automatically on internet return. +``` + +--- + +### ── Why Auto-Start Is Forbidden ───────────────────────────────────────────── + +If HOST1 lost internet and its DDNS auto-started when internet returned, there is a +window where both HOST1 and HOST2 are running the same DDNS — each updating the same +domain record with different IPs. DNS TTL is 1 minute. During that window users get +routed randomly between two servers. Authentication sessions do not transfer between +servers. This is split brain. + +Split brain from double-DDNS is exactly the class of failure that produces the most +confusing symptoms — intermittent auth failures, users sometimes getting through and +sometimes not, no clear error state anywhere. The fix is absolute: the script controls +when DDNS starts. Network state coming back is not permission. + +--- + +### ── Full DDNS Lifecycle — HOST1 Outage ───────────────────────────────────── + +``` +Normal operation: + HOST1 DDNS (Gmer4Lfe.com) → ON always on while HOST1 has internet + HOST2 DDNS (Gmer4Lfe.us) → ON always on while HOST2 has internet HOST1 loses internet: - HOST1 DDNS → OFF — immediately stopped - HOST2 DDNS → stays ON (unaffected) - -HOST2 detects HOST1 is down: - HOST1 DDNS (on HOST2) → ON — HOST2 starts it as Tier 1 action - DNS now points at HOST2's IP - -HOST1 returns — handback: - HOST1 DDNS (on HOST2) → OFF — stopped FIRST before anything else - ... rsync runs ... - HOST1 containers start on HOST1 - HOST1 DDNS (on HOST1) → ON — started LAST after containers confirmed up + HOST1 DDNS → OFF stopped immediately — no internet, no updates + HOST2 DDNS → ON unaffected, HOST2 still has internet + +HOST2 detects HOST1 is down (FAILOVER state): + HOST1 DDNS (on HOST2) → ON Tier 1 action — DNS now points at HOST2's IP + Both HOST2 DDNS containers → ON own domain + covering HOST1's domain + +HOST1 returns (handback sequence): + HOST1 DDNS (on HOST2) → OFF FIRST action before anything else + [rsync writeback runs] + [HOST1 containers start] + HOST1 DDNS (on HOST1) → ON LAST action — only after containers confirmed up + +Normal operation restored: + HOST1 DDNS (Gmer4Lfe.com) → ON back on HOST1 + HOST2 DDNS (Gmer4Lfe.us) → ON back to normal ``` -**Why DDNS never auto-starts on internet return:** - -If HOST1 lost internet and its DDNS auto-started when internet returned, you'd have both HOST1 and HOST2 running the same DDNS simultaneously — pointing DNS at two different IPs at the same time. DNS TTL is 1 minute — users would get routed randomly between servers. This is called split brain and it causes exactly the kind of intermittent failures that are hard to diagnose. - -The script is the sole authority over when DDNS starts. Network state coming back is not permission to start DDNS. Only the completion of the full handback sequence is. - -**One domain per server, one DDNS active per domain, always:** - -``` -Gmer4Lfe.com → runs on HOST1 normally, moves to HOST2 during HOST1 outage -Gmer4Lfe.us → runs on HOST2 normally, moves to HOST1 during HOST2 outage -``` +DDNS stops on HOST2 before rsync starts. DDNS starts on HOST1 after containers are +confirmed running. There is a brief window where neither DDNS is updating the record — +this is intentional. DNS TTL caches the last value. During the rsync + container start +window, cached DNS still routes users to HOST2 where the containers are still running. +By the time the cache expires, HOST1's DDNS has started and the record points at HOST1. --- -## Tiered Failover +## ━━━ TIERED FAILOVER ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Not every service needs to start immediately when the other server goes down. Starting the full stack on a secondary server wastes resources for short outages — most are resolved in minutes. - -``` -Tier 1 — Immediate (0 min) - Remote DDNS ← DNS coverage first - Emby ← media server, people are watching - NginxProxyManager ← reverse proxy, everything routes through this - Lldap, Mariadb, Redis ← auth stack, required by everything proxied - Authelia x2 ← authentication - VaultWarden ← passwords, needed immediately - Dispatcharr x3 ← Live TV, people are watching right now - ErsatzTV ← Live TV scheduling - -Tier 2 — After HOST1_TIER2_DELAY minutes (default: 120min) - NextCloud + Postgres ← file access - Immich + PostgreSQL ← photos - Jellyseerr ← media requests - -Tier 3 — After HOST1_TIER3_DELAY minutes (default: 360min) - Organizrv2 ← dashboard - AdGuard-Home ← DNS filtering - UptimeKuma ← monitoring - Gitea ← git server - Collabora-CODE ← document editing - -Tier 4 — After HOST1_TIER4_DELAY minutes (default: 1080min / 18hr) - Full arr stack ← Sonarr, Radarr, Lidarr, Prowlarr etc. - Downloaders ← SABnzbd, Qbittorrent, LidaTube, Pinchflat - 24hr+ outage = full workflow continuity -``` - -**Live TV is Tier 1 because people are watching.** You cannot tell a household mid-game that their live TV will be back in 2 hours. - -**Arrs and downloaders are Tier 4** because they generate significant I/O and have minimal writeback on handback — if HOST1 comes back before 18 hours, the arrs never started on HOST2 and there's nothing to sync back. +Not every service needs to start immediately. Starting the full stack on a secondary +server wastes resources for short outages — most power blips and brief ISP issues +resolve in minutes. The tier system starts only what is needed for the actual outage +duration. --- -## Handback Sequence - -When HOST1 returns after being down, the handback must happen in exactly this order: - -``` -1. Strike confirmation - — FAILOVER_HANDBACK_STRIKES consecutive remote-up checks - — prevents handing back during a brief network blip - — 2 strikes × 120s = 4 minute confirmation window - -2. Pre-flight checks - — Remote array is started - — Remote Docker daemon is responding - — Remote rootfs is not nearly full - — Abort if any check fails — retry next cycle - -3. Stop remote DDNS FIRST - — DNS stops updating before anything moves - — Prevents split brain during the transition window - — This is the most critical ordering step - -4. Stop remote containers - — Clean state before rsync - — No competing writes during transfer - — Containers are only down during the rsync window - — This minimises user disruption - -5. Rsync writeback - — Full bandwidth available — DDNS stopped, containers stopped - — Only critical data synced back: - appdata-Failover/Critical-Data (auth stack) - appdata-Failover/Important-Data (NextCloud + Postgres) - appdata-Failover/Emby (userdata, playstates) - appdata-Failover/Gmer4Lfe (server appdata) - — Media files skipped — already on HOST1, never moved - — Downloads skipped — start fresh is cleaner - -6. Start local containers - — Dependencies respected — databases before apps - — Brief pause to let databases initialise before dependents start - -7. Start local DDNS LAST - — DNS only cuts back after containers are confirmed up - — Users hit HOST1 only after it's actually ready to serve them - -8. Return to NORMAL - — State file reset - — Tier flags cleared - — Next cycle confirms everything is healthy -``` - -**Why containers stop before rsync:** - -Earlier versions synced while containers were still running on the remote. This caused: -- Rsync competing with active container I/O — slower transfers -- Files changing mid-transfer — potential inconsistency -- Database writes during sync — dirty state on handback - -Stopping containers first means rsync gets a clean static source at full bandwidth. The window where containers are down is the rsync duration only — typically minutes. - ---- - -## Mutual Failover — Both Directions - -The same script handles both directions. `detect_hosts()` in `common.sh` determines which server is local and which is remote at runtime, then selects the correct arrays from `Master.conf`. - -``` -HOST2 covers HOST1 (HOST1 goes down): - Uses: FAILOVER_HOST2_RUNS_FOR_HOST1_* arrays - Tiers configured by: HOST1_TIER*_DELAY variables - -HOST1 covers HOST2 (HOST2 goes down): - Uses: FAILOVER_HOST1_RUNS_FOR_HOST2_* arrays - Tiers configured by: HOST2_TIER*_DELAY variables -``` - -Both servers run identical scripts. The configuration in `Master.conf` controls what each server does for the other. - ---- - -## Configuration - -All configuration in `Master.conf` under the `── FAILOVER ──` section. +### ── Tier 1 — Immediate (0 minutes) ───────────────────────────────────────── ```bash -# Core timing -EXTERNAL_IP="8.8.8.8" # internet ping target -FAILOVER_CHECK_INTERVAL=120 # seconds between checks -FAILOVER_HANDBACK_STRIKES=2 # confirmations before handback -FAILOVER_STATE_FILE="/boot/config/failover_state.db" - -# DDNS ownership — one per server, script controlled exclusively -HOST1_DDNS_CONTAINERS=("Gmer4Lfe.com") -HOST2_DDNS_CONTAINERS=("Gmer4Lfe.us") - -# What HOST2 runs for HOST1 (tiered) -FAILOVER_HOST2_RUNS_FOR_HOST1_IMMEDIATE=(...) -FAILOVER_HOST2_RUNS_FOR_HOST1_2HR=(...) -FAILOVER_HOST2_RUNS_FOR_HOST1_6HR=(...) -FAILOVER_HOST2_RUNS_FOR_HOST1_18HR=(...) - -# What HOST1 runs for HOST2 (tiered) -FAILOVER_HOST1_RUNS_FOR_HOST2_IMMEDIATE=(...) -# ... - -# Tier delays — configurable per host, in minutes -HOST1_TIER2_DELAY=120 -HOST1_TIER3_DELAY=360 -HOST1_TIER4_DELAY=1080 -HOST2_TIER2_DELAY=120 -HOST2_TIER3_DELAY=360 -HOST2_TIER4_DELAY=1080 - -# Writeback jobs on handback -FAILOVER_HOST1_WRITEBACK=( - "/mnt/user/appdata-Failover/Critical-Data" - "/mnt/user/appdata-Failover/Important-Data" - "/mnt/user/appdata-Failover/Emby" - "/mnt/user/appdata-Failover/Gmer4Lfe" +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# These start the moment FAILOVER state is entered. +# Vital services + services people are actively using right now. +# +FAILOVER_HOST2_RUNS_FOR_HOST1_TIER1=( + "Gmer4Lfe.com-DDNS" # ALWAYS FIRST — DNS coverage before anything else + "Emby" # media server — people are watching + "NginxProxyManager" # reverse proxy — all external access routes through this + "Lldap-Gmer4Lfe" # user directory — already warm, verify and keep + "Mariadb-Authelia" # auth database — already warm, verify and keep + "Redis-Authelia" # auth session cache — already warm, verify and keep + "Authelia" # SSO — already warm, serving users already + "VaultWarden" # passwords — needed immediately, people lock themselves out + "Dispatcharr" # Live TV scheduler — people are watching right now + "Dispatcharr-Basic" # secondary scheduler + "Dispatcharr-Iptv-Users" # tertiary scheduler + "ErsatzTV-Emby" # channel schedule builder ) +# +# Live TV is Tier 1 because you cannot tell a household mid-game that their +# live TV will be back in 2 hours. The disruption is unacceptable. +# Passwords are Tier 1 because people lock themselves out of everything else +# if VaultWarden is unavailable — it becomes the blocking failure. ``` --- -## Initial Setup Requirements +### ── Tier 2 — After HOST*_TIER2_DELAY (default: 4 hours) ──────────────────── -Before `failover.sh` can run on both servers: - -**1. Tailscale connected on both servers** ```bash -# Verify on HOST1 -tailscale ip -4 unRAID-Jayred365 # should return HOST2's Tailscale IP - -# Verify on HOST2 -tailscale ip -4 unRAID-Gmer4Lfe # should return HOST1's Tailscale IP +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Shared productivity services. Not immediately needed for media access but +# actively used throughout the day. 4 hours is long enough to resolve most +# ISP and power issues without starting services unnecessarily. +# +FAILOVER_HOST2_RUNS_FOR_HOST1_TIER2=( + "NextCloud" # file sync and sharing + "Postgres-NextCloud" # NextCloud's database — must start before NextCloud + "Immich-Gmer4Lfe" # photo library + "PostgreSQL-Immich" # Immich's database — same dependency + "Jellyseerr" # media request management +) + HOST1_TIER2_DELAY=240 # minutes — 4 hours ``` -**2. SSH keys configured** +--- + +### ── Tier 3 — After HOST*_TIER3_DELAY (default: 12 hours) ────────────────── -HOST1 must be able to SSH to HOST2 without a password, and vice versa: ```bash -# From HOST1 +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Secondary services — useful but not critical for daily operation. +# 12 hours covers most genuine outage scenarios while avoiding unnecessary +# resource usage on HOST2 for short-to-medium disruptions. +# +FAILOVER_HOST2_RUNS_FOR_HOST1_TIER3=( + "Organizrv2-Gmer4Lfe" # dashboard + "AdGuard-Home" # DNS filtering + "UptimeKuma" # uptime monitoring + "Gitea" # git server + "Collabora-CODE" # document editing for NextCloud +) + HOST1_TIER3_DELAY=720 # minutes — 12 hours +``` + +--- + +### ── Tier 4 — After HOST*_TIER4_DELAY (default: 24 hours) ────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Full workflow continuity — arrs and downloaders. +# These generate significant I/O and have writeback requirements on handback. +# 24 hours is the threshold where it becomes worthwhile to start them. +# +# Arrs_Stack only starts if HOST1 has genuinely been down for 24+ hours. +# Below this threshold: arrs never started on HOST2 → nothing to sync back. +# Above this threshold: meaningful downloads accumulated → full Tier 4 writeback. +# +FAILOVER_HOST2_RUNS_FOR_HOST1_TIER4=( + "Sonarr-Gmer4Lfe" # TV show management + "Radarr-Gmer4Lfe" # movie management + "Lidarr-Gmer4Lfe" # music management + "Prowlarr-Gmer4Lfe" # indexer management + "SABnzbd-Gmer4Lfe" # usenet downloader + "qBittorrent-Gmer4Lfe" # torrent downloader + "LidaTube" # YouTube music + "Pinchflat" # YouTube video +) + HOST1_TIER4_DELAY=1440 # minutes — 24 hours +``` + +--- + +## ━━━ HANDBACK SEQUENCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +When the remote server returns after a FAILOVER event, handback must happen in exactly +this order. Every step has a reason. Do not reorder. + +--- + +### ── Step 1 — Strike Confirmation ──────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Network blips happen. A server can be reachable for one ping cycle and then +# drop again. If handback triggered on the first successful ping, a brief +# network recovery during an ongoing outage would cause a failed handback — +# containers stop on the covering server, rsync starts, remote goes down again +# mid-transfer. That is worse than not handing back at all. +# +# Strike confirmation requires N consecutive successful remote pings before +# handback begins. With FAILOVER_CHECK_INTERVAL=120s and HANDBACK_STRIKES=2: +# → remote must be up continuously for 4 minutes before handback starts +# → eliminates false triggers from brief network recovery +# + FAILOVER_HANDBACK_STRIKES=2 # consecutive remote-up checks required + FAILOVER_CHECK_INTERVAL=120 # seconds between checks +# +# 2 strikes × 120s = 4 minute confirmation window before handback begins +``` + +--- + +### ── Step 2 — Pre-flight Checks ────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Before starting the handback sequence, verify the remote is actually ready. +# Reachable on Tailscale does not mean the array is started and Docker is +# running — the server might still be booting. +# +# Pre-flight checks (any failure → abort this cycle, retry next): +# ✓ Version parity — both servers on compatible unRAID versions +# ✓ Remote array — /mnt/user is mounted on remote +# ✓ Remote Docker — daemon is responding within DOCKER_TIMEOUT +# ✓ Remote rootfs — not nearly full (headroom for rsync temp files) +# +# If any pre-flight fails: abort handback, reset strike counter, retry. +# Better to wait another cycle than to start a handback that will fail midway. +``` + +--- + +### ── Step 3 — Stop Remote DDNS FIRST ───────────────────────────────────────── + +``` +This is the most critical ordering step in the entire sequence. + +Remote DDNS stops before any containers stop, before rsync starts, +before HOST1 containers start, before HOST1 DDNS starts. + +During the gap between remote DDNS stopping and HOST1 DDNS starting: + DNS TTL caches the last value → users still routed to HOST2 + HOST2 containers still running → users are still served + No split brain window exists because one DDNS stopped before + the other started. The transition is clean. +``` + +--- + +### ── Step 4 — Stop Remote Containers ──────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Containers stop on the covering server before rsync begins. +# Clean static source. No competing I/O. No database writes during transfer. +# Full bandwidth available to rsync. +# +# The user impact window (containers down) is the rsync duration only — typically +# minutes for critical data. Media files are never synced back — they were never +# moved. Downloads are never synced back — start fresh is cleaner. +# +# Containers stop in the correct dependency order: +# Dependents before dependencies (Authelia before MariaDB) +# Same WATCHDOG_DEPENDENCIES ordering used by docker_watchdog.sh +``` + +--- + +### ── Step 5 — Tiered Rsync Writeback ───────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# What gets written back depends on how long the outage lasted. +# Short outages skip writeback entirely — the primary's last-known-good state +# is more reliable than a few minutes of activity on the covering server. +# +# Under 60min → skip all writeback (primary state is cleaner) +# Under 4hr → skip Tier 2+ writeback +# Under 12hr → skip Tier 3+ writeback +# Under 24hr → skip Tier 4 writeback (arrs never started, nothing to sync) +# Over 24hr → full tiered writeback (meaningful delta accumulated) +# +# Real-world outage profile: +# Power blip 2-10min → no writeback, clean restart (most common) +# ISP issue 10-60min → no writeback, clean restart +# Real outage 3-6hr → Tier 1+2 writeback, skip 3+4 +# Extended 24hr+ → full tiered writeback + +FAILOVER_HOST1_WRITEBACK_TIER1=( + "/mnt/user/appdata-Failover/Critical-Data" # auth stack — Authelia + NPM + certs + "/mnt/user/Media_Server/Emby" # Emby userdata — watch history, playstates +) + +FAILOVER_HOST1_WRITEBACK_TIER2=( + "/mnt/user/appdata-Failover/Important-Data" # NextCloud + Postgres + Immich +) + +FAILOVER_HOST1_WRITEBACK_TIER3=( + "/mnt/user/appdata-Failover/Gmer4Lfe" # secondary appdata accumulated changes +) + +# Tier 4 writeback uses HOST2_DAILY_SYNC_SHARES automatically — same list +# daily_sync_maintenance.sh uses, in the opposite direction. +# HOST2 was running HOST1's arrs against mirrored media — that content needs +# to sync back before HOST1's own arrs restart. +# No duplicate list needed — uses the same config, reversed direction. + +# What never gets written back: +# Own media shares (Movies, Tv_Shows, Music) — already on HOST1, never moved +# Downloads — start fresh is cleaner than partial download state +``` + +--- + +### ── Step 6 — Start Remote Containers ─────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Containers start on HOST1 in dependency-safe order. +# Same WATCHDOG_DEPENDENCIES ordering — databases before applications. +# CONTAINER_DELAY seconds between dependency start and dependent start. +# Each container verified running after a settle period before the next starts. +# A container that starts and immediately crashes is caught and reported. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Step 7 — Start Remote DDNS LAST ───────────────────────────────────────── + +``` +DNS cuts back to HOST1 ONLY after all containers are confirmed running. + +Users hit HOST1 only after HOST1 is actually ready to serve them. + +The handback sequence completes. State file resets to NORMAL. +Tier flags cleared. Next cycle confirms everything is healthy. +``` + +--- + +## ━━━ MUTUAL FAILOVER — BOTH DIRECTIONS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +The same `failover.sh` script handles both directions without any code changes. +`detect_hosts()` in `common.sh` determines which server is local and which is remote +at runtime, then selects the correct container arrays and tier delays from `master.conf`. + +```bash +# master.conf — symmetric configuration for both directions +# ───────────────────────────────────────────────────────────────────────────── + +# HOST2 covers HOST1 (HOST1 goes down): +FAILOVER_HOST2_RUNS_FOR_HOST1_TIER1=(...) # what HOST2 starts for HOST1 +FAILOVER_HOST2_RUNS_FOR_HOST1_TIER2=(...) +FAILOVER_HOST2_RUNS_FOR_HOST1_TIER3=(...) +FAILOVER_HOST2_RUNS_FOR_HOST1_TIER4=(...) +HOST1_TIER2_DELAY=240 # delays for HOST1 outage — from HOST1's perspective +HOST1_TIER3_DELAY=720 +HOST1_TIER4_DELAY=1440 + +# HOST1 covers HOST2 (HOST2 goes down): +FAILOVER_HOST1_RUNS_FOR_HOST2_TIER1=(...) # what HOST1 starts for HOST2 +FAILOVER_HOST1_RUNS_FOR_HOST2_TIER2=(...) +HOST2_TIER2_DELAY=240 # delays for HOST2 outage — from HOST2's perspective +HOST2_TIER3_DELAY=720 +HOST2_TIER4_DELAY=1440 + +# Writeback — what each server syncs back on handback +FAILOVER_HOST1_WRITEBACK_TIER1=(...) # what HOST2 syncs back to HOST1 +FAILOVER_HOST2_WRITEBACK_TIER1=(...) # what HOST1 syncs back to HOST2 +``` + +Both servers run identical scripts with identical configuration. The configuration +controls what each server does. The script reads `MY_ID` from `detect_hosts()` and +selects the correct arrays. No hostname comparisons anywhere in the script. + +--- + +## ━━━ INDEPENDENCE — ALWAYS ONE RSYNC STOP AWAY ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +HOST2 is designed to be fully independent if needed. If HOST2 ever wants to fully +separate from HOST1: stop HOST1 pushing data. Any changes HOST2 makes to his own +data stick permanently. His server becomes fully independent immediately — no script +changes, no migration, no data movement required. + +The ecosystem supports this by design. Everything HOST2 runs is self-contained. +The failover and rsync scripts are configuration-driven — stopping the rsync job +is the entire separation process. + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔀 failover.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +The main state machine. Runs continuously as a background process on both servers, +started at array start by `array_start.sh`. Makes all decisions autonomously from +two pings. Never requires human intervention during normal failover and handback. + +```bash +# Started automatically at array start via array_start.sh +# Runs continuously until array stops (SIGTERM → clean shutdown) +# Interval: FAILOVER_CHECK_INTERVAL=120 (2 minutes) +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Normal operation — started by array_start.sh, runs continuously. +# You do not need to run this manually under normal circumstances. +# ───────────────────────────────────────────────────────────────────────────── +failover.sh + +# ───────────────────────────────────────────────────────────────────────────── +# Status — show current state at a glance: +# • Current state (NORMAL / FAILOVER / NO_INTERNET / DARK) +# • Which tier containers are active +# • Outage duration if in FAILOVER +# • Handback strike count if remote has returned +# • DDNS containers currently running on this server +# ───────────────────────────────────────────────────────────────────────────── +failover.sh --status + +# ───────────────────────────────────────────────────────────────────────────── +# Dry run — walk through one full cycle without starting or stopping anything. +# Shows what the script would do based on current network state. +# Useful for verifying configuration before relying on it. +# ───────────────────────────────────────────────────────────────────────────── +failover.sh --dry-run + +# ───────────────────────────────────────────────────────────────────────────── +# Verbose — full detail on every decision made in each cycle. +# ───────────────────────────────────────────────────────────────────────────── +failover.sh --log +``` + +> **To stop:** Click Abort in the User Scripts plugin. Do NOT kill the process directly +> — the state file may be left inconsistent. If the state gets stuck, use +> `failover_state_reset.sh` to recover. + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🧪 failover_test.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Controlled simulation of the full failover lifecycle. Validates the entire sequence +without waiting for a real outage. Contains no failover logic itself — exercises +the real `failover.sh` through an iptables block. Any change to `failover.sh` +is automatically reflected in the test. + +> **⚠️ This starts and stops real containers on both servers. Users will experience +> a brief service interruption. Run during a maintenance window or quiet period. +> Use `--dry-run` first to walk through all phases without making any changes.** + +--- + +### ── Test Phases ───────────────────────────────────────────────────────────── + +``` +Phase 1 — Pre-flight + Verify both servers reachable via Tailscale + Verify Docker daemons responding on both servers + Verify version parity between servers + Verify failover.sh exists and is running on remote + Verify current state is NORMAL on both servers + → Any failure: abort test before touching anything + +Phase 2 — Block Remote + Add iptables rule dropping all traffic to remote Tailscale IP + From this server's perspective: remote is now unreachable + failover.sh on this server will see the remote as down on next check + +Phase 3 — Failover Detection + Wait FAILOVER_TEST_BLOCK_WAIT seconds for failover.sh to detect outage + Verify this server entered FAILOVER state + Verify Tier 1 containers started on this server + +Phase 4 — Container Verification + Check each Tier 1 container is actually running + Report any that failed to start + +Phase 5 — Restore Remote + Remove iptables rule — remote becomes reachable again + (Safety trap also removes rule on any exit — crash, ctrl-c, error) + +Phase 6 — Handback Wait + Wait FAILOVER_TEST_HANDBACK_WAIT seconds for failover.sh to: + Confirm remote up (strike confirmation window) + Complete pre-flight checks + Stop remote DDNS + Stop remote containers + Run rsync writeback (if applicable) + Start local containers + Start local DDNS + Return to NORMAL + +Phase 7 — Handback Verification + Verify state returned to NORMAL + Verify Tier 1 containers stopped on this server (handed back) + Verify DDNS is correct on both servers + +Phase 8 — Report + Full pass/fail per phase with timing + Clear indication of what failed if any phase did not pass +``` + +--- + +### ── Safeguards ────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# IPTABLES SAFETY TRAP — the most important safeguard. +# The iptables rule that blocks remote connectivity is added in Phase 2. +# It is removed via a trap on ANY exit condition: +# Normal completion → rule removed +# Script crashes → rule removed +# Error in any phase → rule removed +# ctrl-c → rule removed +# +# Remote connectivity is always restored regardless of test outcome. +# You cannot accidentally leave the remote permanently blocked. +# +# FAILOVER_ENABLED gate — aborts if FAILOVER_ENABLED=false in master.conf. +# Running a failover test when failover is disabled would be confusing and +# potentially destructive. The gate prevents this. +# +# Version parity check — pre-flight verifies both servers are on compatible +# unRAID versions before any iptables rules are added. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Timing Configuration ──────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# These must be set correctly for the test to pass. +# Too short: test fails because failover.sh hasn't had time to complete its work. +# Too long: test runs unnecessarily long. +# + FAILOVER_TEST_BLOCK_WAIT=150 # must be > FAILOVER_CHECK_INTERVAL + buffer + # failover.sh checks every 120s — 150s gives + # one full cycle plus 30s margin + + FAILOVER_TEST_HANDBACK_WAIT=360 # must cover: strike confirmation window + # + pre-flight time + # + rsync duration (critical data only) + # + container start time + # 2 strikes × 120s + ~2min rsync + ~1min start + # = ~6 minutes = 360 seconds +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Dry run — ALWAYS run this first. +# Walks through all 8 phases with full output but no iptables changes and +# no container starts/stops. Verifies configuration and timing before +# committing to a live test. +# ───────────────────────────────────────────────────────────────────────────── +failover_test.sh --dry-run + +# ───────────────────────────────────────────────────────────────────────────── +# Live test — run during maintenance window. +# Full sequence with real iptables rules and real container lifecycle. +# Users will experience a brief service interruption — schedule accordingly. +# ───────────────────────────────────────────────────────────────────────────── +failover_test.sh + +# ───────────────────────────────────────────────────────────────────────────── +# Status — show current failover state and test configuration without running. +# ───────────────────────────────────────────────────────────────────────────── +failover_test.sh --status + +# Verbose — full detail on every check in every phase +failover_test.sh --log +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔧 failover_state_reset.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Manual recovery tool. Resets the state file to NORMAL and clears all tier flags. +Use when the state file is stuck in a non-NORMAL state after testing, a failed +handback, or killing `failover.sh` mid-cycle. + +> **Does NOT start or stop containers — state file only.** After reset, `failover.sh` +> will resume from NORMAL on its next cycle. Verify that containers are actually in +> the right state before resetting — the state file should reflect reality. + +--- + +### ── When to Use This ──────────────────────────────────────────────────────── + +``` +After failover_test.sh didn't complete cleanly + → state file left in FAILOVER but containers are actually back to normal + +After a failed handback + → state shows FAILOVER but remote is back up and containers are split + +After killing failover.sh directly (not via User Scripts Abort) + → state is unknown, cycle was interrupted mid-operation + +After a dev/debug session + → state left in a non-NORMAL state from testing +``` + +--- + +### ── Safety Checks ─────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# VERIFY BEFORE RESETTING: +# ✓ Right containers running on the right server +# ✓ DDNS pointing at the correct server (check with nslookup Gmer4Lfe.com) +# ✓ No actual failover in progress (remote is genuinely up) +# ✓ Both servers can see each other (tailscale ping) +# +# Resetting during an actual failover causes failover.sh to think everything +# is normal and stop covering the remote — services go offline until the next +# detection cycle catches it again. +# +# The script warns if failover.sh is currently running when you attempt reset. +# It does not block you — but heed the warning. If failover.sh is running and +# you reset mid-cycle, the next cycle will start from a clean NORMAL state +# regardless of what was in progress. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Status — show current state file contents without changing anything. +# Also shows whether failover.sh is currently running. +# Use this first to understand the situation before resetting. +# ───────────────────────────────────────────────────────────────────────────── +failover_state_reset.sh --status + +# ───────────────────────────────────────────────────────────────────────────── +# Interactive reset — prompts for "YES" before proceeding. +# Default mode — use when running manually at a terminal. +# ───────────────────────────────────────────────────────────────────────────── +failover_state_reset.sh + +# ───────────────────────────────────────────────────────────────────────────── +# Dry run — show current state and what would be written, without writing. +# ───────────────────────────────────────────────────────────────────────────── +failover_state_reset.sh --dry-run + +# ───────────────────────────────────────────────────────────────────────────── +# Force — non-interactive, no confirmation prompt. +# For use in scripts or when running without a terminal. +# ───────────────────────────────────────────────────────────────────────────── +failover_state_reset.sh --force +``` + +--- + +## ━━━ STATE FILE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +```bash +# /boot/config/failover_state.db +# ───────────────────────────────────────────────────────────────────────────── +# Lives on /boot/ — survives reboots intentionally. +# If a server was in FAILOVER state when it rebooted, it should resume +# FAILOVER on restart — not assume everything is NORMAL again. +# +state=NORMAL # current state machine state +failover_start=0 # epoch timestamp when FAILOVER began (0 = not in FAILOVER) +handback_strikes=0 # consecutive remote-up checks accumulated toward handback +tier2_started=false # whether Tier 2 containers have been started this event +tier3_started=false # whether Tier 3 containers have been started +tier4_started=false # whether Tier 4 containers have been started +last_reset=2026-04-14 03:00:00 +reset_by=HOST1 # which server last reset the state (set by failover_state_reset.sh) +``` + +--- + +## ━━━ INITIAL SETUP REQUIREMENTS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Before `failover.sh` can work on both servers: + +--- + +### 1. Tailscale Connected on Both Servers + +```bash +# Verify from HOST1 — should return HOST2's Tailscale IP +tailscale ip -4 unRAID-Jayred365 + +# Verify from HOST2 — should return HOST1's Tailscale IP +tailscale ip -4 unRAID-Gmer4Lfe + +# Test reachability from HOST1 +tailscale ping unRAID-Jayred365 +``` + +--- + +### 2. SSH Keys Configured — No Password Prompt + +```bash +# From HOST1 — should print HOST2's hostname without asking for password ssh -i /root/.ssh/Gmer4Lfe-rsync-key root@[HOST2-tailscale-ip] "hostname" -# From HOST2 +# From HOST2 — should print HOST1's hostname without asking for password ssh -i /root/.ssh/Jayred365-rsync-key root@[HOST1-tailscale-ip] "hostname" ``` -**3. Container names match** - -Failover containers must exist on the server that will start them. If HOST2 starts `Emby` for HOST1, the `Emby` container must be created (but stopped) on HOST2 with its volume mounts pointing to the mirrored data. - -**4. Data mirrored** - -Critical appdata synced to the remote server before failover is needed — not after. The daily rsync profiles keep this current: -``` -appdata-Failover/Critical-Data → auth stack -appdata-Failover/Important-Data → NextCloud + Postgres -appdata-Failover/Emby → Emby userdata -appdata-Failover/Gmer4Lfe → server appdata -``` - -**5. DDNS TTL set to 1 minute** - -In your DDNS provider settings. Higher TTL means users continue hitting the old IP for longer after failover. 1 minute is the minimum most providers allow — it means worst-case 1 minute of disruption. - -**6. Both servers running failover.sh** - -Both servers must be running the script simultaneously. Failover only works in one direction if only one server is running it. - --- -## Scripts +### 3. Container Names Match -### `failover.sh` - -The main state machine. Run as a background task at array start on both servers. +Failover containers must be created (but stopped) on the server that will run them. +If HOST2 will start `Emby` for HOST1, the `Emby` container must exist on HOST2 with +its volume mounts pointing at the mirrored share paths. ```bash -# Scheduled as: At Startup of Array (Background Script) -/mnt/user/appdata/unraid_scripts/Failover/failover.sh - -# Check current state without restarting the loop -/mnt/user/appdata/unraid_scripts/Failover/failover.sh --status - -# Test logic without touching containers -/mnt/user/appdata/unraid_scripts/Failover/failover.sh --dry-run --log +# Verify the container exists on HOST2 (stopped is expected) +ssh root@[HOST2-ip] "docker inspect Emby --format '{{.State.Status}}'" +# Expected: created or exited — not "no such container" ``` -**To stop:** Click Abort in the User Scripts plugin. Do NOT kill the process directly — the state file may be left inconsistent. Use `Tools/failover_state_reset.sh` to recover from a stuck state. - --- -### `failover_test.sh` - -Controlled simulation of the full failover lifecycle. Validates everything works before you need it. +### 4. Critical Data Mirrored Before Failover is Needed ```bash -# Always dry run first -/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh --dry-run +# These shares must exist on HOST2 with current data from HOST1: +/mnt/user/appdata-Failover/Critical-Data # auth stack +/mnt/user/appdata-Failover/Important-Data # NextCloud + Postgres +/mnt/user/Media_Server/Emby # Emby userdata +/mnt/user/appdata-Failover/Gmer4Lfe # server appdata -# Live test — run during maintenance window -/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh +# Verify data is current — check modification times +ssh root@[HOST2-ip] "ls -la /mnt/user/appdata-Failover/Critical-Data/" ``` -**What it does:** +--- -1. Verifies both servers reachable and state is NORMAL -2. Adds iptables rule blocking all traffic to remote IP -3. Waits `FAILOVER_TEST_BLOCK_WAIT` seconds for `failover.sh` to detect outage -4. Verifies FAILOVER state and Tier 1 containers started -5. Removes iptables rule — remote becomes reachable again -6. Waits `FAILOVER_TEST_HANDBACK_WAIT` seconds for handback -7. Verifies containers returned and state is NORMAL -8. Full pass/fail report per phase +### 5. DDNS TTL Set to 1 Minute -**Safety trap:** The iptables rule is removed via `trap` on ANY exit — crash, error, ctrl-c, or normal completion. Remote connectivity is always restored regardless of test outcome. +In your DDNS provider settings. Higher TTL means users continue hitting the old IP +for longer after failover. 1 minute is the minimum most providers allow. At 5 minute +TTL, users can be hitting a downed server for up to 5 minutes before DNS switches. -**⚠️ Run during a maintenance window.** Real containers start and stop during the test — users will experience a brief interruption. Schedule it for 3am or a quiet period. +--- + +### 6. Both Servers Running failover.sh + +Failover only works in one direction if only one server is running the script. +For mutual coverage, both servers must be running it continuously. -**Timing configuration:** ```bash -FAILOVER_TEST_BLOCK_WAIT=150 # must be > FAILOVER_CHECK_INTERVAL + buffer -FAILOVER_TEST_HANDBACK_WAIT=360 # covers strikes × interval + rsync time +# Verify failover.sh is running on both servers +pgrep -f "failover.sh" + +# Check the state file directly +cat /boot/config/failover_state.db ``` --- -## State File +## ━━━ MONITORING ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -The state file at `/boot/config/failover_state.db` persists across reboots — it's on `/boot/` not `/tmp/`. This means the script remembers what state it was in before a reboot and can resume correctly. +**Sunday coffee report** (`sunday_morning_coffee_report.sh`) includes a Failover section +showing current state, outage duration if not NORMAL, Tailscale reachability, and +whether the failover.sh process is running. +**Weekly health digest** (`weekly_health_digest.sh`) reads the state file. If +`DIGEST_SMART_ON_FAILOVER=true` and state is not NORMAL, it sends a notification +even in smart mode — a non-NORMAL state at digest time needs attention. + +**Direct status check:** + +```bash +failover.sh --status # full state snapshot from current server ``` -state=NORMAL -failover_start=0 -handback_strikes=0 -tier2_started=false -tier3_started=false -tier4_started=false -last_reset=2026-04-14 03:00:00 -``` - -If the state file gets stuck in a non-NORMAL state after testing or a failed handback, use `Tools/failover_state_reset.sh` to reset it manually after verifying both servers are in their correct states. --- -## Monitoring - -`Monitors/weekly_health_digest.sh` reads the failover state file and includes it in the weekly digest. If `DIGEST_SMART_ON_FAILOVER=true` and the state is not NORMAL, the digest sends a notification even in smart mode. - -`failover.sh --status` gives an instant snapshot of current state, active tier flags, outage duration, and handback strike count. +## ━━━ TROUBLESHOOTING ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ --- -## What Gets Written Back on Handback - -| Data | Written back | Condition | -|------|-------------|-----------| -| Emby userdata / playstates | ✅ If outage > TIER1_WRITEBACK_DELAY (60min) | Small, important — watch history | -| Auth stack data | ✅ If outage > TIER1_WRITEBACK_DELAY (60min) | Authelia sessions, LLDAP data | -| NextCloud data | ✅ If outage > TIER2_DELAY (120min) | File changes during outage | -| Secondary appdata | ✅ If outage > TIER3_DELAY (360min) | Any accumulated changes | -| Opposing host's media shares | ✅ If outage > TIER4_DELAY (1080min) | HOST1_DAILY_SYNC_SHARES or HOST2_DAILY_SYNC_SHARES — same list daily_sync.sh uses, opposite direction | -| Edge case paths | ✅ If outage > TIER4_DELAY | Anything in FAILOVER_HOST*_WRITEBACK_TIER4 | -| Media files (own truth shares) | ❌ Never | Already on primary, never moved | -| Downloads | ❌ Never | Start fresh — cleaner than partial state | - -**Writeback skip window — short outages skip writeback entirely:** - -Emby syncs every 30 minutes from a live running container (dirty sync). A clean full sync runs nightly at 2:30am with Emby stopped. After a short outage, the primary's clean nightly state is more reliable than what accumulated on the covering server — skip writeback entirely. +### 🔴 Failover Not Triggering ``` -Under 60min → skip Tier 1-3 writeback — primary has cleaner data -Under 120min → skip Tier 2-3 writeback -Under 360min → skip Tier 3 writeback -Under 1080min → skip Tier 4 writeback — arrs never started, nothing to sync back -Over 1080min → full tiered writeback — meaningful delta accumulated -``` +Is failover.sh actually running on the covering server? + → User Scripts plugin → check status of the failover script -Real world outage profile: -``` -Power blip 2-10min → no writeback, clean restart (most common) -ISP issue 10-60min → no writeback, clean restart -Real outage 3hr → Tier 1+2 writeback, skip 3+4 -Extended 18hr+ → full tiered writeback including media shares -``` +Is Tailscale connected on the covering server? + → tailscale status (should show the remote peer) -**Tier 4 media writeback — opposing orch list:** - -At Tier 4, the covering server has been running the other server's arrs against the mirrored media shares. New content downloaded during the outage needs to sync back before the primary's own arrs restart. Rather than maintaining a duplicate share list, Tier 4 writeback automatically uses the opposing host's `HOST*_DAILY_SYNC_SHARES` — the same list `daily_sync.sh` uses, but in the opposite direction. +Can the covering server reach the remote's Tailscale IP? + → ping [remote-tailscale-ip] +What state is the covering server in? + → failover.sh --status (should show FAILOVER if remote is down) + → cat /boot/config/failover_state.db ``` -Normal (daily_sync.sh): HOST1 → pushes Movies, Tv_Shows → HOST2 -Tier 4 writeback: HOST2 → pushes Movies, Tv_Shows → HOST1 (opposite direction) -Same list, correct direction for the situation, no duplication. -``` - -Additional edge case paths outside the daily sync list can be added to `FAILOVER_HOST*_WRITEBACK_TIER4` in Master.conf. --- -## Troubleshooting +### 🔴 Handback Not Completing -**Failover not triggering:** -- Is `failover.sh` running on HOST2? Check User Scripts plugin -- Is HOST2's Tailscale connected and can it ping HOST1? -- Check the state file — what state is HOST2 in? +``` +Is HOST1's array fully started? + → ls /mnt/user (should show share directories) -**Handback not completing:** -- Is HOST1's array fully started? -- Is Docker responding on HOST1? `docker ps` should work -- Is HOST1's rootfs below `ROOTFS_WARN`? -- Check rsync writeback jobs — a stalled rsync blocks handback +Is Docker responding on HOST1? + → docker ps (should return a list, not hang) -**DDNS not cutting over:** -- Check the DDNS container is actually running on the covering server -- Check DNS TTL — if set high users won't see the cutover for a while -- Check your DDNS provider — are updates being accepted? +Is HOST1's rootfs below the warn threshold? + → df / (rootfs nearly full blocks handback pre-flight) -**State file stuck:** -- Use `Tools/failover_state_reset.sh --status` to see what's in it -- Verify both servers manually — right containers on right server, DDNS correct -- Run `Tools/failover_state_reset.sh` to reset +Is rsync running and stuck? + → pgrep rsync (a stalled rsync blocks the handback sequence) + → rsync_stop.sh to clear it, then let failover.sh retry +``` -**Split brain (both DDNS running):** -- This should not happen if the handback sequence is followed -- Check both servers — one should have DDNS stopped -- Manually stop the duplicate DDNS container -- Reset the state file and restart `failover.sh` \ No newline at end of file +--- + +### 🔴 DDNS Not Cutting Over + +``` +Is the DDNS container actually running on the covering server? + → docker ps | grep DDNS + +What TTL is your DNS record set to? + → nslookup Gmer4Lfe.com 8.8.8.8 (check TTL in response) + → High TTL means slow propagation + +Is your DDNS provider accepting updates? + → Check DDNS container logs: docker logs [ddns-container] --tail 50 +``` + +--- + +### 🔴 Split Brain (Both DDNS Running) + +``` +This should not happen if the handback sequence completed correctly. +If it has happened: + +1. Check both servers — which DDNS containers are running where + HOST1: docker ps | grep DDNS + HOST2: docker ps | grep DDNS + +2. Manually stop the duplicate + docker stop [duplicate-ddns-container] + +3. Reset the state file on the server in a bad state + failover_state_reset.sh --status (understand the situation) + failover_state_reset.sh --force (after verifying it is safe) + +4. Restart failover.sh (it will have been stopped by the state reset) + Start via User Scripts plugin +``` + +--- + +### 🔴 State File Stuck in FAILOVER After Testing + +``` +Normal — this is what failover_state_reset.sh is for. + +1. Verify everything is actually back to normal: + - Right containers on right server + - DDNS correct (nslookup Gmer4Lfe.com) + - Both servers visible on Tailscale + +2. Reset the state file: + failover_state_reset.sh --status (confirm what you're resetting) + failover_state_reset.sh (interactive — prompts YES) + +failover.sh will resume from NORMAL on its next cycle. +``` \ No newline at end of file diff --git a/Failover/failover.sh b/Failover/failover.sh index 85be47e..3685273 100644 --- a/Failover/failover.sh +++ b/Failover/failover.sh @@ -45,43 +45,65 @@ # ── HANDBACK SEQUENCE ───────────────────────────────────────────────────────────────────────── # When remote returns after FAILOVER: # 1. Strike confirmation — FAILOVER_HANDBACK_STRIKES consecutive remote-up checks -# prevents handing back during a brief network blip -# 2. Pre-flight checks — remote array started, Docker healthy, rootfs not full +# 2. Pre-flight checks — version parity, remote array, remote Docker daemon # 3. Stop remote DDNS first — prevents split brain DNS during rsync -# 4. Stop remote containers — clean state, no dirty writes during rsync -# 5. Tiered rsync writeback — skip if outage under threshold (short outages = cleaner to skip) -# Tier 1: skip if under HOST1_TIER1_WRITEBACK_DELAY (60min default) -# Tier 2: skip if under HOST1_TIER2_DELAY -# Tier 3: skip if under HOST1_TIER3_DELAY -# Tier 4: skip if under HOST*_TIER4_DELAY — opposing daily sync shares + edge cases -# 6. Start local containers — confirmed up before DNS cuts over -# 7. Start local DDNS — DNS cuts back ONLY after containers confirmed up +# 4. Stop remote containers — clean state before rsync +# 5. Tiered rsync writeback — skip if outage under threshold +# 6. Start remote containers — in dependency order with verification +# 7. Start remote DDNS last — DNS cuts back ONLY after containers confirmed up # 8. Return to NORMAL # # ── TIERED FAILOVER ─────────────────────────────────────────────────────────────────────────── -# Tier 1 — Immediate — vital services + Live TV — can't wait -# Tier 2 — 2hr default — shared productivity services -# Tier 3 — 6hr default — secondary services -# Tier 4 — 18hr default — arrs + downloaders — minimal writeback on handback -# Delays configurable per host in Master.conf +# Tier 1 — Immediate — vital services + Live TV — cannot wait +# Tier 2 — HOST*_TIER2_DELAY — shared productivity services (default 4hr) +# Tier 3 — HOST*_TIER3_DELAY — secondary services (default 12hr) +# Tier 4 — HOST*_TIER4_DELAY — arrs + downloaders (default 24hr) +# Tier delays configurable per host in master_host*.conf +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# FAILOVER_ENABLED gate — exits cleanly if disabled in master.conf +# Version parity check — refuses handback if unRAID versions mismatch +# Remote Docker daemon — checks remote daemon before issuing any remote commands +# Timeout protection — all docker and SSH commands wrapped in timeouts +# Container verification — verifies containers came up after start (failover critical) +# MY_ID-based routing — all tier/DDNS/writeback arrays selected via MY_ID not hostname +# Command validation — validates unRAID notify script before use +# Silent by default — state transitions warn(), routine cycle checks log() +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_DDNS_CONTAINERS — DDNS containers this host manages +# FAILOVER_HOST*_STOP_ON_NO_NET — containers stopped on internet loss +# FAILOVER_HOST*_RUNS_FOR_HOST*_TIER1-4 — what this host runs for the other +# HOST*_TIER2_DELAY / TIER3_DELAY / TIER4_DELAY — tier activation delays in minutes +# HOST*_TIER1_WRITEBACK_DELAY — skip Tier 1 writeback if outage under this +# FAILOVER_HOST*_WRITEBACK_TIER1-4 — paths synced back on handback per tier +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# FAILOVER_ENABLED — false = exit cleanly (HOST2 being rebuilt etc.) +# FAILOVER_CHECK_INTERVAL — seconds between checks +# FAILOVER_HANDBACK_STRIKES — consecutive remote-up checks before handback +# FAILOVER_STATE_FILE — /boot/config path — survives reboots +# FAILOVER_RSYNC_ENABLED — gate for writeback rsync jobs +# EXTERNAL_IP — IP to ping for internet check (default 8.8.8.8) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# failover.sh — normal start (continuous loop) +# failover.sh --dry-run — preview state changes without acting on containers +# failover.sh --status — show current state and exit +# failover.sh --log — verbose cycle output # # ── TO STOP THIS SCRIPT ─────────────────────────────────────────────────────────────────────── -# Click Abort beside the script in unRAID's User Scripts plugin page. -# Do NOT kill the process directly — state file may be left in inconsistent state. -# -# All configuration in Master.conf under Failover section. -# Supports --dry-run (no container changes) and --status (show current state and exit). +# Click Abort in unRAID User Scripts — do NOT kill directly, state file may corrupt. # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" # ============================================================================================== -# ━━━ $ICON_GEAR Setup ━━━ +# ━━━ Setup ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" @@ -91,20 +113,37 @@ if [[ "$EUID" -ne 0 ]]; then exit 1 fi -success "Running as root" +# FAILOVER_ENABLED gate — exits cleanly when disabled (e.g. HOST2 being rebuilt) +if [[ "${FAILOVER_ENABLED:-false}" == false ]]; then + warn "FAILOVER_ENABLED=false — failover monitoring disabled" + warn "Set FAILOVER_ENABLED=true in master.conf when both servers are ready" + exit 0 +fi -acquire_lock # failover.sh — strict single instance, notifies on collision +acquire_lock # strict single instance — notifies on collision detect_hosts resolve_remote_ip +# Validate unRAID notify script — used throughout for state change notifications +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no container or DDNS changes will be made" +# Timeout for all docker and SSH docker commands +DOCKER_TIMEOUT=15 +SSH_TIMEOUT=10 +CONTAINER_VERIFY_WAIT=5 # seconds after start before verifying container is up + # ============================================================================================== -# STATE FILE HELPERS -# State file format — key=value, one per line, persists on /boot/ -# Tracks: state, failover_start, handback_strikes, tier2_started, tier3_started, tier4_started +# ── STATE FILE HELPERS ──────────────────────────────────────────────────────────────────────── # ============================================================================================== +# State file on /boot/config — survives reboots. +# Format: key=value one per line. +# Keys: state, failover_start, handback_strikes, tier2_started, tier3_started, tier4_started state_get() { grep "^${1}=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2 @@ -122,35 +161,56 @@ state_set() { state_init() { mkdir -p "$(dirname "$FAILOVER_STATE_FILE")" [[ ! -f "$FAILOVER_STATE_FILE" ]] && touch "$FAILOVER_STATE_FILE" - [[ -z "$(state_get state)" ]] && state_set state "NORMAL" - [[ -z "$(state_get failover_start)" ]] && state_set failover_start "0" - [[ -z "$(state_get handback_strikes)" ]] && state_set handback_strikes "0" - [[ -z "$(state_get tier2_started)" ]] && state_set tier2_started "false" - [[ -z "$(state_get tier3_started)" ]] && state_set tier3_started "false" - [[ -z "$(state_get tier4_started)" ]] && state_set tier4_started "false" + [[ -z "$(state_get state)" ]] && state_set state "NORMAL" + [[ -z "$(state_get failover_start)" ]] && state_set failover_start "0" + [[ -z "$(state_get handback_strikes)" ]] && state_set handback_strikes "0" + [[ -z "$(state_get tier2_started)" ]] && state_set tier2_started "false" + [[ -z "$(state_get tier3_started)" ]] && state_set tier3_started "false" + [[ -z "$(state_get tier4_started)" ]] && state_set tier4_started "false" } # ============================================================================================== -# CONTAINER HELPERS +# ── CONTAINER HELPERS ───────────────────────────────────────────────────────────────────────── # ============================================================================================== +# All docker commands wrapped in DOCKER_TIMEOUT. +# All SSH docker commands wrapped in SSH_TIMEOUT. +# Verification after start — failover is critical, confirm containers came up. -# Start a container locally +# Start a container locally and verify it came up local_start() { local container="$1" [[ -z "$container" ]] && return local status - status=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \ + "$container" 2>/dev/null) if [[ "$status" == "true" ]]; then log "$container already running locally" return 0 fi - info "$ICON_START Starting $container locally..." - if [[ "$DRY_RUN" == false ]]; then - docker start "$container" >/dev/null 2>&1 && \ - success "$ICON_STARTED $container started" || \ - error "Failed to start $container locally" - else + log "$ICON_START Starting $container locally..." + if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would start $container locally" + return 0 + fi + if timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1; then + sleep "$CONTAINER_VERIFY_WAIT" + local post_status + post_status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \ + "$container" 2>/dev/null) + if [[ "$post_status" == "true" ]]; then + log "$ICON_STARTED $container started and running ✅" + return 0 + else + error "$container started but crashed immediately" + notify "$container failed to stay running during failover on $(hostname)" \ + "Failover" "warning" + return 1 + fi + else + error "Failed to start $container locally" + notify "Failed to start $container locally during failover on $(hostname)" \ + "Failover" "warning" + return 1 fi } @@ -159,40 +219,63 @@ local_stop() { local container="$1" [[ -z "$container" ]] && return local status - status=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \ + "$container" 2>/dev/null) if [[ "$status" != "true" ]]; then log "$container already stopped locally" return 0 fi - info "$ICON_STOP Stopping $container locally..." - if [[ "$DRY_RUN" == false ]]; then - docker stop "$container" >/dev/null 2>&1 && \ - success "$ICON_STOPPED $container stopped" || \ - error "Failed to stop $container locally" - else + log "$ICON_STOP Stopping $container locally..." + if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would stop $container locally" + return 0 fi + timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1 && \ + log "$ICON_STOPPED $container stopped" || \ + error "Failed to stop $container locally" } -# Start a container on remote via SSH +# Start a container on remote via SSH and verify it came up remote_start() { local container="$1" [[ -z "$container" ]] && return local status - status=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ - "docker inspect -f '{{.State.Running}}' $container 2>/dev/null" 2>/dev/null) + status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \ + root@"$REMOTE_SERVER" \ + "timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \ + $container 2>/dev/null" 2>/dev/null) if [[ "$status" == "true" ]]; then log "$container already running on $REMOTE_SERVER_NAME" return 0 fi - info "$ICON_START Starting $container on $REMOTE_SERVER_NAME..." - if [[ "$DRY_RUN" == false ]]; then - ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ - "docker start $container" >/dev/null 2>&1 && \ - success "$ICON_STARTED $container started on $REMOTE_SERVER_NAME" || \ - error "Failed to start $container on $REMOTE_SERVER_NAME" - else + log "$ICON_START Starting $container on $REMOTE_SERVER_NAME..." + if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would start $container on $REMOTE_SERVER_NAME" + return 0 + fi + if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \ + root@"$REMOTE_SERVER" \ + "timeout $DOCKER_TIMEOUT docker start $container" >/dev/null 2>&1; then + sleep "$CONTAINER_VERIFY_WAIT" + local post_status + post_status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" root@"$REMOTE_SERVER" \ + "timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \ + $container 2>/dev/null" 2>/dev/null) + if [[ "$post_status" == "true" ]]; then + log "$ICON_STARTED $container started on $REMOTE_SERVER_NAME ✅" + return 0 + else + error "$container started on $REMOTE_SERVER_NAME but crashed immediately" + notify "$container failed after start on $REMOTE_SERVER_NAME during handback on $(hostname)" \ + "Failover" "warning" + return 1 + fi + else + error "Failed to start $container on $REMOTE_SERVER_NAME" + notify "Failed to start $container on $REMOTE_SERVER_NAME during handback on $(hostname)" \ + "Failover" "warning" + return 1 fi } @@ -201,226 +284,209 @@ remote_stop() { local container="$1" [[ -z "$container" ]] && return local status - status=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ - "docker inspect -f '{{.State.Running}}' $container 2>/dev/null" 2>/dev/null) + status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \ + root@"$REMOTE_SERVER" \ + "timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \ + $container 2>/dev/null" 2>/dev/null) if [[ "$status" != "true" ]]; then log "$container already stopped on $REMOTE_SERVER_NAME" return 0 fi - info "$ICON_STOP Stopping $container on $REMOTE_SERVER_NAME..." - if [[ "$DRY_RUN" == false ]]; then - ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ - "docker stop $container" >/dev/null 2>&1 && \ - success "$ICON_STOPPED $container stopped on $REMOTE_SERVER_NAME" || \ - error "Failed to stop $container on $REMOTE_SERVER_NAME" - else + log "$ICON_STOP Stopping $container on $REMOTE_SERVER_NAME..." + if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would stop $container on $REMOTE_SERVER_NAME" + return 0 fi + timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \ + root@"$REMOTE_SERVER" \ + "timeout $DOCKER_TIMEOUT docker stop $container" >/dev/null 2>&1 && \ + log "$ICON_STOPPED $container stopped on $REMOTE_SERVER_NAME" || \ + error "Failed to stop $container on $REMOTE_SERVER_NAME" } # ============================================================================================== -# DDNS HELPERS -# DDNS is managed exclusively by this script — never by network state +# ── DDNS HELPERS ────────────────────────────────────────────────────────────────────────────── # ============================================================================================== +# DDNS managed exclusively by this script — never by network state alone. +# Stopping DDNS on internet loss prevents split-brain DNS updates. -# Stop local DDNS — called on internet loss local_ddns_stop() { - info "$ICON_NET Stopping local DDNS — internet lost" + warn "$ICON_NET Stopping local DDNS — internet lost" for container in "${LOCAL_DDNS_CONTAINERS[@]}"; do local_stop "$container" done } -# Start local DDNS — called at end of handback ONLY after containers confirmed up local_ddns_start() { - info "$ICON_NET Starting local DDNS — containers confirmed up" + warn "$ICON_NET Starting local DDNS — handback complete, containers confirmed up" for container in "${LOCAL_DDNS_CONTAINERS[@]}"; do local_start "$container" done } -# Stop remote DDNS — FIRST step of handback sequence, prevents split brain remote_ddns_stop() { - info "$ICON_NET Stopping remote DDNS on $REMOTE_SERVER_NAME — handback starting" + warn "$ICON_NET Stopping remote DDNS on $REMOTE_SERVER_NAME — prevents split-brain during rsync" for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do remote_stop "$container" done } # ============================================================================================== -# TIERED FAILOVER HELPERS -# Containers selected based on which server is local and which arrays to use +# ── TIERED FAILOVER HELPERS ─────────────────────────────────────────────────────────────────── # ============================================================================================== +# All routing uses MY_ID — not hostname string comparison. +# MY_ID set by detect_hosts() — "HOST1" or "HOST2" etc. +# "This server covers the other" — MY_ID selects which tier arrays to use. -# Get the correct tier arrays based on local host -# HOST2 runs for HOST1, HOST1 runs for HOST2 get_tier_containers() { local tier="$1" - if [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]]; then - # HOST2 is running — covers HOST1 - case "$tier" in - 1) echo "${FAILOVER_HOST2_RUNS_FOR_HOST1_IMMEDIATE[@]}" ;; - 2) echo "${FAILOVER_HOST2_RUNS_FOR_HOST1_2HR[@]}" ;; - 3) echo "${FAILOVER_HOST2_RUNS_FOR_HOST1_6HR[@]}" ;; - 4) echo "${FAILOVER_HOST2_RUNS_FOR_HOST1_18HR[@]}" ;; - esac - else - # HOST1 is running — covers HOST2 - case "$tier" in - 1) echo "${FAILOVER_HOST1_RUNS_FOR_HOST2_IMMEDIATE[@]}" ;; - 2) echo "${FAILOVER_HOST1_RUNS_FOR_HOST2_2HR[@]}" ;; - 3) echo "${FAILOVER_HOST1_RUNS_FOR_HOST2_6HR[@]}" ;; - 4) echo "${FAILOVER_HOST1_RUNS_FOR_HOST2_18HR[@]}" ;; - esac - fi + local remote_id="${REMOTE_ID}" + local var_name="FAILOVER_${MY_ID}_RUNS_FOR_${remote_id}_TIER${tier}" + eval "echo \"\${${var_name}[@]:-}\"" } get_tier2_delay() { - [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]] && \ - echo "$HOST1_TIER2_DELAY" || echo "$HOST2_TIER2_DELAY" + local var_name="${REMOTE_ID}_TIER2_DELAY" + echo "${!var_name:-240}" } get_tier3_delay() { - [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]] && \ - echo "$HOST1_TIER3_DELAY" || echo "$HOST2_TIER3_DELAY" + local var_name="${REMOTE_ID}_TIER3_DELAY" + echo "${!var_name:-720}" } get_tier4_delay() { - [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]] && \ - echo "$HOST1_TIER4_DELAY" || echo "$HOST2_TIER4_DELAY" + local var_name="${REMOTE_ID}_TIER4_DELAY" + echo "${!var_name:-1440}" } get_tier1_writeback_delay() { - [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]] && \ - echo "$HOST1_TIER1_WRITEBACK_DELAY" || echo "$HOST2_TIER1_WRITEBACK_DELAY" + local var_name="${REMOTE_ID}_TIER1_WRITEBACK_DELAY" + echo "${!var_name:-60}" } get_writeback_jobs_for_tier() { local tier="$1" - if [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]]; then - # HOST2 is covering HOST1 — write back HOST1's data - case "$tier" in - 1) echo "${FAILOVER_HOST1_WRITEBACK_TIER1[@]}" ;; - 2) echo "${FAILOVER_HOST1_WRITEBACK_TIER2[@]}" ;; - 3) echo "${FAILOVER_HOST1_WRITEBACK_TIER3[@]}" ;; - 4) - # Tier 4 — push HOST1's daily sync shares back (opposing orch list) - # These are HOST1's source-of-truth shares that HOST2's arrs managed during outage - # Plus any edge case paths defined in FAILOVER_HOST1_WRITEBACK_TIER4 - echo "${HOST1_DAILY_SYNC_SHARES[@]}" - echo "${FAILOVER_HOST1_WRITEBACK_TIER4[@]}" - ;; - esac + local remote_id="${REMOTE_ID}" + if [[ "$tier" -eq 4 ]]; then + # Tier 4 — daily sync shares of the remote + any extra writeback paths + local shares_var="${remote_id}_DAILY_SYNC_SHARES" + local extra_var="FAILOVER_${remote_id}_WRITEBACK_TIER4" + eval "echo \"\${${shares_var}[@]:-} \${${extra_var}[@]:-}\"" else - # HOST1 is covering HOST2 — write back HOST2's data - case "$tier" in - 1) echo "${FAILOVER_HOST2_WRITEBACK_TIER1[@]}" ;; - 2) echo "${FAILOVER_HOST2_WRITEBACK_TIER2[@]}" ;; - 3) echo "${FAILOVER_HOST2_WRITEBACK_TIER3[@]}" ;; - 4) - # Tier 4 — push HOST2's daily sync shares back (opposing orch list) - # These are HOST2's source-of-truth shares that HOST1's arrs managed during outage - # Plus any edge case paths defined in FAILOVER_HOST2_WRITEBACK_TIER4 - echo "${HOST2_DAILY_SYNC_SHARES[@]}" - echo "${FAILOVER_HOST2_WRITEBACK_TIER4[@]}" - ;; - esac + local var_name="FAILOVER_${remote_id}_WRITEBACK_TIER${tier}" + eval "echo \"\${${var_name}[@]:-}\"" fi } -# Select correct DDNS arrays based on local host +# Select DDNS arrays based on MY_ID set_ddns_arrays() { - if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - LOCAL_DDNS_CONTAINERS=("${HOST1_DDNS_CONTAINERS[@]}") - REMOTE_DDNS_CONTAINERS=("${HOST2_DDNS_CONTAINERS[@]}") - else - LOCAL_DDNS_CONTAINERS=("${HOST2_DDNS_CONTAINERS[@]}") - REMOTE_DDNS_CONTAINERS=("${HOST1_DDNS_CONTAINERS[@]}") - fi + local local_ddns_var="${MY_ID}_DDNS_CONTAINERS" + local remote_ddns_var="${REMOTE_ID}_DDNS_CONTAINERS" + eval "LOCAL_DDNS_CONTAINERS=(\"\${${local_ddns_var}[@]:-}\")" + eval "REMOTE_DDNS_CONTAINERS=(\"\${${remote_ddns_var}[@]:-}\")" +} + +# Select internet-loss stop list based on MY_ID +get_stop_on_no_net() { + local var_name="FAILOVER_${MY_ID}_STOP_ON_NO_NET" + eval "echo \"\${${var_name}[@]:-}\"" } # ============================================================================================== -# STATUS DISPLAY +# ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then state_init set_ddns_arrays CURRENT_STATE=$(state_get state) - FAILOVER_START=$(state_get failover_start) + FAILOVER_START_TS=$(state_get failover_start) TIER2=$(state_get tier2_started) TIER3=$(state_get tier3_started) TIER4=$(state_get tier4_started) STRIKES=$(state_get handback_strikes) + local_ver=$(grep -oP '(?<=version=")[^"]+' /etc/unraid-version 2>/dev/null || echo "unknown") + echo "" echo "━━━━━ $ICON_SUMMARY FAILOVER STATUS ━━━━━" - echo "$ICON_HOST Local: $LOCAL_SERVER_NAME" - echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)" - echo "$ICON_FAILOVER State: $CURRENT_STATE" - echo "$ICON_NET Local DDNS: ${LOCAL_DDNS_CONTAINERS[*]}" - echo "$ICON_NET Remote DDNS: ${REMOTE_DDNS_CONTAINERS[*]}" - echo "$ICON_TIME Interval: ${FAILOVER_CHECK_INTERVAL}s" - echo "$ICON_FAILOVER Strikes: $STRIKES / $FAILOVER_HANDBACK_STRIKES" + echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_HOST Remote ID: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)" + echo "$ICON_GEAR unRAID ver: $local_ver" + echo "$ICON_FAILOVER State: $CURRENT_STATE" + echo "$ICON_NET Local DDNS: ${LOCAL_DDNS_CONTAINERS[*]:-none}" + echo "$ICON_NET Remote DDNS: ${REMOTE_DDNS_CONTAINERS[*]:-none}" + echo "$ICON_TIME Interval: ${FAILOVER_CHECK_INTERVAL}s" + echo "$ICON_FAILOVER Strikes: $STRIKES / $FAILOVER_HANDBACK_STRIKES" - if [[ "$CURRENT_STATE" == "FAILOVER" && "$FAILOVER_START" -gt 0 ]]; then - ELAPSED=$(( ($(date +%s) - FAILOVER_START) / 60 )) - echo "$ICON_TIME Outage: ${ELAPSED}min" - echo "$ICON_FAILOVER Tier 2: $TIER2 (delay: $(get_tier2_delay)min)" - echo "$ICON_FAILOVER Tier 3: $TIER3 (delay: $(get_tier3_delay)min)" - echo "$ICON_FAILOVER Tier 4: $TIER4 (delay: $(get_tier4_delay)min)" + if [[ "$CURRENT_STATE" == "FAILOVER" && "$FAILOVER_START_TS" -gt 0 ]]; then + ELAPSED=$(( ($(date +%s) - FAILOVER_START_TS) / 60 )) + echo "$ICON_TIME Outage: ${ELAPSED}min" + echo "$ICON_FAILOVER Tier 2: $TIER2 (delay: $(get_tier2_delay)min)" + echo "$ICON_FAILOVER Tier 3: $TIER3 (delay: $(get_tier3_delay)min)" + echo "$ICON_FAILOVER Tier 4: $TIER4 (delay: $(get_tier4_delay)min)" fi - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_GEAR Enabled: $FAILOVER_ENABLED" + echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== -# HANDBACK SEQUENCE -# Called when remote returns after FAILOVER state -# Critical sequencing — do not reorder without understanding the consequences +# ── HANDBACK SEQUENCE ───────────────────────────────────────────────────────────────────────── # ============================================================================================== +# Called when remote returns after FAILOVER state. +# CRITICAL sequencing — do not reorder without understanding the consequences. +# Each step must succeed before proceeding — aborts and retries next cycle on failure. + run_handback() { echo "" echo "━━━ $ICON_FAILOVER Handback Sequence — $(date '+%Y-%m-%d %H:%M:%S') ━━━" - info "Remote $REMOTE_SERVER_NAME has returned — beginning handback" + warn "$REMOTE_SERVER_NAME has returned — beginning handback" # ── Step 1: Pre-flight checks ──────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_SHIELD Pre-flight ━━━" + # Version parity — refuse handback if unRAID versions mismatch + if ! check_unraid_version_parity; then + warn "Version parity check failed — aborting handback, will retry next cycle" + state_set handback_strikes 0 + return 1 + fi + + # Remote array must be mounted before rsync if ! check_remote_array; then warn "Remote array not ready — aborting handback, will retry next cycle" state_set handback_strikes 0 return 1 fi - if ! check_remote_docker; then - warn "Remote Docker not ready — aborting handback, will retry next cycle" + # Remote Docker daemon must be responsive before issuing container commands + if ! check_remote_docker_daemon; then + warn "Remote Docker daemon not ready — aborting handback, will retry next cycle" state_set handback_strikes 0 return 1 fi - success "Pre-flight checks passed" + log "Pre-flight checks passed ✅" # ── Step 2: Stop remote DDNS FIRST ────────────────────────────────────────────────────── - # CRITICAL — prevents split brain DNS during rsync window - # Remote DDNS must be OFF before any data moves + # CRITICAL — prevents split-brain DNS while data is being synced echo "" echo "━━━ $ICON_NET DDNS Handoff ━━━" remote_ddns_stop - info "Remote DDNS stopped — DNS will not update during rsync" - sleep 5 # brief pause to ensure DDNS stop propagates + sleep 5 # brief pause to ensure DDNS stop propagates before rsync # ── Step 3: Stop remote containers ────────────────────────────────────────────────────── - # Clean state before rsync — no dirty writes during transfer - # Containers are only down during the rsync window — minimise this time + # Clean state before rsync — no dirty writes during transfer window echo "" echo "━━━ $ICON_STOP Stop Remote Containers ━━━" - # Stop all tiers that were started — in reverse order local ALL_FAILOVER_CONTAINERS=() + # Collect all started tiers in reverse order (highest tier stops first) if [[ "$(state_get tier4_started)" == "true" ]]; then read -r -a t4 <<< "$(get_tier_containers 4)" ALL_FAILOVER_CONTAINERS+=("${t4[@]}") @@ -434,9 +500,8 @@ run_handback() { ALL_FAILOVER_CONTAINERS+=("${t2[@]}") fi - # Tier 1 always started — stop last (DDNS already stopped above) + # Tier 1 always started — stop last (DDNS already handled above) read -r -a t1 <<< "$(get_tier_containers 1)" - # Remove DDNS containers from tier 1 list — already handled for container in "${t1[@]}"; do local is_ddns=false for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do @@ -450,77 +515,71 @@ run_handback() { local_stop "$container" done - success "All failover containers stopped locally" + log "All failover containers stopped locally" # ── Step 4: Rsync writeback ────────────────────────────────────────────────────────────── # Tiered writeback with skip window — short outages do not benefit from writeback. - # Emby syncs every 30min dirty (live container). Clean sync runs nightly at 2:30am. - # After a short outage HOST1's clean nightly state is more reliable than HOST2's - # dirty sync accumulation — skip writeback entirely for short outages. - # - # Tier 1 — skip if under HOST1_TIER1_WRITEBACK_DELAY (default 60min) - # Tier 2 — skip if under HOST1_TIER2_DELAY (reused — if Tier 2 never started, skip) - # Tier 3 — skip if under HOST1_TIER3_DELAY (reused — same logic) - # Tier 4 — always writeback — 18hr+ means meaningful delta accumulated + # After a short outage primary's clean nightly state is more reliable than dirty + # accumulation — skip writeback entirely for outages under the threshold. echo "" echo "━━━ $ICON_SYNC Rsync Writeback ━━━" - local outage_minutes=$(( ($(date +%s) - $(state_get failover_start)) / 60 )) + local outage_minutes + outage_minutes=$(( ($(date +%s) - $(state_get failover_start)) / 60 )) local tier1_wb_delay tier1_wb_delay=$(get_tier1_writeback_delay) - info "Outage duration: ${outage_minutes}min" + warn "Outage duration: ${outage_minutes}min" - # Check rsync gate before running any writeback if ! check_rsync_enabled "FAILOVER"; then warn "FAILOVER_RSYNC_ENABLED=false — skipping all writeback jobs" warn "Handback will complete without syncing state back to primary" - return 0 + else + run_writeback_tier() { + local tier="$1" threshold="$2" label="$3" + local jobs + read -r -a jobs <<< "$(get_writeback_jobs_for_tier "$tier")" + [[ ${#jobs[@]} -eq 0 ]] && return + + if [[ "$outage_minutes" -ge "$threshold" ]]; then + warn "Tier $tier writeback ($label) — outage ${outage_minutes}min >= ${threshold}min" + for job in "${jobs[@]}"; do + [[ -z "$job" ]] && continue + log "Syncing: $job" + if [[ "$DRY_RUN" == false ]]; then + if [[ "$(basename "$job")" == "Emby" ]]; then + bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" --profile=emby-failover + else + bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" + fi + else + warn "DRY RUN — would rsync: $job" + fi + done + else + log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min — primary state is cleaner" + fi + } + + local t2_delay t3_delay t4_delay + t2_delay=$(get_tier2_delay) + t3_delay=$(get_tier3_delay) + t4_delay=$(get_tier4_delay) + + run_writeback_tier 1 "$tier1_wb_delay" "Emby + auth stack" + run_writeback_tier 2 "$t2_delay" "productivity services" + run_writeback_tier 3 "$t3_delay" "secondary services" + run_writeback_tier 4 "$t4_delay" "media shares + edge cases" fi - run_writeback_tier() { - local tier="$1" - local threshold="$2" - local label="$3" - local jobs - read -r -a jobs <<< "$(get_writeback_jobs_for_tier "$tier")" - [[ ${#jobs[@]} -eq 0 ]] && return - - if [[ "$outage_minutes" -ge "$threshold" ]]; then - info "Tier $tier writeback ($label) — outage ${outage_minutes}min >= ${threshold}min" - for job in "${jobs[@]}"; do - [[ -z "$job" ]] && continue - info "Syncing: $job" - if [[ "$DRY_RUN" == false ]]; then - # Emby writeback uses emby-failover profile — critical data only, Emby stays running - if [[ "$(basename "$job")" == "Emby" ]]; then - bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" --profile=emby-failover - else - bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" - fi - else - warn "DRY RUN — would rsync: $job" - fi - done - else - info "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min threshold" - info "Primary has cleaner state — no writeback needed" - fi - } - - run_writeback_tier 1 "$tier1_wb_delay" "Emby + auth stack" - run_writeback_tier 2 "$HOST1_TIER2_DELAY" "NextCloud + Immich" - run_writeback_tier 3 "$HOST1_TIER3_DELAY" "secondary services" - run_writeback_tier 4 "$(get_tier4_delay)" "media shares + edge cases" - - success "Writeback complete" + log "Writeback complete" # ── Step 5: Start remote containers ───────────────────────────────────────────────────── - # Start in dependency order — databases before apps + # Start in tier order with dependency awareness — databases before apps echo "" echo "━━━ $ICON_START Start Remote Containers ━━━" - # Start tier 1 on remote (excluding DDNS — handled separately) + # Tier 1 — excluding DDNS (handled separately as final step) for container in "${t1[@]}"; do [[ -z "$container" ]] && continue local is_ddns=false @@ -530,39 +589,34 @@ run_handback() { [[ "$is_ddns" == false ]] && remote_start "$container" done - # Brief pause — give databases time to initialise before apps - sleep 10 + sleep 10 # give databases time to initialise before apps - if [[ "$(state_get tier2_started)" == "true" ]]; then + [[ "$(state_get tier2_started)" == "true" ]] && \ for container in "${t2[@]}"; do [[ -n "$container" ]] && remote_start "$container" done - fi - if [[ "$(state_get tier3_started)" == "true" ]]; then + [[ "$(state_get tier3_started)" == "true" ]] && \ for container in "${t3[@]}"; do [[ -n "$container" ]] && remote_start "$container" done - fi - if [[ "$(state_get tier4_started)" == "true" ]]; then + [[ "$(state_get tier4_started)" == "true" ]] && \ for container in "${t4[@]}"; do [[ -n "$container" ]] && remote_start "$container" done - fi - success "Remote containers started" + log "Remote containers started" # ── Step 6: Start remote DDNS LAST ────────────────────────────────────────────────────── - # DNS cuts over ONLY after containers confirmed up - # This is the final step — after this users hit remote + # DNS cuts over ONLY after containers confirmed up — this is the final step echo "" echo "━━━ $ICON_NET DNS Cutover ━━━" sleep 5 # brief pause to ensure containers are accepting connections for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do remote_start "$container" done - success "Remote DDNS started — DNS now points at $REMOTE_SERVER_NAME" + warn "$ICON_NET Remote DDNS started — DNS now points at $REMOTE_SERVER_NAME" # ── Step 7: Return to NORMAL ───────────────────────────────────────────────────────────── echo "" @@ -573,12 +627,13 @@ run_handback() { state_set tier3_started "false" state_set tier4_started "false" - success "$ICON_DONE Handback complete — returned to NORMAL" - notify "Failover handback complete on $(hostname) — $REMOTE_SERVER_NAME is back, all containers returned" "Failover" "normal" + warn "$ICON_DONE Handback complete — returned to NORMAL" + notify "Failover handback complete on $(hostname) — $REMOTE_SERVER_NAME is back, all containers returned" \ + "Failover" "normal" } # ============================================================================================== -# MAIN STATE MACHINE +# ━━━ Main State Machine ━━━ # ============================================================================================== state_init set_ddns_arrays @@ -586,11 +641,15 @@ set_ddns_arrays echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo " $ICON_FAILOVER FAILOVER — $(date '+%Y-%m-%d %H:%M:%S')" -echo " $ICON_HOST $LOCAL_SERVER_NAME → monitoring $REMOTE_SERVER_NAME" -echo " $ICON_NET Remote: $REMOTE_SERVER" +echo " $ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → monitoring $REMOTE_ID ($REMOTE_SERVER_NAME)" +echo " $ICON_NET Remote IP: $REMOTE_SERVER" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -while true; do +FAILOVER_RUNNING=true +trap 'FAILOVER_RUNNING=false; warn "Failover received shutdown signal — stopping cleanly"; exit 0' \ + SIGTERM SIGINT + +while [[ "$FAILOVER_RUNNING" == true ]]; do NOW=$(date +%s) CURRENT_STATE=$(state_get state) @@ -598,27 +657,24 @@ while true; do REMOTE_UP=false INTERNET_UP=false - ping_remote && REMOTE_UP=true + ping_remote && REMOTE_UP=true ping_internet && INTERNET_UP=true log "Remote: $REMOTE_UP | Internet: $INTERNET_UP | State: $CURRENT_STATE" - # ── State machine ──────────────────────────────────────────────────────────────────────── - # ════════════════════════════════════════════════════════════════ # NORMAL STATE # ════════════════════════════════════════════════════════════════ if [[ "$CURRENT_STATE" == "NORMAL" ]]; then if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then - # All good — silent operation log "$ICON_SUCCESS NORMAL — all systems up" elif [[ "$REMOTE_UP" == false && "$INTERNET_UP" == true ]]; then # Remote is down — enter FAILOVER echo "" echo "━━━ $ICON_FAILOVER Entering FAILOVER — $(date '+%Y-%m-%d %H:%M:%S') ━━━" - warn "$REMOTE_SERVER_NAME is unreachable — internet is up — starting failover" + warn "$REMOTE_SERVER_NAME ($REMOTE_ID) is unreachable — internet is up — starting failover" state_set state "FAILOVER" state_set failover_start "$NOW" @@ -627,7 +683,6 @@ while true; do state_set tier3_started "false" state_set tier4_started "false" - # Start Tier 1 immediately — DDNS first, then vital services echo "" echo "━━━ $ICON_START Tier 1 — Immediate ━━━" read -r -a tier1 <<< "$(get_tier_containers 1)" @@ -635,7 +690,8 @@ while true; do [[ -n "$container" ]] && local_start "$container" done - notify "FAILOVER started on $(hostname) — $REMOTE_SERVER_NAME is down — Tier 1 containers started" "Failover" "warning" + notify "FAILOVER started on $(hostname) — $REMOTE_SERVER_NAME is down — Tier 1 started" \ + "Failover" "warning" elif [[ "$INTERNET_UP" == false ]]; then # Lost internet — enter NO_INTERNET @@ -646,18 +702,14 @@ while true; do state_set state "NO_INTERNET" local_ddns_stop - # Stop any containers configured to stop without internet - if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - for container in "${FAILOVER_HOST1_STOP_ON_NO_NET[@]}"; do - [[ -n "$container" ]] && local_stop "$container" - done - else - for container in "${FAILOVER_HOST2_STOP_ON_NO_NET[@]}"; do - [[ -n "$container" ]] && local_stop "$container" - done - fi + # Stop containers configured to stop on internet loss + read -r -a stop_on_no_net <<< "$(get_stop_on_no_net)" + for container in "${stop_on_no_net[@]}"; do + [[ -n "$container" ]] && local_stop "$container" + done - notify "NO_INTERNET on $(hostname) — DDNS stopped, waiting for recovery" "Failover" "warning" + notify "NO_INTERNET on $(hostname) — DDNS stopped, waiting for recovery" \ + "Failover" "warning" fi # ════════════════════════════════════════════════════════════════ @@ -665,8 +717,8 @@ while true; do # ════════════════════════════════════════════════════════════════ elif [[ "$CURRENT_STATE" == "FAILOVER" ]]; then - FAILOVER_START=$(state_get failover_start) - ELAPSED_MIN=$(( (NOW - FAILOVER_START) / 60 )) + FAILOVER_START_TS=$(state_get failover_start) + ELAPSED_MIN=$(( (NOW - FAILOVER_START_TS) / 60 )) if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then # Remote returned — increment handback strikes @@ -674,7 +726,7 @@ while true; do STRIKES=$(( STRIKES + 1 )) state_set handback_strikes "$STRIKES" - info "$ICON_FAILOVER $REMOTE_SERVER_NAME is back — handback strike $STRIKES/$FAILOVER_HANDBACK_STRIKES" + warn "$ICON_FAILOVER $REMOTE_SERVER_NAME is back — handback strike $STRIKES/$FAILOVER_HANDBACK_STRIKES" if [[ "$STRIKES" -ge "$FAILOVER_HANDBACK_STRIKES" ]]; then run_handback @@ -687,13 +739,13 @@ while true; do warn "Lost internet during failover — entering DARK state" state_set state "DARK" local_ddns_stop - notify "DARK state on $(hostname) — lost internet during failover" "Failover" "warning" + notify "DARK state on $(hostname) — lost internet during failover" \ + "Failover" "warning" else # Still in failover — check tier escalation - state_set handback_strikes "0" # reset strikes — remote still down + state_set handback_strikes "0" - # Tier 2 TIER2_DELAY=$(get_tier2_delay) if [[ "$(state_get tier2_started)" == "false" && \ "$ELAPSED_MIN" -ge "$TIER2_DELAY" ]]; then @@ -704,10 +756,10 @@ while true; do [[ -n "$container" ]] && local_start "$container" done state_set tier2_started "true" - notify "Failover Tier 2 started on $(hostname) — ${ELAPSED_MIN}min outage" "Failover" "warning" + notify "Failover Tier 2 started on $(hostname) — ${ELAPSED_MIN}min outage" \ + "Failover" "warning" fi - # Tier 3 TIER3_DELAY=$(get_tier3_delay) if [[ "$(state_get tier3_started)" == "false" && \ "$ELAPSED_MIN" -ge "$TIER3_DELAY" ]]; then @@ -718,10 +770,10 @@ while true; do [[ -n "$container" ]] && local_start "$container" done state_set tier3_started "true" - notify "Failover Tier 3 started on $(hostname) — ${ELAPSED_MIN}min outage" "Failover" "warning" + notify "Failover Tier 3 started on $(hostname) — ${ELAPSED_MIN}min outage" \ + "Failover" "warning" fi - # Tier 4 TIER4_DELAY=$(get_tier4_delay) if [[ "$(state_get tier4_started)" == "false" && \ "$ELAPSED_MIN" -ge "$TIER4_DELAY" ]]; then @@ -732,10 +784,11 @@ while true; do [[ -n "$container" ]] && local_start "$container" done state_set tier4_started "true" - notify "Failover Tier 4 started on $(hostname) — ${ELAPSED_MIN}min outage — full workflow active" "Failover" "warning" + notify "Failover Tier 4 started on $(hostname) — ${ELAPSED_MIN}min outage — full workflow active" \ + "Failover" "warning" fi - log "$ICON_FAILOVER FAILOVER active — ${ELAPSED_MIN}min — T2:$(state_get tier2_started) T3:$(state_get tier3_started) T4:$(state_get tier4_started)" + log "$ICON_FAILOVER FAILOVER — ${ELAPSED_MIN}min — T2:$(state_get tier2_started) T3:$(state_get tier3_started) T4:$(state_get tier4_started)" fi # ════════════════════════════════════════════════════════════════ @@ -744,22 +797,17 @@ while true; do elif [[ "$CURRENT_STATE" == "NO_INTERNET" ]]; then if [[ "$INTERNET_UP" == true ]]; then - # Internet recovered echo "" echo "━━━ $ICON_NET Internet Recovered — $(date '+%Y-%m-%d %H:%M:%S') ━━━" if [[ "$REMOTE_UP" == true ]]; then - # Remote up, internet up — return to NORMAL - info "Remote is up — returning to NORMAL" + warn "Remote is up — returning to NORMAL" state_set state "NORMAL" - # DDNS does NOT auto-start here — it only starts via handback sequence - # or explicit NORMAL state management - # If local DDNS should be running in NORMAL — start it now local_ddns_start - notify "Internet recovered on $(hostname) — returning to NORMAL" "Failover" "normal" + notify "Internet recovered on $(hostname) — returning to NORMAL" \ + "Failover" "normal" else - # Internet back but remote still down — enter FAILOVER - info "Remote still down — entering FAILOVER" + warn "Internet back but $REMOTE_SERVER_NAME still down — entering FAILOVER" state_set state "FAILOVER" state_set failover_start "$NOW" state_set handback_strikes "0" @@ -773,7 +821,8 @@ while true; do for container in "${tier1[@]}"; do [[ -n "$container" ]] && local_start "$container" done - notify "Internet recovered on $(hostname) but $REMOTE_SERVER_NAME still down — entering FAILOVER" "Failover" "warning" + notify "Internet recovered on $(hostname) but $REMOTE_SERVER_NAME still down — entering FAILOVER" \ + "Failover" "warning" fi else log "$ICON_NET NO_INTERNET — waiting for connectivity" @@ -789,13 +838,12 @@ while true; do echo "━━━ $ICON_NET Emerging from DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━" if [[ "$REMOTE_UP" == true ]]; then - info "Remote up, internet up — returning to NORMAL" - # Was in failover before DARK — need to handback + warn "Remote up, internet up — transitioning through FAILOVER for handback" + # Was in failover before DARK — route through FAILOVER state for handback state_set state "FAILOVER" state_set handback_strikes "0" - # Will trigger handback on next cycle via FAILOVER + remote up logic else - info "Internet back but remote still down — entering FAILOVER" + warn "Internet back but $REMOTE_SERVER_NAME still down — entering FAILOVER" state_set state "FAILOVER" state_set failover_start "$NOW" state_set handback_strikes "0" @@ -816,6 +864,7 @@ while true; do # ── Sleep until next check ─────────────────────────────────────────────────────────────── log "Next check in ${FAILOVER_CHECK_INTERVAL}s — $(date '+%H:%M:%S')" - sleep "$FAILOVER_CHECK_INTERVAL" + sleep "$FAILOVER_CHECK_INTERVAL" & + wait $! done \ No newline at end of file diff --git a/Failover/failover_test.sh b/Failover/failover_test.sh index 075075f..fae4454 100644 --- a/Failover/failover_test.sh +++ b/Failover/failover_test.sh @@ -1,47 +1,71 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Failover Test ---------------------------------------------- -# ----------------------------------------------------------------------------------------------- -# Controlled simulation of the failover scenario — validates the entire failover lifecycle +# ============================================================================================== +# ================================= Failover Test ============================================== +# ============================================================================================== +# Controlled simulation of the failover lifecycle — validates the entire failover sequence # without waiting for a real outage. # -# This script is a TEST HARNESS only — it does not contain failover logic. -# All failover logic lives in failover.sh and is called directly from here. -# Any changes to failover.sh are automatically reflected in this test. +# ── WHAT THIS SCRIPT IS ─────────────────────────────────────────────────────────────────────── +# A test harness only — contains no failover logic. +# All failover logic lives in failover.sh and is exercised by this test. +# Any changes to failover.sh are automatically reflected here. # -# Test sequence: -# 1. Pre-flight — verify both servers reachable, failover.sh exists, state is NORMAL -# 2. Block — add iptables rule dropping all traffic to remote IP -# 3. Detect — run failover.sh one cycle — confirm FAILOVER state detected -# 4. Start — verify failover containers started locally -# 5. Restore — remove iptables rule, remote becomes reachable again -# 6. Handback — wait for failover.sh to confirm handback strikes and hand back -# 7. Verify — confirm containers returned to remote, local copies stopped -# 8. Report — full pass/fail summary per phase +# ── TEST SEQUENCE ───────────────────────────────────────────────────────────────────────────── +# Phase 1 — Pre-flight verify both servers reachable, daemons healthy, +# version parity, failover.sh exists, state is NORMAL +# Phase 2 — Block Remote iptables rule drops all traffic to remote IP +# Phase 3 — Failover Detection wait for failover.sh to detect outage and enter FAILOVER +# Phase 4 — Container Start verify Tier 1 failover containers started locally +# Phase 5 — Restore remove iptables rule, remote becomes reachable +# Phase 6 — Handback wait for failover.sh to complete handback to NORMAL +# Phase 7 — Container Handback verify Tier 1 containers stopped locally after handback +# Phase 8 — Report full pass/fail summary per phase # -# Safety: iptables rule is removed via trap on ANY exit — crash, error, ctrl-c, or normal. -# Remote connectivity is always restored regardless of test outcome. +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# FAILOVER_ENABLED gate — aborts if failover monitoring is disabled +# iptables safety trap — rule ALWAYS removed on exit (crash, error, ctrl-c, normal) +# remote connectivity always restored regardless of outcome +# Version parity check — pre-flight verifies both servers on compatible unRAID versions +# Remote Docker daemon — pre-flight verifies remote daemon is responsive +# DOCKER_TIMEOUT — all docker calls protected against daemon hangs +# MY_ID-based routing — tier containers selected via MY_ID not hostname comparison +# Command validation — iptables and notify validated before use +# Dry-run safe — full sequence walkthrough without touching iptables or containers # -# ⚠️ This script starts and stops real containers on both servers. +# ── WARNING ─────────────────────────────────────────────────────────────────────────────────── +# ⚠️ This script starts and stops REAL containers on both servers. # Run during a maintenance window — users will experience a brief service interruption. -# Use --dry-run to walk through the sequence without touching containers or iptables. +# Use --dry-run to walk through the sequence without any real changes. # -# All configuration in Master.conf under Failover and Failover Test sections. -# ----------------------------------------------------------------------------------------------- +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# FAILOVER_TEST_BLOCK_WAIT — seconds to wait for failover.sh to detect outage +# FAILOVER_TEST_HANDBACK_WAIT — seconds to wait for failover.sh to complete handback +# FAILOVER_CHECK_INTERVAL — check interval of the running failover.sh (informational) +# FAILOVER_HANDBACK_STRIKES — strikes required before handback (informational) +# FAILOVER_STATE_FILE — state file path to read current state +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# failover_test.sh — run full test sequence +# failover_test.sh --dry-run — walk through all phases without changes +# failover_test.sh --status — show current failover state and test config +# failover_test.sh --log — verbose output +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" FAILOVER_SCRIPT="$SCRIPT_DIR/failover.sh" +DOCKER_TIMEOUT=15 + +# ============================================================================================== +# ── SAFETY TRAP — always remove iptables rule on exit ───────────────────────────────────────── +# ============================================================================================== +# Fires on normal exit, error exit, ctrl-c, and script crashes. +# Remote connectivity is ALWAYS restored regardless of test outcome. -# ----------------------------------------------------------------------------------------------- -# SAFETY TRAP — always remove iptables rule on exit -# Fires on normal exit, error exit, ctrl-c, and script crashes -# ----------------------------------------------------------------------------------------------- IPTABLES_RULE_ACTIVE=false cleanup() { @@ -51,7 +75,7 @@ cleanup() { if [[ "$DRY_RUN" == false ]]; then iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null IPTABLES_RULE_ACTIVE=false - success "iptables rule removed — remote connectivity restored" + warn "iptables rule removed — remote connectivity restored" else warn "DRY RUN — would remove iptables rule" fi @@ -60,9 +84,9 @@ cleanup() { trap cleanup EXIT -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" @@ -71,93 +95,127 @@ if [[ "$EUID" -ne 0 ]]; then exit 1 fi -success "Running as root" - -if ! command -v iptables >/dev/null 2>&1; then - error "iptables not found — required for connectivity simulation" - exit 1 +# FAILOVER_ENABLED gate — no point testing if failover is disabled +if [[ "${FAILOVER_ENABLED:-false}" == false ]]; then + warn "FAILOVER_ENABLED=false — failover test aborted" + warn "Enable failover in master.conf before running this test" + exit 0 fi -success "iptables available" +acquire_lock # strict single instance — modifies iptables and containers + +detect_hosts +resolve_remote_ip + +# Validate commands used by this script +validate_unraid_cmd \ + "$(which iptables 2>/dev/null || echo /sbin/iptables)" \ + "--version" "iptables" \ + "iptables" || { error "iptables not found — required for connectivity simulation"; exit 1; } + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" if [[ ! -f "$FAILOVER_SCRIPT" ]]; then error "failover.sh not found at $FAILOVER_SCRIPT" exit 1 fi - -success "failover.sh found" - -detect_hosts -resolve_remote_ip +log "failover.sh found at $FAILOVER_SCRIPT" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no iptables rules or container changes will be made" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then + local_ver=$(grep -oP '(?<=version=")[^"]+' /etc/unraid-version 2>/dev/null || echo "unknown") + echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_HOST Local: $LOCAL_SERVER_NAME" - echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)" + echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_HOST Remote ID: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)" + echo "$ICON_GEAR unRAID ver: $local_ver" echo "$ICON_FAILOVER Block wait: ${FAILOVER_TEST_BLOCK_WAIT}s" echo "$ICON_FAILOVER Handback wait: ${FAILOVER_TEST_HANDBACK_WAIT}s" + echo "$ICON_FAILOVER Check interval: ${FAILOVER_CHECK_INTERVAL}s" + echo "$ICON_FAILOVER Handback strikes: ${FAILOVER_HANDBACK_STRIKES}" echo "$ICON_GEAR Dry Run: $DRY_RUN" - # Current failover state if [[ -f "$FAILOVER_STATE_FILE" ]]; then CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) echo "$ICON_FAILOVER Current state: ${CURRENT_STATE:-unknown}" else echo "$ICON_FAILOVER Current state: no state file" fi + + # Show Tier 1 containers for this host + TIER1_VAR="FAILOVER_${MY_ID}_RUNS_FOR_${REMOTE_ID}_TIER1" + eval "TIER1_CONTAINERS=(\"\${${TIER1_VAR}[@]:-}\")" + echo "$ICON_CONTAINERS Tier 1 to test: ${TIER1_CONTAINERS[*]:-none configured}" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -# ----------------------------------------------------------------------------------------------- -# PHASE TRACKING -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ── PHASE TRACKING ──────────────────────────────────────────────────────────────────────────── +# ============================================================================================== PHASES_PASS=() PHASES_FAIL=() TOTAL_START=$(date +%s) -phase_pass() { PHASES_PASS+=("$1"); success "$ICON_DONE Phase: $1 — PASSED"; } -phase_fail() { PHASES_FAIL+=("$1"); error "$ICON_ERROR Phase: $1 — FAILED"; } +phase_pass() { PHASES_PASS+=("$1"); warn "$ICON_DONE Phase: $1 — PASSED ✅"; } +phase_fail() { PHASES_FAIL+=("$1"); error "Phase: $1 — FAILED ❌"; } -# ----------------------------------------------------------------------------------------------- -# ━━━ PHASE 1 — Pre-flight ━━━ -# ----------------------------------------------------------------------------------------------- +# Get Tier 1 containers for this server's failover responsibility +TIER1_VAR="FAILOVER_${MY_ID}_RUNS_FOR_${REMOTE_ID}_TIER1" +eval "TIER1_CONTAINERS=(\"\${${TIER1_VAR}[@]:-}\")" + +# ============================================================================================== +# ━━━ Phase 1 — Pre-flight ━━━ +# ============================================================================================== echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo " $ICON_SHIELD FAILOVER TEST — $(date '+%Y-%m-%d %H:%M:%S')" -echo " $ICON_HOST Local: $LOCAL_SERVER_NAME" -echo " $ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)" +echo " $ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → $REMOTE_ID ($REMOTE_SERVER_NAME)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" echo "━━━ $ICON_SHIELD Phase 1 — Pre-flight ━━━" -# Check remote reachable -info "Checking remote reachability..." +# Remote reachable if ping_remote; then - success "Remote $REMOTE_SERVER_NAME is reachable" + log "$REMOTE_SERVER_NAME is reachable" else - error "Remote $REMOTE_SERVER_NAME is not reachable — cannot run test" + error "$REMOTE_SERVER_NAME is not reachable — cannot run test" phase_fail "Pre-flight" exit 1 fi -# Check internet reachable -info "Checking internet connectivity..." +# Internet reachable if ping_internet; then - success "Internet is reachable" + log "Internet is reachable" else error "No internet connectivity — cannot run test" phase_fail "Pre-flight" exit 1 fi -# Check current failover state is NORMAL +# Version parity — test may produce misleading results on mismatch +if ! check_unraid_version_parity; then + error "unRAID version mismatch — test aborted to prevent misleading results" + phase_fail "Pre-flight" + exit 1 +fi + +# Remote Docker daemon — must be responsive before test manipulates containers +if ! check_remote_docker_daemon; then + error "Remote Docker daemon not responsive — cannot run test" + phase_fail "Pre-flight" + exit 1 +fi + +# Failover state must be NORMAL before test if [[ -f "$FAILOVER_STATE_FILE" ]]; then CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) if [[ "$CURRENT_STATE" != "NORMAL" ]]; then @@ -165,28 +223,37 @@ if [[ -f "$FAILOVER_STATE_FILE" ]]; then phase_fail "Pre-flight" exit 1 fi - success "Failover state is NORMAL" + log "Failover state is NORMAL" else warn "No state file found — assuming NORMAL (first run)" fi +# Tier 1 containers configured +if [[ ${#TIER1_CONTAINERS[@]} -eq 0 ]]; then + error "No Tier 1 containers configured for $MY_ID → $REMOTE_ID" + error "Check FAILOVER_${MY_ID}_RUNS_FOR_${REMOTE_ID}_TIER1 in master_host*.conf" + phase_fail "Pre-flight" + exit 1 +fi +log "Tier 1 containers: ${TIER1_CONTAINERS[*]}" + phase_pass "Pre-flight" -# ----------------------------------------------------------------------------------------------- -# ━━━ PHASE 2 — Block Remote ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Phase 2 — Block Remote Connectivity ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_PING Phase 2 — Block Remote Connectivity ━━━" -warn "Adding iptables rule — dropping all traffic to $REMOTE_SERVER" +warn "Adding iptables rule — dropping all traffic to $REMOTE_SERVER ($REMOTE_SERVER_NAME)" if [[ "$DRY_RUN" == false ]]; then iptables -I OUTPUT -d "$REMOTE_SERVER" -j DROP IPTABLES_RULE_ACTIVE=true - success "iptables rule active — $REMOTE_SERVER_NAME appears unreachable" # Verify block is working + sleep 2 if ! ping -c1 -W2 "$REMOTE_SERVER" &>/dev/null; then - success "Connectivity block confirmed — ping to remote fails as expected" + log "Connectivity block confirmed — ping to remote fails as expected" phase_pass "Block Remote" else error "iptables rule did not block connectivity — ping still succeeds" @@ -194,30 +261,29 @@ if [[ "$DRY_RUN" == false ]]; then exit 1 fi else - warn "DRY RUN — would block $REMOTE_SERVER with iptables" + warn "DRY RUN — would block $REMOTE_SERVER with iptables DROP rule" phase_pass "Block Remote" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ PHASE 3 — Failover Detection ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Phase 3 — Failover Detection ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_FAILOVER Phase 3 — Failover Detection ━━━" -info "Waiting ${FAILOVER_TEST_BLOCK_WAIT}s for failover.sh to detect outage..." -info "failover.sh check interval is ${FAILOVER_CHECK_INTERVAL}s" +warn "Waiting ${FAILOVER_TEST_BLOCK_WAIT}s for failover.sh to detect outage..." +log "failover.sh check interval: ${FAILOVER_CHECK_INTERVAL}s" if [[ "$DRY_RUN" == false ]]; then sleep "$FAILOVER_TEST_BLOCK_WAIT" - # Check state file updated to FAILOVER if [[ -f "$FAILOVER_STATE_FILE" ]]; then NEW_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) if [[ "$NEW_STATE" == "FAILOVER" ]]; then - success "State changed to FAILOVER — outage detected correctly" + log "State changed to FAILOVER — outage detected correctly ✅" phase_pass "Failover Detection" else error "State is $NEW_STATE — expected FAILOVER after ${FAILOVER_TEST_BLOCK_WAIT}s" - warn "failover.sh may not be running — check User Scripts plugin" + warn "Is failover.sh running? Check User Scripts plugin" phase_fail "Failover Detection" fi else @@ -225,30 +291,25 @@ if [[ "$DRY_RUN" == false ]]; then phase_fail "Failover Detection" fi else - warn "DRY RUN — would wait ${FAILOVER_TEST_BLOCK_WAIT}s and check for FAILOVER state" + warn "DRY RUN — would wait ${FAILOVER_TEST_BLOCK_WAIT}s then check for FAILOVER state" phase_pass "Failover Detection" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ PHASE 4 — Container Start Verification ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Phase 4 — Container Start Verification ━━━ +# ============================================================================================== echo "" -echo "━━━ $ICON_CONTAINERS Phase 4 — Failover Containers Started ━━━" - -# Determine which containers should have started on this host -if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - EXPECTED_CONTAINERS=("${FAILOVER_HOST1_STARTS_FOR_HOST2[@]}") -else - EXPECTED_CONTAINERS=("${FAILOVER_HOST2_STARTS_FOR_HOST1[@]}") -fi +echo "━━━ $ICON_CONTAINERS Phase 4 — Tier 1 Containers Started Locally ━━━" +log "Checking Tier 1 containers: ${TIER1_CONTAINERS[*]}" if [[ "$DRY_RUN" == false ]]; then CONTAINERS_OK=true - for container in "${EXPECTED_CONTAINERS[@]}"; do + for container in "${TIER1_CONTAINERS[@]}"; do [[ -z "$container" ]] && continue - STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \ + "$container" 2>/dev/null) if [[ "$STATUS" == "true" ]]; then - success "$ICON_RUNNING $container is running locally" + log "$ICON_RUNNING $container is running locally ✅" else error "$ICON_NOT_RUNNING $container is NOT running locally" CONTAINERS_OK=false @@ -261,29 +322,27 @@ if [[ "$DRY_RUN" == false ]]; then phase_fail "Container Start" fi else - warn "DRY RUN — would verify these containers started: ${EXPECTED_CONTAINERS[*]}" + warn "DRY RUN — would verify these Tier 1 containers started: ${TIER1_CONTAINERS[*]}" phase_pass "Container Start" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ PHASE 5 — Restore Connectivity ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Phase 5 — Restore Remote Connectivity ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_PING Phase 5 — Restore Remote Connectivity ━━━" -info "Removing iptables block — remote becomes reachable again" +warn "Removing iptables block — $REMOTE_SERVER_NAME becomes reachable again" if [[ "$DRY_RUN" == false ]]; then iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null IPTABLES_RULE_ACTIVE=false - success "iptables rule removed" - # Verify connectivity restored sleep 3 if ping_remote; then - success "Remote $REMOTE_SERVER_NAME is reachable again" + log "$REMOTE_SERVER_NAME is reachable again ✅" phase_pass "Restore Connectivity" else - error "Remote still unreachable after removing iptables rule" + error "$REMOTE_SERVER_NAME still unreachable after removing iptables rule" phase_fail "Restore Connectivity" fi else @@ -291,14 +350,14 @@ else phase_pass "Restore Connectivity" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ PHASE 6 — Handback ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Phase 6 — Handback ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_FAILOVER Phase 6 — Handback ━━━" -info "Waiting ${FAILOVER_TEST_HANDBACK_WAIT}s for failover.sh to confirm handback..." -info "Requires $FAILOVER_HANDBACK_STRIKES consecutive remote-up checks at ${FAILOVER_CHECK_INTERVAL}s intervals" -info "Estimated minimum wait: $(( FAILOVER_HANDBACK_STRIKES * FAILOVER_CHECK_INTERVAL ))s" +warn "Waiting ${FAILOVER_TEST_HANDBACK_WAIT}s for failover.sh to complete handback..." +log "Requires $FAILOVER_HANDBACK_STRIKES consecutive checks at ${FAILOVER_CHECK_INTERVAL}s" +log "Minimum handback time: $(( FAILOVER_HANDBACK_STRIKES * FAILOVER_CHECK_INTERVAL ))s" if [[ "$DRY_RUN" == false ]]; then sleep "$FAILOVER_TEST_HANDBACK_WAIT" @@ -306,10 +365,11 @@ if [[ "$DRY_RUN" == false ]]; then if [[ -f "$FAILOVER_STATE_FILE" ]]; then FINAL_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) if [[ "$FINAL_STATE" == "NORMAL" ]]; then - success "State returned to NORMAL — handback completed" + log "State returned to NORMAL — handback completed ✅" phase_pass "Handback" else - error "State is $FINAL_STATE — expected NORMAL after handback wait" + error "State is $FINAL_STATE — expected NORMAL after ${FAILOVER_TEST_HANDBACK_WAIT}s" + warn "Handback may still be in progress — check failover.sh output" phase_fail "Handback" fi else @@ -317,23 +377,25 @@ if [[ "$DRY_RUN" == false ]]; then phase_fail "Handback" fi else - warn "DRY RUN — would wait ${FAILOVER_TEST_HANDBACK_WAIT}s and verify NORMAL state" + warn "DRY RUN — would wait ${FAILOVER_TEST_HANDBACK_WAIT}s then verify NORMAL state" phase_pass "Handback" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ PHASE 7 — Container Handback Verification ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Phase 7 — Container Handback Verification ━━━ +# ============================================================================================== echo "" -echo "━━━ $ICON_CONTAINERS Phase 7 — Failover Containers Stopped Locally ━━━" +echo "━━━ $ICON_CONTAINERS Phase 7 — Tier 1 Containers Stopped Locally ━━━" +log "Verifying Tier 1 containers returned to $REMOTE_SERVER_NAME" if [[ "$DRY_RUN" == false ]]; then HANDBACK_OK=true - for container in "${EXPECTED_CONTAINERS[@]}"; do + for container in "${TIER1_CONTAINERS[@]}"; do [[ -z "$container" ]] && continue - STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \ + "$container" 2>/dev/null) if [[ "$STATUS" != "true" ]]; then - success "$ICON_NOT_RUNNING $container stopped locally — handed back" + log "$ICON_NOT_RUNNING $container stopped locally — handed back ✅" else error "$ICON_RUNNING $container still running locally — handback may have failed" HANDBACK_OK=false @@ -346,20 +408,20 @@ if [[ "$DRY_RUN" == false ]]; then phase_fail "Container Handback" fi else - warn "DRY RUN — would verify failover containers stopped locally after handback" + warn "DRY RUN — would verify Tier 1 containers stopped locally after handback" phase_pass "Container Handback" fi TOTAL_END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Test Report ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Test Report ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY FAILOVER TEST REPORT ━━━━━" -echo "$ICON_HOST Local: $LOCAL_SERVER_NAME" -echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME" -echo "$ICON_TIME Duration: $(format_duration $((TOTAL_END - TOTAL_START)))" +echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_HOST Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)" +echo "$ICON_TIME Duration: $(format_duration $((TOTAL_END - TOTAL_START)))" echo "" echo " Phase Results:" for phase in "${PHASES_PASS[@]}"; do @@ -375,13 +437,15 @@ FAIL_COUNT=${#PHASES_FAIL[@]} TOTAL_PHASES=$(( PASS_COUNT + FAIL_COUNT )) if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no changes made" + warn "DRY RUN — no changes made" elif [[ "$FAIL_COUNT" -eq 0 ]]; then - echo "$ICON_DONE Status: $ICON_SUCCESS ALL $TOTAL_PHASES PHASES PASSED" - notify "Failover test PASSED on $(hostname) — all $TOTAL_PHASES phases completed successfully" "Failover Test" "normal" + warn "$ICON_DONE ALL $TOTAL_PHASES PHASES PASSED" + notify "Failover test PASSED on $(hostname) — all $TOTAL_PHASES phases completed" \ + "Failover Test" "normal" else - echo "$ICON_ERROR Status: $FAIL_COUNT/$TOTAL_PHASES PHASES FAILED" - notify "Failover test FAILED on $(hostname) — $FAIL_COUNT/$TOTAL_PHASES phases failed: ${PHASES_FAIL[*]}" "Failover Test" "warning" + error "$FAIL_COUNT/$TOTAL_PHASES PHASES FAILED" + notify "Failover test FAILED on $(hostname) — $FAIL_COUNT/$TOTAL_PHASES phases failed: ${PHASES_FAIL[*]}" \ + "Failover Test" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" diff --git a/Media/README-Media.md b/Media/README-Media.md index ff6e745..ca94ff1 100644 --- a/Media/README-Media.md +++ b/Media/README-Media.md @@ -1,316 +1,970 @@ -# Media +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# 🎬 MEDIA +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Scripts that maintain the health, cleanliness, and consistency of your media library. Permissions, junk file removal, and orphaned file cleanup across Lidarr, Sonarr, and Radarr. +**Library health, cleanliness, and consistency for a multi-server arr stack.** +Correct permissions so arrs can do their job. Junk file removal so orphan detection +is not confused by scene debris. Orphaned file cleanup against live arr APIs so +deleted shows and removed albums actually leave the disk. Emby notified automatically +so users never see ghost library entries. -These scripts are run sequentially by `Orchestrators/media_management.sh` — not individually scheduled. The orchestrator handles ordering, pass/fail tracking, and the combined summary. +> **These scripts permanently delete files.** The arr cleanup scripts are protected +> by multiple safety layers that must all pass before anything is touched — but dry +> runs and log review are still the right first step on any new system or after any +> configuration change. The testing procedure at the end of this document exists +> for a reason. --- -## Why Order Matters - -``` -1. media_shares_permissions.sh ← permissions first -2. media_cleaner.sh anime ← clean junk before arr scripts scan -3. media_cleaner.sh media ← same -4. lidarr_cleanup.sh ← arr cleanup last -5. sonarr_cleanup.sh -6. radarr_cleanup.sh -``` - -**Permissions before everything else** — arr cleanup scripts need correct ownership to delete files. If a file is owned by root and the script runs as nobody, the delete fails silently. - -**Cleaner before arr cleanup** — junk files (.sfv, .rar, .txt etc.) mixed in with media files create noise in the orphan detection logic. Clean the junk first so arr cleanup only deals with real media files. - -**Arr cleanup last** — depends on clean folders and correct permissions to work reliably. +## ━━━ THE PROBLEM THAT BUILT THIS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ --- -## Scripts +### 🔴 Files Owned by Root That Arrs Can't Touch -### `media_shares_permissions.sh` +Download clients running without explicit PUID/PGID configuration write files owned +by root. Arrs running as `nobody:users` cannot rename, move, or delete those files. +Import fails. Upgrade attempts fail. Deletion requests from arr UI do nothing. +The file sits on disk permanently in a state where the arr thinks it's managed but +can't actually manage it. + +The failure mode is subtle — arr shows the file as imported, everything looks correct +in the UI, but the file is untouchable by the processes that need to touch it. You +only discover this when an upgrade is requested and the old version refuses to delete. + +The fix: `media_shares_permissions.sh` applies `nobody:users` ownership and correct +permissions to every file across every configured share, daily. Even if a container +is misconfigured, even if a manual admin copy created root-owned files, even after an +rsync that didn't preserve ownership — the next daily run normalises everything. + +--- + +### 🔴 Scene Junk Files Confusing Orphan Detection + +Scene releases — and many download groups for anime in particular — include a +constellation of junk files alongside the actual media: `.sfv` checksum files, `.nfo` +scene info, `.rar` source archives, `.sample` preview files, `.url` group websites. +After extraction and import these are worthless. They sit in the media folder forever. + +The problem for arr cleanup: these files are not tracked by any arr. They are not +music files, TV files, or movie files. They look like orphans — but classifying them +as orphans means the orphan detection logic is constantly processing noise. Worse, +some cleanup implementations would try to delete them and log every single one as +a "cleaned orphan," making the output unreadable for finding actual orphaned media. + +The fix: `media_cleaner.sh` runs before any arr cleanup script and removes all +known junk patterns first. By the time arr cleanup runs, every file it finds that +isn't tracked by the arr is actually a media file that shouldn't be there — not +a `.sfv` that was never tracked by anyone. + +--- + +### 🔴 Deleted Shows and Removed Albums Still on Disk + +When you remove a series from Sonarr, Sonarr deletes its database entries and — if +configured — sends delete commands for the files. When that fails (permission issue, +container wasn't running, file was on a different path than Sonarr expected), the +files stay on disk permanently. Sonarr has forgotten about them. Radarr has forgotten +about the movie. Lidarr has forgotten about the album. The files just sit there, +consuming space, invisible to the arr that should manage them. + +Over years on an active library this accumulates. A drive that should have 2TB free +has 400GB of content that no arr knows about. + +The fix: arr cleanup scripts that query the live API for every tracked file path, walk +the disk, and identify files that exist on disk but are absent from the API response. +These are genuine orphans — content the arr has definitively moved on from. With a +configurable age threshold to avoid deleting files that are mid-import, and a maximum +deletion size gate to catch configuration errors before they become disasters. + +--- + +### 🔴 Emby Showing Ghost Entries After Cleanup + +After arr cleanup deletes files, Emby's library still shows them — ghost entries that +produce "file not found" errors when clicked. Emby's library scan runs on its own +schedule, which might not happen for hours. Users see broken entries in the library +until the next scheduled scan. + +The fix: `notify_emby_scan()` — called automatically after every arr cleanup deletion. +Triggers Emby's "Clean Missing Files" task immediately. By the time the user refreshes +the library, the ghost entries are already gone. + +--- + +### 🔴 No Safety Net on Deletion Size + +A misconfigured root path — pointing arr cleanup at the wrong directory, or a root +path mismatch between master.conf and the arr's own settings — means the API returns +zero tracked files for a root that actually contains thousands. Every file walks as +an orphan. Everything gets deleted. A library is gone. + +This is the catastrophic failure mode. It happened during development — once, with a +test library. It cannot happen again. + +The fix: `LIDARR_MAX_DELETE_GB`, `SONARR_MAX_DELETE_GB`, `RADARR_MAX_DELETE_GB`. +If the total size of files queued for deletion exceeds the configured limit, the script +stops and requires `--i-know-what-im-doing` to proceed. The flag name is long and +annoying by design — it cannot be added by accident and it cannot be forgotten +what it means. + +--- + +## ━━━ WHY ORDER MATTERS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +These scripts are called in sequence by `Orchestrators/media_management.sh`. +The order is not arbitrary — each script depends on the previous ones having run. + +```bash +# master.conf — MEDIA_MAINTENANCE_JOBS defines the order +# ───────────────────────────────────────────────────────────────────────────── +MEDIA_MAINTENANCE_JOBS=( + "Media/media_shares_permissions.sh" # 1. Permissions — always first + "Media/media_cleaner.sh anime" # 2. Junk removal — before orphan scan + "Media/media_cleaner.sh media" # 3. Same for media shares + "Media/lidarr_cleanup.sh" # 4. Arr cleanup — after permissions + clean + "Media/sonarr_cleanup.sh" # 5. + "Media/radarr_cleanup.sh" # 6. +) +``` + +**Permissions before everything else:** +Arr cleanup needs `nobody:users` ownership to delete files. If a file is owned by +`root:root` and the script runs as `nobody`, the deletion fails silently — the file +looks like it was processed but is still on disk. Permissions first ensures this +can't happen. + +**Junk removal before arr cleanup:** +The arr cleanup scripts walk the disk and compare against the arr API. Junk files +(`.sfv`, `.rar`, `.nfo`) are not tracked by any arr — they look like orphans. Removing +them first means the orphan detection only encounters actual media files. Cleaner +output, more accurate detection, less noise. + +**Arr cleanup last:** +Depends on clean folders and correct ownership. Both are guaranteed by the time +arr cleanup runs. + +--- + +## ━━━ HOST AWARENESS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Each script runs on both servers but only operates on the shares and arrs that belong +to that server. `detect_hosts()` aliases all `HOST*_` prefixed vars to their +unprefixed names. + +``` +HOST1 — source of truth for: HOST2 — source of truth for: + Movies (Radarr) Anime_Movies (his Radarr) + Tv_Shows (Sonarr) Anime_Shows (his Sonarr) + Music (Lidarr) + +Permissions and cleaner scripts run on both servers against their own shares. +Arr cleanup scripts check MY_ID and only run against the arr they own. +Lidarr only runs on HOST1 — exits cleanly on HOST2 with no action. +``` + +No manual `HOST1`/`HOST2` comparisons exist in any of these scripts. Everything +routes through `MY_ID`. + +--- + +## ━━━ SCRIPTS AT A GLANCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +| Script | Purpose | When | +|--------|---------|------| +| `media_shares_permissions.sh` | Apply correct ownership + permissions to all media shares | Daily via `media_management.sh` | +| `media_cleaner.sh anime` | Remove junk files from anime share folders | Daily via `media_management.sh` | +| `media_cleaner.sh media` | Remove junk files from media share folders | Daily via `media_management.sh` | +| `lidarr_cleanup.sh` | Delete orphaned music files not tracked by Lidarr | Daily via `media_management.sh` | +| `sonarr_cleanup.sh` | Delete orphaned TV files not tracked by Sonarr | Daily via `media_management.sh` | +| `radarr_cleanup.sh` | Delete orphaned movie files not tracked by Radarr | Daily via `media_management.sh` | + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔐 media_shares_permissions.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ Applies correct ownership and permissions recursively to all configured media shares. - -```bash -# Called by media_management.sh — not scheduled directly -# Run manually when needed: -/mnt/user/appdata/unraid_scripts/Media/media_shares_permissions.sh -/mnt/user/appdata/unraid_scripts/Media/media_shares_permissions.sh --dry-run -``` - -**What it applies:** - -```bash -PERMISSIONS_MODE="777" # chmod applied recursively -PERMISSIONS_OWNER="nobody:users" # chown applied recursively -``` - -`777` and `nobody:users` is the standard for unRAID media shares accessible by Docker containers. All media server containers (Emby, Tdarr, arr stack) run as `nobody:users` — this ensures they can read, write, and delete files without permission errors. - -**Why recursive takes time:** - -On a large library with millions of files this can run for 20-30 minutes. This is expected and normal. Run it overnight via the orchestrator — not during peak usage hours. - -**Add or remove shares** in `Master.conf` under `MEDIA_PERMISSION_SHARES`. The script reads this list at runtime — no script changes needed. +The first job in the daily maintenance sequence — arr cleanup depends on this running +first. A failsafe that runs daily whether or not anything appears to need fixing. --- -### `media_cleaner.sh` - -Removes junk files from media shares using configurable file pattern lists. Two profiles with independent folder lists and patterns. +### ── The Permissions Model ─────────────────────────────────────────────────── ```bash -# Called by media_management.sh with profile argument -# Run manually with profile required: -/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh anime -/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh media -/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh anime --dry-run -/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh media --dry-run +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# unRAID's standard for Docker-accessible media shares. +# All arr containers, Emby, Tdarr, and download clients use nobody:users. +# These values match the PUID=99 PGID=100 container environment variables. +# +# Directories: 755 nobody:users +# Owner (nobody) — rwx enter, list, create files +# Group (users) — r-x enter and list +# Others — r-x Samba guests can browse +# No world-write — prevents accidental deletion by unauthenticated access +# +# Files: 664 nobody:users +# Owner (nobody) — rw read + write +# Group (users) — rw arrs can import, rename, delete +# Others — r Samba guests can read +# No execute bit — media files are never executable +# + PERMISSIONS_DIR_MODE="755" + PERMISSIONS_FILE_MODE="664" + PERMISSIONS_OWNER="nobody:users" ``` -**Always --dry-run first** — especially on first use or after adding new patterns. - -#### Anime Profile - -```bash -ANIME_CLEAN_FOLDERS=( - /mnt/user/Anime_Movies - /mnt/user/Anime_Movies-Old - /mnt/user/Anime_Shows - /mnt/user/Anime_Shows-Old -) - -ANIME_FILE_PATTERNS=( - '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' - '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' - '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp' - '*.log' '*.json' -) -``` - -Anime downloads from groups commonly include verification files (`.sfv`, `.md5`), RAR archives after extraction, proof files, and samples. These are safe to delete after the video files have been imported. - -#### Media Profile - -```bash -MEDIA_CLEAN_FOLDERS=( - /mnt/user/Kids_Movies - /mnt/user/Kids_Tv_Shows - /mnt/user/Movies - /mnt/user/Music - /mnt/user/Sports - /mnt/user/stand-up_comedy - /mnt/user/Tv_Shows -) -``` - -The media profile includes `*.iso` and `*.lrc` in addition to the anime patterns — ISO disc images after ripping and lyric files that aren't needed in the media folders. - -#### Adding New Patterns - -Add to the appropriate array in `Master.conf` — no script changes needed: - -```bash -ANIME_FILE_PATTERNS=( - '*.sfv' '*.md5' ... - '*.new-pattern' # ← just add here -) -``` - -#### Safety Note - -The cleaner deletes by pattern — it does not check what arr thinks about the files. It runs before arr cleanup specifically so arr cleanup sees clean folders. Do not add patterns that match media files you want to keep (`.mkv`, `.mp4` etc.). +**Two separate passes — not a single recursive chmod:** +Directories need the execute bit (to enter them). Files must never have the execute +bit. A single `chmod -R 664` would remove execute from directories, making them +inaccessible. The script runs `find -type d` and `find -type f` separately with the +correct mode for each. --- -### `lidarr_cleanup.sh` +### ── Why It Runs Daily Even When Things Look Fine ──────────────────────────── + +```bash +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# Ownership drift happens from multiple sources, not just misconfigured containers: +# +# rsync without --chown → brings source server's nobody:users correctly +# but a misconfigured rsync brings root:root +# Manual admin copies → scp and cp default to root:root +# New containers → default to root before PUID/PGID is set +# unRAID updates → can reset container environment variables +# Download client restarts → if PUID/PGID lost on restart, next download is root +# +# Running daily means the window between "wrong ownership created" and +# "arr fails to manage the file" is at most 24 hours. Usually it is overnight. +# +# When this script corrects zero files per run, everything is configured correctly. +# When it corrects many files per run, a container has wrong PUID/PGID. +# Check these first: SABnzbd, qBittorrent, slskd — most common culprits. +# Correct values: PUID=99 PGID=100 in Docker template environment variables. +# +HOST1_MEDIA_PERMISSION_SHARES=( + "/mnt/user/Movies" + "/mnt/user/Tv_Shows" + "/mnt/user/Music" + "/mnt/user/Kids_Movies" + "/mnt/user/Kids_Tv_Shows" + "/mnt/user/Sports" + "/mnt/user/stand-up_comedy" +) +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Normal — called by media_management.sh, rarely run directly. +# Safe to run manually at any time — idempotent, only changes what's wrong. +# ───────────────────────────────────────────────────────────────────────────── +media_shares_permissions.sh + +# ───────────────────────────────────────────────────────────────────────────── +# Dry run — show how many files would be corrected per share. +# If the number is unexpectedly large, check container PUID/PGID settings first. +# ───────────────────────────────────────────────────────────────────────────── +media_shares_permissions.sh --dry-run + +# ───────────────────────────────────────────────────────────────────────────── +# Status — show configured share list and current ownership state. +# ───────────────────────────────────────────────────────────────────────────── +media_shares_permissions.sh --status + +# Verbose — show ownership correction count per share +media_shares_permissions.sh --log +``` + +> **On large libraries this runs for 20-30 minutes.** Millions of files with a +> recursive walk and chown/chmod on each takes time. This is expected and normal — +> it is why it runs overnight in the maintenance window, not during peak hours. + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🧹 media_cleaner.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Removes junk files from media share folders using configurable pattern lists. +Two profiles — `anime` and `media` — each with their own folder list and file patterns. +Runs after `media_shares_permissions.sh` and before any arr cleanup script. + +--- + +### ── What Gets Removed ──────────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# These patterns appear in anime releases and general media downloads. +# None of these are content you want in your library after extraction and import. +# +ANIME_FILE_PATTERNS=( + "*.sfv" # checksum verification — useless after download verified + "*.md5" "*.sha1" # other checksum formats — same reason + "*.nfo" # scene info file — group name, release notes, not metadata + "*.url" "*.lnk" # website shortcuts — never needed + "*.rar" "*.zip" # source archives — kept by some clients after extraction + "*.info" # tool output files + "*.torrent" # torrent descriptor left by some clients + "*.sample*" # scene preview clip — never needed after import + "*.proof*" # screenshot proving encode quality — never needed + "*sync-conflict*" # Syncthing conflict copies — should not be in media folders + "*.scr" "*.exe" # executables — should NEVER be in a media folder + "*.srr" # scene recovery record — useless post-download + "*.log" # tool/client logs + "*.json" # metadata or tool output +) + +# Media profile has additional patterns not relevant for anime: +MEDIA_FILE_PATTERNS=( + "${ANIME_FILE_PATTERNS[@]}" # all anime patterns plus: + "*.iso" # disc images after ripping — keep the ripped version + "*.lrc" # lyric files in media folders — not needed here +) +``` + +--- + +### ── Two Profiles — Different Folders ─────────────────────────────────────── + +```bash +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# Profiles map to different share sets on each server. +# HOST1 owns Movies/Shows/Music. HOST2 owns Anime_Movies/Anime_Shows. +# detect_hosts() ensures each server only cleans the shares it owns. +# +HOST1_ANIME_CLEAN_FOLDERS=( + # HOST1 mirrors anime FROM HOST2 — clean the mirrored copies + "/mnt/user/Anime_Movies" + "/mnt/user/Anime_Movies-Old" + "/mnt/user/Anime_Shows" + "/mnt/user/Anime_Shows-Old" +) + +HOST1_MEDIA_CLEAN_FOLDERS=( + # HOST1's own shares — source of truth for these + "/mnt/user/Kids_Movies" + "/mnt/user/Kids_Tv_Shows" + "/mnt/user/Movies" + "/mnt/user/Music" + "/mnt/user/Sports" + "/mnt/user/stand-up_comedy" + "/mnt/user/Tv_Shows" +) +``` + +--- + +### ── Safety Note ───────────────────────────────────────────────────────────── + +``` +The cleaner deletes by pattern — it does not check what any arr thinks about +the file. It runs before arr cleanup specifically so arr cleanup sees clean folders. + +DO NOT add patterns that match media you want to keep: + *.mkv *.mp4 *.avi *.m4v — video files + *.flac *.mp3 *.m4a — audio files + *.srt *.sub *.ass — subtitle files (managed by Bazarr) + *.jpg *.png — artwork (generated by arrs and Emby) + +Always use --dry-run when adding new patterns — review before committing. +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Profile is required — no default. +# Always run --dry-run first when adding new patterns or folders. +# ───────────────────────────────────────────────────────────────────────────── +media_cleaner.sh anime # clean anime share folders +media_cleaner.sh media # clean media share folders + +media_cleaner.sh anime --dry-run # preview — show what would be deleted +media_cleaner.sh media --dry-run # same for media profile + +media_cleaner.sh anime --log # verbose — show every file examined +media_cleaner.sh anime --status # show folder list and patterns +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🎵 lidarr_cleanup.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ Removes orphaned music files from the library that Lidarr no longer tracks. +Queries the live Lidarr API for every tracked file path, walks the music root on disk, +and deletes anything that is not tracked, not protected, and old enough to be +definitively past the import window. -```bash -# Called by media_management.sh -# Always test first: -/mnt/user/appdata/unraid_scripts/Media/lidarr_cleanup.sh --dry-run --log -/mnt/user/appdata/unraid_scripts/Media/lidarr_cleanup.sh -``` - -**How it works:** - -1. Queries the Lidarr API for all tracked file paths -2. Walks `LIDARR_MUSIC_ROOT` on disk -3. Classifies every file found: - -| Classification | Condition | Action | -|---------------|-----------|--------| -| TRACKED | Lidarr API knows this exact path | Leave alone | -| PROTECTED | Matches `LIDARR_PROTECTED_PATTERNS` | Never delete | -| ORPHAN | Music extension, not tracked, older than `LIDARR_ORPHAN_AGE` days | Delete | -| JUNK | Not a music extension, not protected | Delete regardless of age | -| RECENT | Not tracked, under `LIDARR_ORPHAN_AGE` days | Skip — may be mid-import | - -**Why protected patterns are critical:** - -Lidarr generates cover art (`*.jpg`), metadata (`*.nfo`), and lyrics (`*.lrc`) alongside your music files. These do not appear in Lidarr's tracked file API response — they would be classified as orphans and deleted without the protected patterns list. This would break artwork display in Emby and Lidarr itself. - -```bash -LIDARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.lrc") -``` - -Never remove patterns from this list without understanding what Lidarr generates in your specific setup. - -**Why the age threshold matters:** - -When Lidarr downloads a file it exists on disk before it's fully processed and imported. The `LIDARR_ORPHAN_AGE=7` day window ensures files that are mid-import are never touched. 7 days is conservative — adjust if your import workflow takes longer than expected. - -**Configuration:** -```bash -LIDARR_URL="http://192.168.50.2:8686" -LIDARR_API_KEY="your-api-key" -LIDARR_MUSIC_ROOT="/mnt/user/Music-New" # must match Lidarr root path exactly -LIDARR_ORPHAN_AGE=7 -LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") -LIDARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.lrc") -``` - -`LIDARR_MUSIC_ROOT` must match the root path configured in Lidarr Settings → Media Management → Root Folders exactly. A mismatch means all files appear untracked and everything gets deleted. +> **Lidarr runs on HOST1 only.** This script exits cleanly on HOST2 with no action. +> HOST1 is source of truth for Music. --- -### `sonarr_cleanup.sh` +### ── File Classification ────────────────────────────────────────────────────── + +``` +Every file found on disk falls into exactly one category: + +TRACKED → Lidarr API returned this exact path → leave it alone +PROTECTED → matches LIDARR_PROTECTED_PATTERNS → never delete +ORPHAN → music extension, not tracked, old enough → delete +JUNK → not a music extension, not protected → delete (any age) +RECENT → not tracked, under LIDARR_ORPHAN_AGE days → skip for now +``` + +--- + +### ── Why Protected Patterns Are Critical ──────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Lidarr generates these files alongside your music — they are NOT in the +# tracked file API response. Without this list they would be deleted as orphans. +# +# Deleting *.jpg removes cover art from every album in the library. +# Emby and Lidarr both lose artwork display. Requires a full rescan to recover. +# +# Deleting *.nfo removes metadata Lidarr generated for media center compatibility. +# +# Deleting *.lrc removes synced lyrics used by music players that support them. +# + LIDARR_PROTECTED_PATTERNS=( + "*.jpg" "*.jpeg" "*.png" # cover art — generated by Lidarr per album + "*.nfo" # metadata — generated by Lidarr for media centers + "*.lrc" # synced lyrics — generated/downloaded by Lidarr + ) +# +# Never remove from this list without understanding what Lidarr generates in +# your specific setup. The consequences are visible and annoying to fix. +``` + +--- + +### ── Safety Layers — All Must Pass Before Any Deletion ────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Every check is a gate. If any gate fails, the script exits without touching +# a single file. There is no way to "push through" a failed safety check +# without the explicit override flag. +# +# 1. Container running + healthy +# Lidarr must be running and not in starting/unhealthy state. +# A stopped container has an empty (or inaccessible) API. +# +# 2. API reachable +# curl to the Lidarr endpoint must succeed. No API = no tracked file list. +# Without the tracked list, everything on disk looks like an orphan. +# +# 3. API version matches +# Major version must match the tested version in master.conf. +# API breaking changes between major versions would corrupt classification. +# +# 4. Artist count > 0 +# If Lidarr has no artists, something is wrong with the database or config. +# Proceeding would delete everything. +# +# 5. Tracked file count > 0 +# If Lidarr reports zero tracked files, the API returned an empty response. +# Proceeding would delete everything. +# +# 6. Tracked count >= LIDARR_MIN_TRACKED_PCT % of last known count +# If the API returns far fewer tracked files than the last run, something +# changed dramatically. Could be a Lidarr database corruption or a root +# path change. Stop and alert rather than delete thousands of files. +# +# 7. Deletion size < LIDARR_MAX_DELETE_GB +# If the total size of queued deletions exceeds this limit, stop. +# Require --i-know-what-im-doing to proceed. +# This is the last line of defense against a misconfigured root path. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Age Threshold ──────────────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# When Lidarr downloads a file it exists on disk before it is fully processed. +# The import queue can take hours for large batches. A newly downloaded file +# that isn't tracked yet is not an orphan — it is mid-import. +# +# LIDARR_ORPHAN_AGE defines the safe window. Files under this age are +# classified as RECENT and skipped entirely, regardless of tracked status. +# +# 7 days is conservative — adjust if your import workflow is slower than this. +# Be careful reducing this below 3 days — Lidarr's import retry logic can +# hold files in queue for 24-48 hours after initial download failure. +# + LIDARR_ORPHAN_AGE=7 # days — files newer than this are never classified as orphans +``` + +--- + +### ── Override Flags ──────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# --i-know-what-im-doing +# Required when deletion would exceed LIDARR_MAX_DELETE_GB. +# The flag name is long and specific by design — you must type it deliberately. +# It cannot be added by accident. It cannot be forgotten what it means. +# Use when you have intentionally removed a large portion of your library +# and want cleanup to follow through. +# +# --skip-strike-list +# Bypasses the LIDARR_ORPHAN_AGE age check. +# Deletes RECENT files too — files that are under the age threshold. +# Use when you know a batch of recently downloaded files are actually orphans +# and you want them cleaned without waiting for the age window. +# +# ⚠️ NUCLEAR MODE — both flags active simultaneously: +# Age check bypassed. Size threshold bypassed. Deletes everything on first pass. +# Use when Soularr has filled the gaps across your whole library and you want +# a clean one-pass wipe of everything Lidarr doesn't track. +# The script author takes no responsibility for data loss with both flags active. +# The user accepts full responsibility. This is intentional by design. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Configuration ──────────────────────────────────────────────────────────── + +```bash +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# HOST1 only — Lidarr runs on HOST1, Music is HOST1's share. +# + HOST1_LIDARR_URL="http://192.168.50.2:8686" + HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc" + HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music" # must match Lidarr root path EXACTLY +# +# ───────────────────────────────────────────────────────────────────────────── +# LIDARR_MUSIC_ROOT must match the path configured in Lidarr: +# Lidarr UI → Settings → Media Management → Root Folders +# +# A mismatch means every file on disk looks untracked. +# All of them appear as orphans. LIDARR_MAX_DELETE_GB is the only thing +# between a path mismatch and losing your entire music library. +# ───────────────────────────────────────────────────────────────────────────── + +# master.conf + LIDARR_ORPHAN_AGE=7 + LIDARR_MIN_TRACKED_PCT=80 # alert if API returns < 80% of last known count + LIDARR_MAX_DELETE_GB=50 # stop if total deletion size exceeds 50GB + LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") + LIDARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.lrc") +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# ALWAYS run --dry-run --log first. Review every classification decision. +# See the safe testing procedure at the end of this document. +# ───────────────────────────────────────────────────────────────────────────── +lidarr_cleanup.sh --dry-run --log # full review before committing + +lidarr_cleanup.sh # live run after dry run passes review +lidarr_cleanup.sh --log # live run with verbose output +lidarr_cleanup.sh --status # show configuration and API status + +# Override flags — use with deliberate intention +lidarr_cleanup.sh --i-know-what-im-doing +lidarr_cleanup.sh --skip-strike-list +lidarr_cleanup.sh --i-know-what-im-doing --skip-strike-list # nuclear +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 📺 sonarr_cleanup.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ Removes orphaned TV episode files from the library that Sonarr no longer tracks. - -```bash -# Always test first: -/mnt/user/appdata/unraid_scripts/Media/sonarr_cleanup.sh --dry-run --log -/mnt/user/appdata/unraid_scripts/Media/sonarr_cleanup.sh -``` - Same classification logic as `lidarr_cleanup.sh` applied to TV files. -**Protected patterns cover:** -- Show artwork (`*.jpg`, `*.png`) — Sonarr generates per-series and per-episode artwork -- Metadata (`*.nfo`) — Sonarr generates NFO files for media center compatibility -- Subtitles (`*.srt`, `*.sub`, `*.ass`, `*.ssa`) — managed by Bazarr via Sonarr - -```bash -SONARR_URL="http://192.168.50.2:8989" -SONARR_API_KEY="your-api-key" -SONARR_TV_ROOT="/mnt/user/Tv_Shows" -SONARR_ORPHAN_AGE=7 -SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") -SONARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa") -``` - -**`.ts` in extensions:** Transport stream files from Live TV recordings. These are tracked by Sonarr for recorded episodes — include this extension to allow cleanup of orphaned recordings. +> **Sonarr is host-aware.** HOST1 Sonarr manages `Tv_Shows`. HOST2 Sonarr manages +> `Anime_Shows`. `detect_hosts()` aliases the correct URL, API key, and root path. +> The same script works correctly on both servers with no manual routing. --- -### `radarr_cleanup.sh` +### ── Protected Patterns — Sonarr ──────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Sonarr generates artwork and metadata that do NOT appear in the tracked file +# API response. Additionally, Bazarr manages subtitles through Sonarr — +# subtitle files are not tracked by Sonarr directly. +# + SONARR_PROTECTED_PATTERNS=( + "*.jpg" "*.jpeg" "*.png" # show + episode artwork (per series and per episode) + "*.nfo" # metadata — NFO files for media center compatibility + "*.srt" "*.sub" # subtitles — managed by Bazarr via Sonarr + "*.ass" "*.ssa" # advanced subtitle formats — same + ) +# +# *.ts (transport stream) IS in SONARR_EXTENSIONS — not protected. +# Live TV recordings are .ts files tracked by Sonarr for recorded episodes. +# Orphaned .ts recordings should be cleaned up like any other orphaned episode. +``` + +--- + +### ── Post-Deletion Emby Notification ───────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# After deleting any files, notify_emby_scan() is called automatically. +# Triggers Emby's "Clean Missing Files" task immediately. +# Users see clean library entries within seconds of deletion — not hours later +# when Emby's own scheduled scan would eventually catch up. +# +# Configured via: + HOST1_EMBY_URL="http://192.168.50.2:8096" + HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829" +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Configuration ──────────────────────────────────────────────────────────── + +```bash +# master_host1.conf (HOST2 has matching HOST2_ vars for Anime_Shows) + HOST1_SONARR_URL="http://192.168.50.2:8989" + HOST1_SONARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" + HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows" # must match Sonarr root path EXACTLY + +# master.conf + SONARR_ORPHAN_AGE=7 + SONARR_MAX_DELETE_GB=50 + SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") + SONARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa") +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +sonarr_cleanup.sh --dry-run --log # always first +sonarr_cleanup.sh # live run +sonarr_cleanup.sh --log # verbose +sonarr_cleanup.sh --status # configuration and API status +sonarr_cleanup.sh --i-know-what-im-doing # size override +sonarr_cleanup.sh --skip-strike-list # age override +sonarr_cleanup.sh --i-know-what-im-doing --skip-strike-list # nuclear +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🎞️ radarr_cleanup.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ Removes orphaned movie files from the library that Radarr no longer tracks. - -```bash -# Always test first: -/mnt/user/appdata/unraid_scripts/Media/radarr_cleanup.sh --dry-run --log -/mnt/user/appdata/unraid_scripts/Media/radarr_cleanup.sh -``` - Same classification logic applied to movie files. -**Protected patterns cover:** -- Movie artwork (`*.jpg`, `*.png`) -- Metadata (`*.nfo`) -- Subtitles (`*.srt`, `*.sub`, `*.ass`, `*.ssa`) — managed by Bazarr +> **Radarr is host-aware.** HOST1 Radarr manages `Movies`. HOST2 Radarr manages +> `Anime_Movies`. `detect_hosts()` aliases the correct URL, API key, and root path. + +--- + +### ── Protected Patterns — Radarr ──────────────────────────────────────────── ```bash -RADARR_URL="http://192.168.50.2:7878" -RADARR_API_KEY="your-api-key" -RADARR_MOVIES_ROOT="/mnt/user/Movies" -RADARR_ORPHAN_AGE=7 -RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") -RADARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa") +# master.conf + RADARR_PROTECTED_PATTERNS=( + "*.jpg" "*.jpeg" "*.png" # movie artwork — fanart, posters, thumbnails + "*.nfo" # metadata — NFO for media center compatibility + "*.srt" "*.sub" # subtitles — managed by Bazarr via Radarr + "*.ass" "*.ssa" # advanced subtitle formats — same + ) ``` --- -## Safe Testing Procedure +### ── Configuration ──────────────────────────────────────────────────────────── -The arr cleanup scripts permanently delete files. Always test before running live — especially on first use, after API key changes, or after root path changes. - -**Step 1 — Dry run with logging:** ```bash -/mnt/user/appdata/unraid_scripts/Media/lidarr_cleanup.sh --dry-run --log -/mnt/user/appdata/unraid_scripts/Media/sonarr_cleanup.sh --dry-run --log -/mnt/user/appdata/unraid_scripts/Media/radarr_cleanup.sh --dry-run --log +# master_host1.conf (HOST2 has matching HOST2_ vars for Anime_Movies) + HOST1_RADARR_URL="http://192.168.50.2:7878" + HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" + HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies" # must match Radarr root path EXACTLY + +# master.conf + RADARR_ORPHAN_AGE=7 + RADARR_MAX_DELETE_GB=50 + RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") + RADARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa") ``` -`--log` enables verbose output showing every file classification decision. Review carefully: -- Are TRACKED files the ones you expect? -- Are ORPHAN files actually orphans or recently downloaded files? -- Are PROTECTED files being correctly identified? -- Is the root path correct — no files showing as orphans that shouldn't be? - -**Step 2 — Check the numbers make sense:** - -If dry run shows 50,000 files as orphans on a library you know is healthy — something is wrong. Common causes: -- Root path mismatch between Master.conf and arr settings -- API key incorrect — returns empty tracked list -- Arr library scan not complete — recently added files not yet indexed - -**Step 3 — Run live:** -```bash -/mnt/user/appdata/unraid_scripts/Media/lidarr_cleanup.sh -``` - -**Step 4 — Verify in arr UI:** - -After running, check the arr's library count hasn't dropped unexpectedly. A healthy cleanup removes a small number of genuinely orphaned files — not a significant percentage of your library. - --- -## Configuration Quick Reference - -All configuration in `Master.conf` under `── MEDIA ──` section. +### ── Usage ─────────────────────────────────────────────────────────────────── ```bash -# Permissions -PERMISSIONS_MODE="777" -PERMISSIONS_OWNER="nobody:users" -MEDIA_PERMISSION_SHARES=(...) +radarr_cleanup.sh --dry-run --log # always first +radarr_cleanup.sh # live run +radarr_cleanup.sh --log # verbose +radarr_cleanup.sh --status # configuration and API status +radarr_cleanup.sh --i-know-what-im-doing # size override +radarr_cleanup.sh --skip-strike-list # age override +radarr_cleanup.sh --i-know-what-im-doing --skip-strike-list # nuclear +``` -# Cleaner -ANIME_CLEAN_FOLDERS=(...) -MEDIA_CLEAN_FOLDERS=(...) -ANIME_FILE_PATTERNS=(...) -MEDIA_FILE_PATTERNS=(...) +--- -# Orchestrator job order -MEDIA_MAINTENANCE_JOBS=( - "Media/media_shares_permissions.sh" - "Media/media_cleaner.sh anime" - "Media/media_cleaner.sh media" - "Media/lidarr_cleanup.sh" - "Media/sonarr_cleanup.sh" - "Media/radarr_cleanup.sh" +## ━━━ SAFE TESTING PROCEDURE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +> **The arr cleanup scripts permanently delete files.** There is no recycle bin. +> There is no undo. Follow this procedure on first use, after any root path +> change, after any API key change, and after any significant arr library change. + +--- + +### Step 1 — Dry Run With Full Logging + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# --dry-run: no files are touched +# --log: every classification decision is printed +# Review the complete output before proceeding. +# ───────────────────────────────────────────────────────────────────────────── +lidarr_cleanup.sh --dry-run --log +sonarr_cleanup.sh --dry-run --log +radarr_cleanup.sh --dry-run --log +``` + +--- + +### Step 2 — Review the Output Carefully + +``` +Ask these questions about the dry run output: + +Are TRACKED files the ones you expect? + → Files you know are in the arr should show as TRACKED. + → If known files show as ORPHAN, the root path may be wrong. + +Is the ORPHAN count reasonable? + → A healthy library cleanup removes dozens to hundreds of files, not tens of thousands. + → If the number is unexpectedly large, stop. Do not proceed. + +Are PROTECTED patterns working? + → You should see artwork (*.jpg) and subtitles (*.srt) classified as PROTECTED. + → If they show as ORPHAN, the protected patterns are wrong. + +Are RECENT files being correctly skipped? + → Files downloaded in the last 7 days should show as RECENT, not ORPHAN. + → If recent downloads show as ORPHAN, the age threshold may be too aggressive. + +Is the root path correct? + → The scan should start at the right directory. + → Check the first few lines of output — the scan root is logged. +``` + +--- + +### Step 3 — Check the Numbers + +```bash +# If dry run reports an unexpectedly large number of orphans, check: + +# 1. Root path mismatch — what does arr think the root is? +# Lidarr: Settings → Media Management → Root Folders +# Sonarr: Settings → Media Management → Root Folders +# Radarr: Settings → Media Management → Root Folders +# Must exactly match LIDARR_MUSIC_ROOT / SONARR_TV_ROOT / RADARR_MOVIES_ROOT + +# 2. API returning empty — is the arr running? +docker ps | grep -E "Lidarr|Sonarr|Radarr" + +# 3. Library scan not complete — recently added content not yet indexed +# Trigger a manual library scan in the arr UI and wait for completion +``` + +--- + +### Step 4 — Run Live + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Only after dry run review passes. Start with the arr you're most confident about. +# ───────────────────────────────────────────────────────────────────────────── +lidarr_cleanup.sh +sonarr_cleanup.sh +radarr_cleanup.sh +``` + +--- + +### Step 5 — Verify in Arr UI + +``` +After running, check in the arr's UI: + +Library count: should not have dropped significantly + a healthy cleanup removes a small number of files, + not a large percentage of the library + +Missing files: check if any monitored content shows as missing + this would indicate a tracked file was incorrectly deleted + +Emby library: should show no ghost entries (notify_emby_scan handles this automatically) + if ghost entries appear, trigger a manual "Clean Missing Files" task in Emby +``` + +--- + +## ━━━ CONFIGURATION REFERENCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +--- + +### 📋 master_host*.conf — Per-Host Configuration + +```bash +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── + +# Shares this server applies permissions to +HOST1_MEDIA_PERMISSION_SHARES=( + "/mnt/user/Movies" + "/mnt/user/Tv_Shows" + "/mnt/user/Music" ) -# Arr cleanup — per arr -LIDARR_URL / LIDARR_API_KEY / LIDARR_MUSIC_ROOT -LIDARR_ORPHAN_AGE / LIDARR_EXTENSIONS / LIDARR_PROTECTED_PATTERNS -# (same pattern for SONARR_ and RADARR_) +# Folders cleaned by each profile +HOST1_ANIME_CLEAN_FOLDERS=( + "/mnt/user/Anime_Movies" + "/mnt/user/Anime_Shows" +) +HOST1_MEDIA_CLEAN_FOLDERS=( + "/mnt/user/Movies" + "/mnt/user/Tv_Shows" + "/mnt/user/Music" +) + +# Arr connection details — must match arr settings exactly +HOST1_LIDARR_URL="http://192.168.50.2:8686" +HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc" +HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music" + +HOST1_SONARR_URL="http://192.168.50.2:8989" +HOST1_SONARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" +HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows" + +HOST1_RADARR_URL="http://192.168.50.2:7878" +HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" +HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies" + +HOST1_EMBY_URL="http://192.168.50.2:8096" +HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829" ``` --- -## Adding a New Arr +### 📋 master.conf — Shared Configuration -To add Readarr or any other arr cleanup to the ecosystem: +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── -1. Copy `radarr_cleanup.sh` as the template — same classification logic applies -2. Update the API endpoint, variable names, and root path -3. Add configuration variables to `Master.conf` -4. Add the script to `MEDIA_MAINTENANCE_JOBS` in `Master.conf` -5. Test with `--dry-run --log` before running live +# ── Permissions ──────────────────────────────────────────────────────────── + PERMISSIONS_DIR_MODE="755" + PERMISSIONS_FILE_MODE="664" + PERMISSIONS_OWNER="nobody:users" -The orchestrator picks it up automatically — no changes to `media_management.sh` needed. \ No newline at end of file +# ── File Patterns ────────────────────────────────────────────────────────── + ANIME_FILE_PATTERNS=("*.sfv" "*.md5" "*.sha1" ...) + MEDIA_FILE_PATTERNS=("${ANIME_FILE_PATTERNS[@]}" "*.iso" "*.lrc") + +# ── Arr Cleanup Thresholds ───────────────────────────────────────────────── + LIDARR_ORPHAN_AGE=7 # days — files newer than this are RECENT + LIDARR_MIN_TRACKED_PCT=80 # alert if API returns < 80% of last known count + LIDARR_MAX_DELETE_GB=50 # stop if total deletion exceeds this + LIDARR_EXTENSIONS=(...) + LIDARR_PROTECTED_PATTERNS=(...) + + SONARR_ORPHAN_AGE=7 + SONARR_MAX_DELETE_GB=50 + SONARR_EXTENSIONS=(...) + SONARR_PROTECTED_PATTERNS=(...) + + RADARR_ORPHAN_AGE=7 + RADARR_MAX_DELETE_GB=50 + RADARR_EXTENSIONS=(...) + RADARR_PROTECTED_PATTERNS=(...) + +# ── Orchestrator Job Order ───────────────────────────────────────────────── + MEDIA_MAINTENANCE_JOBS=( + "Media/media_shares_permissions.sh" + "Media/media_cleaner.sh anime" + "Media/media_cleaner.sh media" + "Media/lidarr_cleanup.sh" + "Media/sonarr_cleanup.sh" + "Media/radarr_cleanup.sh" + ) +``` + +--- + +## ━━━ ADDING A NEW ARR ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +To add Readarr, Whisparr, or any other arr cleanup to the ecosystem: + +```bash +# 1. Copy radarr_cleanup.sh as the template — same classification logic applies +cp radarr_cleanup.sh readarr_cleanup.sh + +# 2. Update the API endpoint variable names and root path variable names +# Replace RADARR_ with READARR_ throughout + +# 3. Add configuration to master_host*.conf +HOST1_READARR_URL="http://192.168.50.2:8787" +HOST1_READARR_API_KEY="your-api-key" +HOST1_READARR_BOOKS_ROOT="/mnt/user/Books" + +# 4. Add thresholds to master.conf +READARR_ORPHAN_AGE=7 +READARR_MAX_DELETE_GB=50 +READARR_EXTENSIONS=("epub" "pdf" "mobi" "azw3" "cbz" "cbr") +READARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo") + +# 5. Add to MEDIA_MAINTENANCE_JOBS in master.conf +MEDIA_MAINTENANCE_JOBS=( + ...existing jobs... + "Media/readarr_cleanup.sh" # add at the end — after permissions and cleaner +) + +# ───────────────────────────────────────────────────────────────────────────── +# media_management.sh picks it up automatically. +# No changes to the orchestrator needed. +# Run --dry-run --log before scheduling. +# ───────────────────────────────────────────────────────────────────────────── +``` \ No newline at end of file diff --git a/Media/arrs_failed_stalled_recovery.sh b/Media/arrs_failed_stalled_recovery.sh index 31af498..6d45e81 100644 --- a/Media/arrs_failed_stalled_recovery.sh +++ b/Media/arrs_failed_stalled_recovery.sh @@ -1,67 +1,81 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Arrs Failed Stalled Recovery -------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ========================= Arrs Failed / Stalled Recovery ===================================== +# ============================================================================================== # Automatically detects and recovers from failed imports and stalled downloads # across Sonarr, Radarr, and Lidarr. Blocklists the bad release and triggers -# a new search — hands free recovery while you sleep. -# Schedule: 0 */6 * * * (every 6 hours) +# a new search — hands-free recovery while you sleep. # -# What it checks (per-arr toggles in Master.conf): -# HOST1: Sonarr (Tv_Shows) — /api/v3/ -# HOST1: Radarr (Movies) — /api/v3/ -# HOST1: Lidarr (Music) — /api/v1/ ← HOST1 only, exits cleanly on HOST2 -# HOST2: Sonarr (Anime_Shows) — /api/v3/ -# HOST2: Radarr (Anime_Movies) — /api/v3/ -# -# Targets four problem types from the queue API: +# ── WHAT IT CHECKS ──────────────────────────────────────────────────────────────────────────── +# Four problem types from the arr queue API: # importFailed — downloaded successfully but arr couldn't import the file -# importPending — downloaded, stuck waiting to import (won't self-resolve) -# error status — serious failure not covered by importFailed/importPending +# importPending — downloaded, stuck waiting to import (will not self-resolve) +# error status — serious failure not covered by the above two states # stalled — download stuck with no connections or no progress # -# Items newer than ARR_IMPORT_RECOVERY_AGE (6hr) are skipped — gives arr time to retry. # Never touches items with state "downloading" or "imported" — safe to run anytime. +# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first. # -# Action per problem item: -# 1. Blocklist the release — prevents re-grabbing the same bad release -# 2. Remove from queue — cleans up the failed item -# 3. Trigger new search — finds a different release automatically +# ── WHAT IT DOES PER PROBLEM ITEM ───────────────────────────────────────────────────────────── +# 1. Blocklist the release — prevents re-grabbing the same bad release +# 2. Remove from queue — cleans up the failed item +# 3. Trigger new search — finds a different release automatically # -# Configuration in Master.conf: -# ARR_IMPORT_RECOVERY_AGE — hours before item is eligible -# HOST1/2_SONARR_RECOVERY — enable/disable per arr -# HOST1/2_RADARR_RECOVERY — enable/disable per arr -# HOST1_LIDARR_RECOVERY — enable/disable Lidarr (HOST1 only) +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID and aliases all arr vars: +# SONARR_URL / SONARR_API_KEY / SONARR_RECOVERY +# RADARR_URL / RADARR_API_KEY / RADARR_RECOVERY +# LIDARR_URL / LIDARR_API_KEY / LIDARR_RECOVERY (HOST1 only — exits cleanly on HOST2) +# No manual HOST1/HOST2 comparisons needed — MY_ID routes automatically. # -# Age threshold (ARR_IMPORT_RECOVERY_AGE): -# Items newer than threshold are skipped — gives the arr time to retry on its own -# Items older than threshold have not self-resolved — safe to intervene -# Default: 12 hours +# ── API VERSION SAFETY ──────────────────────────────────────────────────────────────────────── +# check_arr_version() verifies the running arr matches the tested major version in master.conf. +# If the API structure changed after an upgrade — exits rather than silently misoperating. +# Sonarr v4 → /api/v3/ (v3 endpoint retained in v4) +# Radarr v6 → /api/v3/ (v3 endpoint retained in v6) +# Lidarr v3 → /api/v1/ (different from Sonarr/Radarr) # -# API versions: -# Sonarr v4 → /api/v3/ (v3 endpoint retained in v4) -# Radarr v6 → /api/v3/ (v3 endpoint retained in v6) -# Lidarr v3 → /api/v1/ (different from Sonarr/Radarr) +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock — prevents concurrent runs overlapping +# jq validation — exits if jq not installed (required for JSON parsing) +# API pre-flight — checks each arr is reachable before querying queue +# Version check — verifies arr major version matches tested version in master.conf +# Age threshold — skips items newer than ARR_IMPORT_RECOVERY_AGE (default 6hr) +# Silent by default — only problems produce output, clean arrs stay silent # -# Per-arr enable/disable toggles in Master.conf. -# Lidarr runs on HOST1 only — exits cleanly on HOST2. -# detect_hosts() selects correct URL and API key per server at runtime. +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_SONARR_URL / HOST*_SONARR_API_KEY / HOST*_SONARR_RECOVERY +# HOST*_RADARR_URL / HOST*_RADARR_API_KEY / HOST*_RADARR_RECOVERY +# HOST1_LIDARR_URL / HOST1_LIDARR_API_KEY / HOST1_LIDARR_RECOVERY +# All aliased by detect_hosts() — script uses unprefixed names # -# Recommended schedule: 0 5 * * * (5am daily) -# Supports --dry-run to show what would be actioned without making changes. -# ----------------------------------------------------------------------------------------------- +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# ARR_IMPORT_RECOVERY_AGE — hours before item is eligible (default 6) +# SONARR_VERSION_MAJOR — expected Sonarr major version (e.g. 4) +# RADARR_VERSION_MAJOR — expected Radarr major version (e.g. 6) +# LIDARR_VERSION_MAJOR — expected Lidarr major version (e.g. 3) +# ARR_RECOVERY_STATS — stats file path (read by coffee report) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# arrs_failed_stalled_recovery.sh — normal run +# arrs_failed_stalled_recovery.sh --dry-run — show what would be actioned, no changes +# arrs_failed_stalled_recovery.sh --log — verbose output +# arrs_failed_stalled_recovery.sh --status — show config and exit +# +# ── SCHEDULE ────────────────────────────────────────────────────────────────────────────────── +# Recommended: 0 5 * * * (5am daily) +# Or every 6hr: 0 */6 * * * (matches ARR_IMPORT_RECOVERY_AGE default) +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" @@ -70,41 +84,73 @@ if [[ "$EUID" -ne 0 ]]; then exit 1 fi -success "Running as root" - acquire_lock +# detect_hosts() sets MY_ID and aliases SONARR_*, RADARR_*, LIDARR_* vars detect_hosts +# jq is required — not optional — for JSON parsing +if ! command -v jq >/dev/null 2>&1; then + error "jq is not installed — required for arr API JSON parsing" + error "Install: apt-get install jq or brew install jq" + notify "arrs_failed_stalled_recovery failed on $(hostname) — jq not installed" \ + "Arr Recovery" "warning" + exit 1 +fi +log "jq found" + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no items will be blocklisted or searched" -# Age threshold in seconds for comparison +# Age threshold in seconds AGE_THRESHOLD_SECONDS=$(( ARR_IMPORT_RECOVERY_AGE * 3600 )) -# Tracking totals TOTAL_ACTIONED=0 TOTAL_SKIPPED=0 ARR_SUMMARIES=() -# ----------------------------------------------------------------------------------------------- -# CORE FUNCTIONS -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_SYNC Sonarr: ${SONARR_URL:-not configured} (recovery: ${SONARR_RECOVERY:-true})" + echo "$ICON_SYNC Radarr: ${RADARR_URL:-not configured} (recovery: ${RADARR_RECOVERY:-true})" + echo "$ICON_SYNC Lidarr: ${LIDARR_URL:-not configured on this host} (recovery: ${LIDARR_RECOVERY:-false})" + echo "$ICON_TIME Age thresh: ${ARR_IMPORT_RECOVERY_AGE}hr" + echo "$ICON_GEAR Sonarr ver: v${SONARR_VERSION_MAJOR} expected" + echo "$ICON_GEAR Radarr ver: v${RADARR_VERSION_MAJOR} expected" + echo "$ICON_GEAR Lidarr ver: v${LIDARR_VERSION_MAJOR} expected" + echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== # Check if a queue item is older than ARR_IMPORT_RECOVERY_AGE +# Returns 0 (old enough) or 1 (too new — skip) item_is_old_enough() { local added="$1" - [[ -z "$added" ]] && return 0 # no date = treat as old enough + [[ -z "$added" ]] && return 0 # no date = treat as old enough, safe to act local added_epoch added_epoch=$(date -d "$added" +%s 2>/dev/null) || return 0 - local now_epoch - now_epoch=$(date +%s) - local age_seconds=$(( now_epoch - added_epoch )) + local age_seconds=$(( $(date +%s) - added_epoch )) [[ "$age_seconds" -ge "$AGE_THRESHOLD_SECONDS" ]] } -# Query the arr queue and return failed/stalled items +# Query the arr queue API and return all records # Args: url, api_key, api_version -get_problem_items() { +get_queue_data() { local url="$1" api_key="$2" api_version="$3" curl -sf --max-time 15 \ -H "X-Api-Key: $api_key" \ @@ -112,7 +158,7 @@ get_problem_items() { 2>/dev/null } -# Blocklist and remove item from queue +# Blocklist and remove a queue item # Args: url, api_key, api_version, queue_id blocklist_item() { local url="$1" api_key="$2" api_version="$3" queue_id="$4" @@ -127,18 +173,18 @@ blocklist_item() { >/dev/null 2>&1 } -# Trigger new search +# Trigger a new search for the media item # Args: url, api_key, api_version, arr_type, media_id trigger_search() { local url="$1" api_key="$2" api_version="$3" arr_type="$4" media_id="$5" local command body case "$arr_type" in - sonarr) command="EpisodeSearch"; body="{\"name\":\"EpisodeSearch\",\"episodeIds\":[$media_id]}" ;; - radarr) command="MoviesSearch"; body="{\"name\":\"MoviesSearch\",\"movieIds\":[$media_id]}" ;; - lidarr) command="AlbumSearch"; body="{\"name\":\"AlbumSearch\",\"albumIds\":[$media_id]}" ;; + sonarr) command="EpisodeSearch"; body="{\"name\":\"EpisodeSearch\",\"episodeIds\":[$media_id]}" ;; + radarr) command="MoviesSearch"; body="{\"name\":\"MoviesSearch\",\"movieIds\":[$media_id]}" ;; + lidarr) command="AlbumSearch"; body="{\"name\":\"AlbumSearch\",\"albumIds\":[$media_id]}" ;; esac if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would trigger $command for ID $media_id" + warn "DRY RUN — would trigger $command for media ID $media_id" return 0 fi curl -sf --max-time 15 \ @@ -150,34 +196,64 @@ trigger_search() { >/dev/null 2>&1 } -# ----------------------------------------------------------------------------------------------- -# PROCESS AN ARR -# Args: arr_name, arr_type, url, api_key, api_version, enabled -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ── PROCESS AN ARR ──────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Args: display_name, arr_type, url, api_key, api_version, enabled, +# version_major, version_api_prefix +# +# Exits cleanly if disabled. +# Checks API reachability and version before touching queue. +# Processes each problem item: blocklist + trigger new search. +# Silent when clean — only warns when problems found or actioned. + process_arr() { - local arr_name="$1" arr_type="$2" url="$3" api_key="$4" api_version="$5" enabled="$6" - local actioned=0 skipped=0 too_new=0 + local arr_name="$1" + local arr_type="$2" + local url="$3" + local api_key="$4" + local api_version="$5" + local enabled="$6" + local version_major="$7" + local version_api_prefix="$8" + + local actioned=0 skipped_new=0 echo "" echo "━━━ $ICON_SYNC $arr_name ━━━" + # Disabled — skip cleanly if [[ "$enabled" != "true" ]]; then - info "$arr_name recovery disabled — skipping" + log "$arr_name recovery disabled — skipping" ARR_SUMMARIES+=("$arr_name: disabled") return fi - # API pre-flight + # URL not configured on this host — skip cleanly + if [[ -z "$url" ]]; then + log "$arr_name not configured on $MY_ID — skipping" + ARR_SUMMARIES+=("$arr_name: not configured on $MY_ID") + return + fi + + # API reachability if ! check_api "$url" "$arr_name" 10; then warn "$arr_name API unreachable — skipping" ARR_SUMMARIES+=("$arr_name: unreachable") return fi - # Get queue + # Version check — exit if API structure may have changed + if ! check_arr_version "$url" "$api_key" "$version_api_prefix" \ + "$version_major" "$arr_name"; then + ARR_SUMMARIES+=("$arr_name: version mismatch — skipped") + return + fi + + # Fetch queue local queue_data - queue_data=$(get_problem_items "$url" "$api_key" "$api_version") - if [[ -z "$queue_data" ]] || ! command -v jq >/dev/null 2>&1; then + queue_data=$(get_queue_data "$url" "$api_key" "$api_version") + if [[ -z "$queue_data" ]]; then warn "$arr_name — could not retrieve queue data" ARR_SUMMARIES+=("$arr_name: queue fetch failed") return @@ -185,13 +261,9 @@ process_arr() { local total_records total_records=$(echo "$queue_data" | jq '.totalRecords // 0' 2>/dev/null) - info "Queue: $total_records total items" + log "$arr_name queue: $total_records total items" # Filter for problem items — never touch downloading or imported - # importFailed = tried to import, actually failed - # importPending = downloaded, stuck waiting to import (won't self-resolve) - # error status = serious failure not covered by above states - # stalled = download stuck with no connections or progress local problem_items problem_items=$(echo "$queue_data" | jq -c ' .records // [] | @@ -205,48 +277,46 @@ process_arr() { .trackedDownloadStatus == "error" or (.status == "warning" and ( (.errorMessage // "" | ascii_downcase | contains("stalled")) or - (.statusMessages // [] | .[] | .messages // [] | .[] | ascii_downcase | contains("stalled")) + (.statusMessages // [] | .[] | .messages // [] | .[] | + ascii_downcase | contains("stalled")) )) ) ) ' 2>/dev/null) if [[ -z "$problem_items" ]]; then - success "$arr_name — no failed imports or stalled downloads found" + log "$arr_name — clean ✅ no failed imports or stalled downloads" ARR_SUMMARIES+=("$arr_name: clean ✅") return fi local problem_count problem_count=$(echo "$problem_items" | wc -l) - info "Found $problem_count problem item(s)" + warn "$arr_name — found $problem_count problem item(s)" # Process each problem item while IFS= read -r item; do [[ -z "$item" ]] && continue - local queue_id title added problem_type media_id - queue_id=$(echo "$item" | jq -r '.id // empty' 2>/dev/null) - title=$(echo "$item" | jq -r '.title // "Unknown"' 2>/dev/null) - added=$(echo "$item" | jq -r '.added // empty' 2>/dev/null) + local queue_id title added tracked_state tracked_status problem_type media_id - # Determine problem type for display - local tracked_state tracked_status + queue_id=$(echo "$item" | jq -r '.id // empty' 2>/dev/null) + title=$(echo "$item" | jq -r '.title // "Unknown"' 2>/dev/null) + added=$(echo "$item" | jq -r '.added // empty' 2>/dev/null) tracked_state=$(echo "$item" | jq -r '.trackedDownloadState // ""' 2>/dev/null) tracked_status=$(echo "$item" | jq -r '.trackedDownloadStatus // ""' 2>/dev/null) + + # Human-readable problem type case "$tracked_state" in importFailed) problem_type="import failed" ;; importPending) problem_type="import pending/stuck" ;; *) - if [[ "$tracked_status" == "error" ]]; then - problem_type="error" - else - problem_type="stalled" - fi + [[ "$tracked_status" == "error" ]] && \ + problem_type="error" || problem_type="stalled" ;; esac - # Get media ID for search trigger (episode, movie, or album) + # Media ID for search trigger case "$arr_type" in sonarr) media_id=$(echo "$item" | jq -r '.episodeId // .episode.id // empty' 2>/dev/null) ;; radarr) media_id=$(echo "$item" | jq -r '.movieId // .movie.id // empty' 2>/dev/null) ;; @@ -255,129 +325,124 @@ process_arr() { [[ -z "$queue_id" ]] && continue - # Age check + # Age check — skip items that are too new to have self-resolved if ! item_is_old_enough "$added"; then - info " $ICON_TIME Skipping (too new < ${ARR_IMPORT_RECOVERY_AGE}hr): $title" - ((too_new++)) - ((skipped++)) + log " Skipping (too new < ${ARR_IMPORT_RECOVERY_AGE}hr): $title" + (( skipped_new++ )) + (( TOTAL_SKIPPED++ )) continue fi - info " $ICON_TRASH $problem_type: $title" + warn " $ICON_TRASH $problem_type — $title" - # Blocklist + remove - if blocklist_item "$url" "$api_key" "$api_version" "$queue_id"; then - log " Blocklisted queue item: $queue_id" - else + # Step 1: Blocklist + remove from queue + if ! blocklist_item "$url" "$api_key" "$api_version" "$queue_id"; then warn " Failed to blocklist: $title" - ((skipped++)) + (( TOTAL_SKIPPED++ )) continue fi + log " Blocklisted: $queue_id" - # Trigger new search if we have a media ID + # Step 2: Trigger new search if [[ -n "$media_id" ]]; then if trigger_search "$url" "$api_key" "$api_version" "$arr_type" "$media_id"; then - log " New search triggered for: $title" - ((actioned++)) - ((TOTAL_ACTIONED++)) + log " New search triggered: $title" else warn " Blocklisted but search trigger failed: $title" - ((actioned++)) - ((TOTAL_ACTIONED++)) fi else - warn " Blocklisted but no media ID found for search: $title" - ((actioned++)) - ((TOTAL_ACTIONED++)) + warn " Blocklisted but no media ID found — search not triggered: $title" fi + (( actioned++ )) + (( TOTAL_ACTIONED++ )) + done <<< "$problem_items" - if [[ "$DRY_RUN" == true ]]; then - info "$arr_name — dry run complete" + if [[ "$actioned" -gt 0 ]]; then + warn "$arr_name — actioned: $actioned | skipped (too new): $skipped_new" else - success "$arr_name — actioned: $actioned | skipped (too new): $too_new" + log "$arr_name — nothing actioned | skipped (too new): $skipped_new" fi - ARR_SUMMARIES+=("$arr_name: actioned $actioned | too new $too_new") - TOTAL_SKIPPED=$(( TOTAL_SKIPPED + skipped )) + ARR_SUMMARIES+=("$arr_name: actioned $actioned | too new $skipped_new") } -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SYNC Process Each Arr ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_SYNC Arrs Failed Stalled Recovery — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -echo "$ICON_TIME Age threshold: ${ARR_IMPORT_RECOVERY_AGE}hr (items newer than this are skipped)" +# ============================================================================================== +# ━━━ Process Each Arr ━━━ +# ============================================================================================== echo "" +echo "━━━ $ICON_SYNC Arrs Failed/Stalled Recovery — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)" +log "Age threshold: ${ARR_IMPORT_RECOVERY_AGE}hr" START=$(date +%s) -# Sonarr -if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - process_arr "Sonarr (Tv_Shows)" "sonarr" \ - "$HOST1_SONARR_URL" "$HOST1_SONARR_API_KEY" "v3" \ - "${HOST1_SONARR_RECOVERY:-true}" -else - process_arr "Sonarr (Anime_Shows)" "sonarr" \ - "$HOST2_SONARR_URL" "$HOST2_SONARR_API_KEY" "v3" \ - "${HOST2_SONARR_RECOVERY:-true}" -fi +# Sonarr — uses aliased vars set by detect_hosts() +process_arr \ + "Sonarr" \ + "sonarr" \ + "${SONARR_URL:-}" \ + "${SONARR_API_KEY:-}" \ + "v3" \ + "${SONARR_RECOVERY:-true}" \ + "${SONARR_VERSION_MAJOR:-4}" \ + "v3" -# Radarr -if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - process_arr "Radarr (Movies)" "radarr" \ - "$HOST1_RADARR_URL" "$HOST1_RADARR_API_KEY" "v3" \ - "${HOST1_RADARR_RECOVERY:-true}" -else - process_arr "Radarr (Anime_Movies)" "radarr" \ - "$HOST2_RADARR_URL" "$HOST2_RADARR_API_KEY" "v3" \ - "${HOST2_RADARR_RECOVERY:-true}" -fi +# Radarr — uses aliased vars set by detect_hosts() +process_arr \ + "Radarr" \ + "radarr" \ + "${RADARR_URL:-}" \ + "${RADARR_API_KEY:-}" \ + "v3" \ + "${RADARR_RECOVERY:-true}" \ + "${RADARR_VERSION_MAJOR:-6}" \ + "v3" -# Lidarr — HOST1 only -if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - process_arr "Lidarr (Music)" "lidarr" \ - "$HOST1_LIDARR_URL" "$HOST1_LIDARR_API_KEY" "v1" \ - "${HOST1_LIDARR_RECOVERY:-true}" -else - info "Lidarr runs on $HOST1 only — skipping on $LOCAL_SERVER_NAME" -fi +# Lidarr — HOST1 only, LIDARR_URL empty on HOST2 → exits cleanly via "not configured" guard +process_arr \ + "Lidarr" \ + "lidarr" \ + "${LIDARR_URL:-}" \ + "${LIDARR_API_KEY:-}" \ + "v1" \ + "${LIDARR_RECOVERY:-false}" \ + "${LIDARR_VERSION_MAJOR:-3}" \ + "v1" END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" -echo "━━━━━ $ICON_SUMMARY ARR IMPORT RECOVERY SUMMARY ━━━━━" -echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" -echo "$ICON_TRASH Actioned: $TOTAL_ACTIONED items blocklisted + searched" -echo "$ICON_RUNNING Skipped: $TOTAL_SKIPPED items (too new or unreachable)" +echo "━━━━━ $ICON_SUMMARY ARR RECOVERY SUMMARY ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" +echo "$ICON_TRASH Actioned: $TOTAL_ACTIONED items blocklisted + searched" +echo "$ICON_SKIP Skipped: $TOTAL_SKIPPED items (too new)" echo "" -echo " Problem types detected: importFailed | importPending | error | stalled" -echo "" - for summary in "${ARR_SUMMARIES[@]}"; do echo " $ICON_SUMMARY $summary" done - echo "" if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no changes made" + warn "DRY RUN — no changes made" elif [[ "$TOTAL_ACTIONED" -gt 0 ]]; then - echo "$ICON_DONE Status: $ICON_SUCCESS DONE" - notify "Arr import recovery on $(hostname) — $TOTAL_ACTIONED item(s) blocklisted and re-searched. Check arrs for new downloads." "Arr Recovery" "normal" + warn "$ICON_DONE Done — $TOTAL_ACTIONED item(s) blocklisted and re-searched" + notify "Arr recovery on $(hostname) — $TOTAL_ACTIONED item(s) blocklisted and re-searched" \ + "Arr Recovery" "warning" else - echo "$ICON_DONE Status: $ICON_SUCCESS DONE — nothing to recover" + log "$ICON_DONE Done — nothing to recover (all arrs clean)" fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Write stats for sunday_morning_coffee_report.sh if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_RECOVERY_STATS:-}" ]]; then - DATE=$(date '+%Y-%m-%d') - TIME=$(date '+%H:%M') - echo "${DATE}|${TIME}|${TOTAL_ACTIONED}|${TOTAL_SKIPPED}" \ + echo "$(date '+%Y-%m-%d')|$(date '+%H:%M')|${TOTAL_ACTIONED}|${TOTAL_SKIPPED}" \ >> "$ARR_RECOVERY_STATS" 2>/dev/null || true fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file + +exit 0 \ No newline at end of file diff --git a/Media/lidarr_cleanup.sh b/Media/lidarr_cleanup.sh index 08bdbda..1fb847f 100644 --- a/Media/lidarr_cleanup.sh +++ b/Media/lidarr_cleanup.sh @@ -1,91 +1,118 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Lidarr Cleanup Script -------------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= Lidarr Cleanup ============================================= +# ============================================================================================== # Removes orphaned music files from the library that Lidarr no longer tracks. # Uses the Lidarr API to build a complete list of tracked file paths then compares -# against what exists on disk — anything not tracked and older than LIDARR_ORPHAN_AGE +# against what exists on disk — anything untracked and older than LIDARR_ORPHAN_AGE # days is considered an orphan and deleted. # -# File classification: +# ── FILE CLASSIFICATION ─────────────────────────────────────────────────────────────────────── # TRACKED — Lidarr API knows about this exact file path → leave it alone -# PROTECTED — matches LIDARR_PROTECTED_PATTERNS → never delete (cover art, .nfo etc.) +# PROTECTED — matches LIDARR_PROTECTED_PATTERNS → never delete (cover art, .nfo, .lrc etc.) # ORPHAN — music file, not tracked, older than LIDARR_ORPHAN_AGE days → delete # JUNK — not a music extension, not protected → delete regardless of age # RECENT — not tracked, under LIDARR_ORPHAN_AGE days old → skip (may be mid-import) # -# Why protected patterns matter: -# Lidarr generates cover art (*.jpg), metadata (*.nfo) and lyrics (*.lrc) but does -# not include these in its tracked file API response. Without protection these would -# be classified as orphans and deleted — breaking Lidarr and Emby metadata display. +# ── WHY PROTECTED PATTERNS MATTER ───────────────────────────────────────────────────────────── +# Lidarr generates cover art (*.jpg), metadata (*.nfo) and lyrics (*.lrc) but does NOT +# include these in its tracked file API response. Without protection these would be +# classified as orphans and deleted — breaking Lidarr and Emby metadata display. # -# Safety layers — all must pass before any file is touched: -# 1. Container must be running and healthy +# ── SAFETY LAYERS — ALL MUST PASS BEFORE ANY FILE IS TOUCHED ───────────────────────────────── +# 1. Container must be running and not starting/unhealthy # 2. API must be reachable -# 3. Artist count must be > 0 -# 4. Tracked file count must be > 0 -# 5. Tracked count must be >= LIDARR_MIN_TRACKED_PCT% of last known count -# 6. Deletion size must be < LIDARR_MAX_DELETE_GB — or --i-know-what-im-doing required +# 3. API version must match tested major version in master.conf +# 4. Artist count must be > 0 +# 5. Tracked file count must be > 0 +# 6. Tracked count must be >= LIDARR_MIN_TRACKED_PCT % of last known count +# 7. Deletion size must be < LIDARR_MAX_DELETE_GB — or --i-know-what-im-doing required # -# --i-know-what-im-doing flag: -# Required when deletion would exceed LIDARR_MAX_DELETE_GB -# Long and annoying by design — cannot be added accidentally +# ── OVERRIDE FLAGS ──────────────────────────────────────────────────────────────────────────── +# --i-know-what-im-doing +# Required when deletion would exceed LIDARR_MAX_DELETE_GB. +# Long and annoying by design — cannot be added accidentally. # -# --skip-strike-list flag: -# Bypasses the LIDARR_ORPHAN_AGE age check — deletes recent files too -# Combined with --i-know-what-im-doing activates NUCLEAR MODE: -# Age check bypassed, size threshold bypassed, deletes on first pass -# Use when Soularr/other tool has filled the gaps — clean one-pass wipe -# ⚠️ The script author takes NO responsibility for data loss with both flags active -# The user accepts full responsibility — this is 100% intentional by design +# --skip-strike-list +# Bypasses the LIDARR_ORPHAN_AGE age check — deletes recent files too. # +# NUCLEAR MODE — both flags active together: +# Age check bypassed, size threshold bypassed, deletes on first pass. +# Use when Soularr has filled the gaps and you want a clean one-pass wipe. +# ⚠️ Script author takes NO responsibility for data loss with both flags active. +# The user accepts full responsibility — this is 100% intentional by design. +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── # Lidarr runs on HOST1 only — music library is HOST1's source of truth. -# If run on HOST2 this script exits cleanly with no action. -# All configuration in Master.conf under Arr Cleanup section. -# Supports --dry-run to preview what would be deleted without making changes. -# ----------------------------------------------------------------------------------------------- +# detect_hosts() sets MY_ID — if MY_ID != HOST1 script exits cleanly with no action. +# LIDARR_URL, LIDARR_API_KEY, LIDARR_MUSIC_ROOT aliased by detect_hosts() automatically. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock "wait" — large scans take time, wait for previous run to finish +# jq + curl validation — exits if either tool missing +# DOCKER_TIMEOUT — container checks protected against daemon hangs +# 7 safety layers — all must pass before any file is touched +# Duplicate detection — temp file of tracked paths, grep before delete +# validate_unraid_cmd — notify script validated before use +# Silent by default — orphans/junk warn(), clean library logs silently +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST1_LIDARR_URL / HOST1_LIDARR_API_KEY / HOST1_LIDARR_MUSIC_ROOT +# HOST1_LIDARR_PATH_MAP — container path → host path translation +# All aliased by detect_hosts() — script uses unprefixed names +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# LIDARR_LOCK_WARN_AGE — override default lock warning age (large libraries) +# LIDARR_ORPHAN_AGE — days before untracked file is eligible for deletion +# LIDARR_MAX_DELETE_GB — require --i-know-what-im-doing above this +# LIDARR_MIN_TRACKED_PCT — abort if tracked count drops below this % of last run +# LIDARR_TRACKED_COUNT_FILE — persistent baseline file path +# LIDARR_EXTENSIONS — music file extensions to consider for orphan classification +# LIDARR_PROTECTED_PATTERNS — file patterns that are never deleted +# LIDARR_VERSION_MAJOR — expected Lidarr major version for API safety check +# ARR_CLEANUP_STATS — stats file path (read by coffee report) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# lidarr_cleanup.sh — normal run +# lidarr_cleanup.sh --dry-run — preview, no deletions +# lidarr_cleanup.sh --log — verbose output +# lidarr_cleanup.sh --status — show config and exit +# lidarr_cleanup.sh --i-know-what-im-doing — bypass size threshold +# lidarr_cleanup.sh --i-know-what-im-doing --skip-strike-list — NUCLEAR MODE +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" -# Check for --i-know-what-im-doing and --skip-strike-list flags before parse_args -# These flags are filtered out before parse_args sees them to avoid unknown flag errors +# ── Special flag pre-processing ─────────────────────────────────────────────────────────────── +# Filter --i-know-what-im-doing and --skip-strike-list before parse_args +# to avoid unknown flag errors — these are handled separately below. I_KNOW=false SKIP_STRIKES=false FILTERED_ARGS=() for arg in "$@"; do - if [[ "$arg" == "--i-know-what-im-doing" ]]; then - I_KNOW=true - elif [[ "$arg" == "--skip-strike-list" ]]; then - SKIP_STRIKES=true - else - FILTERED_ARGS+=("$arg") - fi + case "$arg" in + --i-know-what-im-doing) I_KNOW=true ;; + --skip-strike-list) SKIP_STRIKES=true ;; + *) FILTERED_ARGS+=("$arg") ;; + esac done parse_args "${FILTERED_ARGS[@]}" -# Nuclear mode — both override flags active -# Strike system AND size threshold bypassed — deletes on first pass -# Script author takes no responsibility for data loss when both flags are used. -# This combination is 100% intentional and the user accepts full responsibility. +# ── Nuclear mode warning ────────────────────────────────────────────────────────────────────── if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" != true ]]; then echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "⚠️ WARNING — NUCLEAR MODE ACTIVE" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo " Flags: --i-know-what-im-doing --skip-strike-list" - echo " Strike system: BYPASSED — deletes on first pass" - echo " Size threshold: BYPASSED — no GB limit" - echo " Data recovery: NOT POSSIBLE after deletion" + echo " Flags: --i-know-what-im-doing --skip-strike-list" + echo " Strike system: BYPASSED — deletes on first pass" + echo " Size threshold: BYPASSED — no GB limit" + echo " Data recovery: NOT POSSIBLE after deletion" echo "" - echo " The script author takes no responsibility for data" - echo " loss when both flags are used together. This is a" - echo " 100% intentional action by the user." - echo "" - echo " Review the dry run output before proceeding." + echo " Review --dry-run output before proceeding." echo " You have 10 seconds to cancel (Ctrl+C)..." echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" sleep 10 @@ -93,9 +120,9 @@ if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" != echo "" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" @@ -104,8 +131,7 @@ if [[ "$EUID" -ne 0 ]]; then exit 1 fi -success "Running as root" - +# Tool validation — both required, fail fast if ! command -v curl >/dev/null 2>&1; then error "curl not found — required for Lidarr API calls" exit 1 @@ -117,110 +143,115 @@ if ! command -v jq >/dev/null 2>&1; then exit 1 fi -# Lidarr runs on HOST1 only — exit cleanly if running on HOST2 +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + +# Lock before detect_hosts — large library scans take time, wait mode appropriate +[[ -n "${LIDARR_LOCK_WARN_AGE:-}" ]] && LOCK_WARN_AGE="$LIDARR_LOCK_WARN_AGE" +acquire_lock "wait" + +# detect_hosts() sets MY_ID and aliases LIDARR_URL, LIDARR_API_KEY, LIDARR_MUSIC_ROOT detect_hosts -if [[ "$LOCAL_SERVER_NAME" != "$HOST1" ]]; then - info "Lidarr runs on $HOST1 only — skipping on $LOCAL_SERVER_NAME" +# Lidarr is HOST1 only — exit cleanly on any other host +if [[ "$MY_ID" != "HOST1" ]]; then + log "Lidarr runs on HOST1 only — skipping on $MY_ID ($LOCAL_SERVER_NAME)" exit 0 fi -LIDARR_URL="$HOST1_LIDARR_URL" -LIDARR_API_KEY="$HOST1_LIDARR_API_KEY" -LIDARR_MUSIC_ROOT="$HOST1_LIDARR_MUSIC_ROOT" -LIDARR_CONTAINER="Lidarr" +DOCKER_TIMEOUT=15 +LIDARR_CONTAINER="Lidarr" # container name on HOST1 -# Load path map — translates container paths from API to host paths on disk +# Build path map from HOST1_LIDARR_PATH_MAP for translate_path() declare -A ARR_PATH_MAP -if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - for key in "${!HOST1_LIDARR_PATH_MAP[@]}"; do - ARR_PATH_MAP["$key"]="${HOST1_LIDARR_PATH_MAP[$key]}" - done -else - for key in "${!HOST2_LIDARR_PATH_MAP[@]}"; do - ARR_PATH_MAP["$key"]="${HOST2_LIDARR_PATH_MAP[$key]}" - done -fi - -info "Lidarr instance: $LOCAL_SERVER_NAME → $LIDARR_URL" -info "Music root: $LIDARR_MUSIC_ROOT" +for key in "${!HOST1_LIDARR_PATH_MAP[@]}"; do + ARR_PATH_MAP["$key"]="${HOST1_LIDARR_PATH_MAP[$key]}" +done +# Validate required vars — detect_hosts() should have set these require_var LIDARR_URL require_var LIDARR_API_KEY require_var LIDARR_MUSIC_ROOT if [[ ! -d "$LIDARR_MUSIC_ROOT" ]]; then error "Music root not found: $LIDARR_MUSIC_ROOT" - notify "Lidarr cleanup failed on $(hostname) — music root not found: $LIDARR_MUSIC_ROOT" "Lidarr Cleanup" "warning" + notify "Lidarr cleanup failed on $(hostname) — music root not found: $LIDARR_MUSIC_ROOT" \ + "Lidarr Cleanup" "warning" exit 1 fi -# Large library scans take time — override default lock warn age -[[ -n "${LIDARR_LOCK_WARN_AGE:-}" ]] && LOCK_WARN_AGE="$LIDARR_LOCK_WARN_AGE" +log "Lidarr URL: $LIDARR_URL" +log "Music root: $LIDARR_MUSIC_ROOT" -acquire_lock "wait" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted" +[[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing active" +[[ "$SKIP_STRIKES" == true ]] && warn "OVERRIDE — --skip-strike-list active — age check bypassed" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SHIELD Safety Layer 1 — Container Health ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_SHIELD Safety Checks ━━━" - -# Check container is running -CONTAINER_RUNNING=$(docker inspect -f '{{.State.Running}}' "$LIDARR_CONTAINER" 2>/dev/null) -if [[ "$CONTAINER_RUNNING" != "true" ]]; then - error "$LIDARR_CONTAINER container is not running — aborting" - notify "Lidarr cleanup aborted on $(hostname) — Lidarr container not running" "Lidarr Cleanup" "warning" - exit 1 -fi -success "$LIDARR_CONTAINER is running" - -# Check container health — only fail if explicitly unhealthy -CONTAINER_HEALTH=$(docker inspect -f '{{.State.Health.Status}}' "$LIDARR_CONTAINER" 2>/dev/null) -case "$CONTAINER_HEALTH" in - healthy) - success "$LIDARR_CONTAINER is healthy" ;; - "") - info "$LIDARR_CONTAINER has no health check configured — proceeding" ;; - starting) - error "$LIDARR_CONTAINER is still starting — aborting" - notify "Lidarr cleanup aborted on $(hostname) — Lidarr container still starting" "Lidarr Cleanup" "warning" - exit 1 ;; - unhealthy) - error "$LIDARR_CONTAINER is unhealthy — aborting" - notify "Lidarr cleanup aborted on $(hostname) — Lidarr container unhealthy" "Lidarr Cleanup" "warning" - exit 1 ;; - *) - warn "$LIDARR_CONTAINER health status: $CONTAINER_HEALTH — proceeding with caution" ;; -esac - -success "All safety checks passed" -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted" -[[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing flag active" -[[ "$SKIP_STRIKES" == true ]] && warn "OVERRIDE — --skip-strike-list flag active — strike system bypassed" - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_GEAR Lidarr URL: $LIDARR_URL" - echo "$ICON_GEAR Music root: $LIDARR_MUSIC_ROOT" - echo "$ICON_TIME Orphan age: ${LIDARR_ORPHAN_AGE} days" - echo "$ICON_GEAR Max delete: ${LIDARR_MAX_DELETE_GB}GB (requires --i-know-what-im-doing)" - echo "$ICON_GEAR Min tracked %: ${LIDARR_MIN_TRACKED_PCT}% of last run" - echo "$ICON_GEAR Extensions: ${LIDARR_EXTENSIONS[*]}" - echo "$ICON_GEAR Protected patterns: ${LIDARR_PROTECTED_PATTERNS[*]}" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_GEAR Lidarr URL: $LIDARR_URL" + echo "$ICON_GEAR Music root: $LIDARR_MUSIC_ROOT" + echo "$ICON_TIME Orphan age: ${LIDARR_ORPHAN_AGE} days" + echo "$ICON_GEAR Max delete: ${LIDARR_MAX_DELETE_GB}GB (requires --i-know-what-im-doing)" + echo "$ICON_GEAR Min tracked %: ${LIDARR_MIN_TRACKED_PCT}%" + echo "$ICON_GEAR Lidarr ver: v${LIDARR_VERSION_MAJOR} expected" + echo "$ICON_GEAR Extensions: ${LIDARR_EXTENSIONS[*]}" + echo "$ICON_GEAR Protected patterns: ${LIDARR_PROTECTED_PATTERNS[*]}" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_GEAR I know: $I_KNOW" + echo "$ICON_GEAR Skip strikes: $SKIP_STRIKES" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -# ----------------------------------------------------------------------------------------------- -# HELPERS -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Safety Layer 1 — Container Health ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_SHIELD Safety Checks ━━━" +CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ + '{{.State.Running}}' "$LIDARR_CONTAINER" 2>/dev/null) +if [[ "$CONTAINER_RUNNING" != "true" ]]; then + error "$LIDARR_CONTAINER is not running — aborting" + notify "Lidarr cleanup aborted on $(hostname) — container not running" \ + "Lidarr Cleanup" "warning" + exit 1 +fi + +CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ + '{{.State.Health.Status}}' "$LIDARR_CONTAINER" 2>/dev/null) +case "$CONTAINER_HEALTH" in + healthy) log "$LIDARR_CONTAINER is healthy" ;; + "") log "$LIDARR_CONTAINER has no health check — proceeding" ;; + starting) + error "$LIDARR_CONTAINER is still starting — aborting" + notify "Lidarr cleanup aborted on $(hostname) — container still starting" \ + "Lidarr Cleanup" "warning" + exit 1 ;; + unhealthy) + error "$LIDARR_CONTAINER is unhealthy — aborting" + notify "Lidarr cleanup aborted on $(hostname) — container unhealthy" \ + "Lidarr Cleanup" "warning" + exit 1 ;; + *) warn "$LIDARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;; +esac + +log "Safety layer 1 passed — container healthy" + +# ============================================================================================== +# ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# Lidarr API call with HTTP status check +# Usage: lidarr_api "artist" | lidarr_api "trackFile?artistId=123" lidarr_api() { local endpoint="$1" local response http_code body @@ -235,13 +266,13 @@ lidarr_api() { body=$(echo "$response" | head -n -1) if [[ "$http_code" != "200" ]]; then - error "Lidarr API returned HTTP $http_code for endpoint: $endpoint" + error "Lidarr API HTTP $http_code for: $endpoint" return 1 fi - echo "$body" } +# Check if a file extension is a tracked music format is_music_file() { local ext="${1##*.}" ext="${ext,,}" @@ -251,6 +282,7 @@ is_music_file() { return 1 } +# Check if a file matches any protected pattern is_protected_file() { local filename filename=$(basename "$1") @@ -263,38 +295,43 @@ is_protected_file() { return 1 } -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SYNC Fetching Lidarr Tracked Files ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Fetch Lidarr Tracked Files ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_SYNC Fetching Lidarr Tracked Files ━━━" # Safety Layer 2 — API reachability -check_api "$LIDARR_URL" "Lidarr" || { +if ! check_api "$LIDARR_URL" "Lidarr" 10; then notify "Lidarr cleanup aborted on $(hostname) — API unreachable" "Lidarr Cleanup" "warning" exit 1 -} +fi -info "Querying Lidarr API: $LIDARR_URL" +# Safety Layer 3 — API version check +check_arr_version "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$LIDARR_VERSION_MAJOR" "Lidarr" || exit 1 -# Step 1 — Get all artists +log "Querying Lidarr API: $LIDARR_URL" + +# Fetch all artists ARTIST_RESPONSE=$(lidarr_api "artist") || { - error "Failed to fetch artists from Lidarr — check URL and API key" - notify "Lidarr cleanup failed on $(hostname) — could not fetch artists" "Lidarr Cleanup" "warning" + error "Failed to fetch artists from Lidarr" + notify "Lidarr cleanup failed on $(hostname) — could not fetch artists" \ + "Lidarr Cleanup" "warning" exit 1 } ARTIST_IDS=$(echo "$ARTIST_RESPONSE" | jq -r '.[].id' 2>/dev/null) ARTIST_COUNT=$(echo "$ARTIST_IDS" | grep -c "[0-9]" 2>/dev/null || echo 0) -# Safety Layer 3 — artist count +# Safety Layer 4 — artist count > 0 if [[ "$ARTIST_COUNT" -eq 0 ]]; then - error "No artists returned from Lidarr API — aborting to prevent mass deletion" - notify "Lidarr cleanup aborted on $(hostname) — API returned 0 artists" "Lidarr Cleanup" "warning" + error "API returned 0 artists — aborting to prevent mass deletion" + notify "Lidarr cleanup aborted on $(hostname) — 0 artists returned" \ + "Lidarr Cleanup" "warning" exit 1 fi -info "Found $ARTIST_COUNT artists — fetching track files..." +log "Found $ARTIST_COUNT artists — fetching track files..." TMP_DIR="/tmp/lidarr_cleanup_$$" mkdir -p "$TMP_DIR" @@ -306,57 +343,65 @@ TRACKED_FILE="$TMP_DIR/tracked_paths.txt" while IFS= read -r artist_id; do [[ -z "$artist_id" ]] && continue ARTIST_TRACKS=$(lidarr_api "trackFile?artistId=${artist_id}" 2>/dev/null) - if [[ -n "$ARTIST_TRACKS" ]]; then if [[ -n "$ARTIST_TRACKS" ]]; then while IFS= read -r api_path; do [[ -z "$api_path" ]] && continue translate_path "$api_path" >> "$TRACKED_FILE" done < <(echo "$ARTIST_TRACKS" | jq -r '.[].path' 2>/dev/null) fi - fi done <<< "$ARTIST_IDS" sort -u "$TRACKED_FILE" -o "$TRACKED_FILE" + +# Build in-memory lookup map — O(1) per lookup vs O(n) grep per file +# Eliminates the main performance bottleneck for large libraries +declare -A TRACKED_MAP +while IFS= read -r _tracked_path; do + [[ -n "$_tracked_path" ]] && TRACKED_MAP["$_tracked_path"]=1 +done < "$TRACKED_FILE" +unset _tracked_path +log "Built in-memory lookup map: ${#TRACKED_MAP[@]} tracked paths" TRACKED_COUNT=$(wc -l < "$TRACKED_FILE") -# Safety Layer 4 — tracked file count +# Safety Layer 5 — tracked count > 0 if [[ "$TRACKED_COUNT" -eq 0 ]]; then error "API returned 0 tracked files — aborting to prevent mass deletion" - notify "Lidarr cleanup aborted on $(hostname) — API returned 0 tracked files" "Lidarr Cleanup" "warning" + notify "Lidarr cleanup aborted on $(hostname) — 0 tracked files returned" \ + "Lidarr Cleanup" "warning" exit 1 fi -success "Lidarr tracks $TRACKED_COUNT files across $ARTIST_COUNT artists" +warn "Lidarr tracks $TRACKED_COUNT files across $ARTIST_COUNT artists" -# Safety Layer 5 — percentage drop check against last known count +# Safety Layer 6 — percentage drop vs last known count if [[ -f "$LIDARR_TRACKED_COUNT_FILE" ]]; then LAST_COUNT=$(cat "$LIDARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0) if [[ "$LAST_COUNT" -gt 0 ]]; then PCT=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $LAST_COUNT) * 100}") if [[ "$PCT" -lt "$LIDARR_MIN_TRACKED_PCT" ]]; then - error "Tracked file count dropped to ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT)" - error "This suggests an API issue — aborting to prevent mass deletion" - error "If this is expected (large library removal), delete: $LIDARR_TRACKED_COUNT_FILE" - notify "Lidarr cleanup aborted on $(hostname) — tracked count dropped to ${PCT}% of last run" "Lidarr Cleanup" "warning" + error "Tracked count dropped to ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT)" + error "Suggests API issue — aborting to prevent mass deletion" + error "If expected (large library removal) delete: $LIDARR_TRACKED_COUNT_FILE" + notify "Lidarr cleanup aborted on $(hostname) — tracked count dropped to ${PCT}%" \ + "Lidarr Cleanup" "warning" exit 1 fi - info "Tracked count check: ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT) ✅" + log "Tracked count: ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT) ✅" fi else - info "No previous count on record — first run, saving baseline" + log "No previous count on record — first run, saving baseline" fi -# Save current count for next run comparison echo "$TRACKED_COUNT" > "$LIDARR_TRACKED_COUNT_FILE" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_CLEAN Scanning Music Root ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Scan Music Root ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_CLEAN Scanning Music Root ━━━" -info "Root: $LIDARR_MUSIC_ROOT" -info "Orphan age: ${LIDARR_ORPHAN_AGE} days" -info "Protected: ${LIDARR_PROTECTED_PATTERNS[*]}" +log "Root: $LIDARR_MUSIC_ROOT" +log "Orphan age: ${LIDARR_ORPHAN_AGE} days" +log "Protected: ${LIDARR_PROTECTED_PATTERNS[*]}" echo "" START=$(date +%s) @@ -374,14 +419,16 @@ MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $LIDARR_MAX_DELETE_GB * 1073741824 while IFS= read -r filepath; do [[ -z "$filepath" ]] && continue - if grep -qF "$filepath" "$TRACKED_FILE" 2>/dev/null; then + # Tracked — leave alone + if [[ -n "${TRACKED_MAP[$filepath]:-}" ]]; then log "TRACKED: $filepath" continue fi + # Protected — never delete if is_protected_file "$filepath"; then log "$ICON_PROTECTED PROTECTED: $filepath" - ((PROTECTED_COUNT++)) + (( PROTECTED_COUNT++ )) continue fi @@ -393,17 +440,17 @@ while IFS= read -r filepath; do if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]]; then log "RECENT (skipping): $filepath" - ((RECENT_COUNT++)) + (( RECENT_COUNT++ )) continue fi warn "$ICON_TRASH ORPHAN: $filepath" - ((ORPHAN_COUNT++)) - ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE)) + (( ORPHAN_COUNT++ )) + ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE )) else log "JUNK: $filepath" - ((JUNK_COUNT++)) - JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE)) + (( JUNK_COUNT++ )) + JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE )) fi done < <(find "$LIDARR_MUSIC_ROOT" -type f 2>/dev/null) @@ -411,47 +458,48 @@ done < <(find "$LIDARR_MUSIC_ROOT" -type f 2>/dev/null) TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES )) TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) -# ----------------------------------------------------------------------------------------------- -# Safety Layer 6 — deletion size threshold -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━ +# ============================================================================================== if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}") if [[ "$I_KNOW" != true ]]; then echo "" - error "Deletion would exceed ${LIDARR_MAX_DELETE_GB}GB threshold — $TOTAL_HUMAN would be deleted" - error "Review the ORPHAN lines above carefully before proceeding" - error "If this is expected, rerun with: --i-know-what-im-doing" - error "To also bypass age check and delete on first pass: add --skip-strike-list" - notify "Lidarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} deletion requires --i-know-what-im-doing" "Lidarr Cleanup" "warning" + error "Deletion would exceed ${LIDARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted" + error "Review ORPHAN lines above carefully before proceeding" + error "Rerun with: --i-know-what-im-doing" + error "To also bypass age check: add --skip-strike-list" + notify "Lidarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \ + "Lidarr Cleanup" "warning" exit 1 else - warn "OVERRIDE — deletion is ${TOTAL_HUMAN} — proceeding because --i-know-what-im-doing" + warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing" fi fi -# All safety layers passed — execute deletions +# ── Execute Deletions ───────────────────────────────────────────────────────────────────────── +# All safety layers passed — delete orphans and junk if [[ "$DRY_RUN" == false ]]; then while IFS= read -r filepath; do [[ -z "$filepath" ]] && continue - - if grep -qF "$filepath" "$TRACKED_FILE" 2>/dev/null; then continue; fi - if is_protected_file "$filepath"; then continue; fi + [[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue + is_protected_file "$filepath" && continue FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0) FILE_AGE=$(( NOW - FILE_MTIME )) if is_music_file "$filepath"; then - [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]] && continue + [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \ + [[ "$SKIP_STRIKES" != true ]] && continue fi rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath" done < <(find "$LIDARR_MUSIC_ROOT" -type f 2>/dev/null) - echo "" - info "Cleaning up empty folders..." + log "Cleaning up empty folders..." find "$LIDARR_MUSIC_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null - success "Empty folders removed" + log "Empty folders removed" fi END=$(date +%s) @@ -467,35 +515,39 @@ format_bytes() { fi } -ORPHAN_HUMAN=$(format_bytes $ORPHAN_BYTES) -JUNK_HUMAN=$(format_bytes $JUNK_BYTES) +ORPHAN_HUMAN=$(format_bytes "$ORPHAN_BYTES") +JUNK_HUMAN=$(format_bytes "$JUNK_BYTES") -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY LIDARR CLEANUP SUMMARY ━━━━━" -echo "$ICON_SYNC Tracked by Lidarr: $TRACKED_COUNT files ($ARTIST_COUNT artists)" -echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (cover art, metadata)" -echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)" -echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)" -echo "$ICON_TIME Recent skipped: $RECENT_COUNT files (under ${LIDARR_ORPHAN_AGE} days)" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_SYNC Tracked: $TRACKED_COUNT files ($ARTIST_COUNT artists)" +echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (cover art, metadata)" +echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)" +echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)" +echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${LIDARR_ORPHAN_AGE} days)" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" + if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no files deleted" + warn "DRY RUN — no files deleted" elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then - echo "$ICON_DONE Status: $ICON_SUCCESS CLEAN — nothing to remove" - notify "Lidarr cleanup complete on $(hostname) — library is clean" "Lidarr Cleanup" "normal" + log "$ICON_DONE Clean — nothing to remove" else - echo "$ICON_DONE Status: $ICON_SUCCESS DONE — $TOTAL_REMOVED files removed (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" - notify "Lidarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "normal" + warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" + notify "Lidarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "warning" + # Notify Emby to clean missing files — removes ghost entries immediately + notify_emby_scan fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Write stats for sunday_morning_coffee_report.sh if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_CLEANUP_STATS:-}" ]]; then - DATE=$(date '+%Y-%m-%d') - echo "${DATE}|lidarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \ + echo "$(date '+%Y-%m-%d')|lidarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \ >> "$ARR_CLEANUP_STATS" 2>/dev/null || true fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file + +exit 0 \ No newline at end of file diff --git a/Media/media_cleaner.sh b/Media/media_cleaner.sh index e3afbd5..9d4cc69 100644 --- a/Media/media_cleaner.sh +++ b/Media/media_cleaner.sh @@ -1,42 +1,81 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Media Cleaner Script --------------------------------------- -# ----------------------------------------------------------------------------------------------- -# Removes unwanted files from media share folders using configurable file patterns. -# Supports two profiles: anime and media — each with their own folder list and file patterns. -# Profiles and patterns are configured in Master.conf. -# Supports --dry-run to preview what would be deleted without making changes. +# ============================================================================================== +# ================================= Media Cleaner ============================================== +# ============================================================================================== +# Removes unwanted junk files from media share folders using configurable file patterns. +# Two profiles — anime and media — each with their own folder list and file patterns. +# Runs daily via DAILY_MAINTENANCE_SCRIPTS after media_shares_permissions.sh. # -# Usage: -# media_cleaner.sh anime — clean anime shares -# media_cleaner.sh media — clean media shares -# media_cleaner.sh anime --dry-run — preview anime clean -# ----------------------------------------------------------------------------------------------- +# ── PROFILES ────────────────────────────────────────────────────────────────────────────────── +# anime — cleans ANIME_CLEAN_FOLDERS using ANIME_FILE_PATTERNS +# typical targets: *.sfv *.nfo *.url *.rar *.zip *.sample* etc. +# +# media — cleans MEDIA_CLEAN_FOLDERS using MEDIA_FILE_PATTERNS +# same patterns plus *.iso *.lrc (media-specific extras) +# +# ── WHAT IT REMOVES ─────────────────────────────────────────────────────────────────────────── +# Junk files left behind by download clients, scene releases, and various tools: +# *.sfv *.md5 *.sha1 — checksum verification files — useless post-download +# *.nfo *.url *.lnk — scene info files — not needed in media library +# *.rar *.zip — archives — source files not needed after extraction +# *.sample* *.proof* — scene samples — never needed +# *sync-conflict* — Syncthing conflict files +# *.scr *.exe — executables — should never be in a media folder +# *.torrent — torrent files left by download clients +# *.log *.json — tool output files +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID and aliases HOST*_ANIME_CLEAN_FOLDERS → ANIME_CLEAN_FOLDERS +# and HOST*_MEDIA_CLEAN_FOLDERS → MEDIA_CLEAN_FOLDERS. +# Each server only cleans the shares it owns — correct folders per host automatically. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock "wait" — wait if previous run still active +# detect_hosts() — correct folder lists per host via MY_ID aliases +# Empty array guards — warns and exits cleanly if no folders or patterns configured +# Folder existence — skips missing folders with warning, continues others +# validate_unraid_cmd — notify script validated before use +# Silent by default — only problems and removals produce output +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_ANIME_CLEAN_FOLDERS — folders cleaned by the anime profile on this host +# HOST*_MEDIA_CLEAN_FOLDERS — folders cleaned by the media profile on this host +# Aliased by detect_hosts() — script uses ANIME_CLEAN_FOLDERS / MEDIA_CLEAN_FOLDERS +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# ANIME_FILE_PATTERNS — file patterns removed by the anime profile +# MEDIA_FILE_PATTERNS — file patterns removed by the media profile +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# media_cleaner.sh anime — clean anime shares +# media_cleaner.sh media — clean media shares +# media_cleaner.sh anime --dry-run — preview anime clean, no deletions +# media_cleaner.sh media --dry-run — preview media clean, no deletions +# media_cleaner.sh anime --log — verbose output +# media_cleaner.sh anime --status — show config and exit +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" -# ----------------------------------------------------------------------------------------------- -# Separate profile argument from flags -# ----------------------------------------------------------------------------------------------- +# ── Separate profile argument from flags ────────────────────────────────────────────────────── +# Profile (anime|media) is a positional arg — separate before parse_args sees flags PROFILE="" RAW_ARGS=() for ARG in "$@"; do case "$ARG" in - --*|*=*) RAW_ARGS+=("$ARG") ;; anime|media) PROFILE="$ARG" ;; - *) RAW_ARGS+=("$ARG") ;; + *) RAW_ARGS+=("$ARG") ;; esac done parse_args "${RAW_ARGS[@]}" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" @@ -45,19 +84,21 @@ if [[ "$EUID" -ne 0 ]]; then exit 1 fi -success "Running as root" +if [[ -z "$PROFILE" ]]; then + error "No profile specified" + error "Usage: media_cleaner.sh [--dry-run] [--log] [--status]" + exit 1 +fi acquire_lock "wait" -if ! command -v find >/dev/null 2>&1; then - error "find command not found — check findutils installation" - exit 1 -fi +# detect_hosts() sets MY_ID and aliases HOST*_ANIME/MEDIA_CLEAN_FOLDERS +detect_hosts -if [[ -z "$PROFILE" ]]; then - error "No profile specified. Usage: media_cleaner.sh [--dry-run]" - exit 1 -fi +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" # Resolve profile folders and patterns case "$PROFILE" in @@ -75,18 +116,35 @@ case "$PROFILE" in ;; esac -info "$ICON_GEAR Profile: $PROFILE" -info "$ICON_CLEAN Folders: ${#CLEAN_FOLDERS[@]}" -info "$ICON_TRASH Patterns: ${#FILE_PATTERNS[@]}" +# Empty array guards +if [[ ${#CLEAN_FOLDERS[@]} -eq 0 ]]; then + warn "No folders configured for profile '$PROFILE' on $MY_ID" + warn "Check HOST*_${PROFILE^^}_CLEAN_FOLDERS in master_host*.conf" + exit 0 +fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +if [[ ${#FILE_PATTERNS[@]} -eq 0 ]]; then + warn "No file patterns configured for profile '$PROFILE'" + warn "Check ${PROFILE^^}_FILE_PATTERNS in master.conf" + exit 0 +fi + +log "Profile: $PROFILE" +log "Folders: ${#CLEAN_FOLDERS[@]}" +log "Patterns: ${#FILE_PATTERNS[@]}" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_GEAR Profile: $PROFILE" - echo "$ICON_CLEAN Folders: ${CLEAN_FOLDERS[*]}" + echo "$ICON_CLEAN Folders:" + for f in "${CLEAN_FOLDERS[@]}"; do + echo " $f" + done echo "$ICON_TRASH Patterns: ${FILE_PATTERNS[*]}" echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "━━━━━━━━━━━━━━━━━━━━━━━" @@ -95,11 +153,11 @@ fi [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_CLEAN Media Cleaner ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Media Cleaner ━━━ +# ============================================================================================== echo "" -echo "━━━ $ICON_CLEAN Media Cleaner — $PROFILE ━━━" +echo "━━━ $ICON_CLEAN Media Cleaner — $PROFILE — $MY_ID ━━━" echo "" START=$(date +%s) @@ -120,7 +178,7 @@ for FOLDER in "${CLEAN_FOLDERS[@]}"; do # Build find command dynamically from FILE_PATTERNS array CMD=(find "$FOLDER" -type f \() - for ((i = 0; i < ${#FILE_PATTERNS[@]}; i++)); do + for (( i = 0; i < ${#FILE_PATTERNS[@]}; i++ )); do CMD+=(-iname "${FILE_PATTERNS[i]}") if [[ $i -lt $(( ${#FILE_PATTERNS[@]} - 1 )) ]]; then CMD+=(-o) @@ -132,12 +190,12 @@ for FOLDER in "${CLEAN_FOLDERS[@]}"; do FILE_COUNT=$("${CMD[@]}" 2>/dev/null | wc -l) if [[ "$FILE_COUNT" -eq 0 ]]; then - success "$FOLDER_NAME — no matching files found" + log "$FOLDER_NAME — clean ✅" echo "" continue fi - info "$ICON_TRASH $FILE_COUNT file(s) found in $FOLDER_NAME" + warn "$ICON_TRASH $FILE_COUNT file(s) to remove from $FOLDER_NAME" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — files that would be deleted:" @@ -147,8 +205,8 @@ for FOLDER in "${CLEAN_FOLDERS[@]}"; do else CLEAN_CMD=("${CMD[@]}" -exec rm -f {} +) if "${CLEAN_CMD[@]}" 2>/dev/null; then - success "$FOLDER_NAME — $FILE_COUNT file(s) removed" - TOTAL_REMOVED=$((TOTAL_REMOVED + FILE_COUNT)) + log "$FOLDER_NAME — $FILE_COUNT file(s) removed" + TOTAL_REMOVED=$(( TOTAL_REMOVED + FILE_COUNT )) else error "$FOLDER_NAME — cleanup failed" FAILED+=("$FOLDER_NAME") @@ -160,24 +218,28 @@ done END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "━━━━━ $ICON_SUMMARY MEDIA CLEANER SUMMARY ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_GEAR Profile: $PROFILE" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_WARN Skipped: ${SKIPPED[*]}" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" +[[ ${#SKIPPED[@]} -gt 0 ]] && warn "Skipped: ${SKIPPED[*]} (folders not found)" [[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no files deleted" + warn "DRY RUN — no files deleted" elif [[ ${#FAILED[@]} -gt 0 ]]; then - echo "$ICON_ERROR Status: $ICON_ERROR SOME FOLDERS FAILED" - notify "Media cleaner ($PROFILE) failed on $(hostname) — ${FAILED[*]}" "Media Cleaner" "warning" + echo "$ICON_ERROR Status: SOME FOLDERS FAILED" + notify "Media cleaner ($PROFILE) failed on $(hostname) — ${FAILED[*]}" \ + "Media Cleaner" "warning" +elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then + log "$ICON_DONE Status: clean — nothing to remove" else - echo "$ICON_TRASH Removed: $TOTAL_REMOVED file(s)" - echo "$ICON_DONE Status: $ICON_SUCCESS DONE" - notify "Media cleaner ($PROFILE) complete on $(hostname) — $TOTAL_REMOVED file(s) removed" "Media Cleaner" "normal" + warn "$ICON_TRASH Removed: $TOTAL_REMOVED file(s)" + notify "Media cleaner ($PROFILE) on $(hostname) — $TOTAL_REMOVED file(s) removed" \ + "Media Cleaner" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" diff --git a/Media/media_shares_permissions.sh b/Media/media_shares_permissions.sh index 86b2a50..ab826d2 100644 --- a/Media/media_shares_permissions.sh +++ b/Media/media_shares_permissions.sh @@ -1,22 +1,78 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Media Permissions Script ----------------------------------- -# ----------------------------------------------------------------------------------------------- -# Applies permissions and ownership to all configured media shares. -# Shares, permissions mode and owner are configured in Master.conf. -# Supports --dry-run to preview what would be changed without making changes. -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ============================= Media Shares Permissions ======================================= +# ============================================================================================== +# Applies correct ownership and permissions to all configured media shares. +# Runs daily via DAILY_MAINTENANCE_SCRIPTS — first job before arr cleanup scripts. +# Arr cleanup depends on correct ownership to rename and delete files safely. +# +# ── WHY THIS EXISTS ─────────────────────────────────────────────────────────────────────────── +# Originally a band-aid for 777 permissions caused by containers running as root. +# Now a proper daily failsafe — even with correct container config, files can arrive +# with wrong ownership from: +# - rsync without --chown (brings source server's ownership) +# - Manual admin copies (creates root:root files) +# - New containers not yet configured with correct PUID/PGID +# - unRAID updates that reset container environments +# +# ── PERMISSIONS MODEL ───────────────────────────────────────────────────────────────────────── +# Directories: 755 nobody:users +# Owner (nobody) — rwx enter, list, create files ✅ +# Group (users) — r-x enter and list ✅ +# Others — r-x Samba guests can browse ✅ +# No world-write — prevents accidental deletion by unauthenticated access +# +# Files: 664 nobody:users +# Owner (nobody) — rw read + write ✅ +# Group (users) — rw arrs can import/rename ✅ +# Others — r Samba guests can read ✅ +# No execute bit — media files are never executable ✅ +# +# ── DIAGNOSTIC — HIGH CORRECTED COUNT ───────────────────────────────────────────────────────── +# If this script corrects many files every run, a container has wrong PUID/PGID: +# Correct values on unRAID: PUID=99 (nobody) PGID=100 (users) +# Add to each container's environment in its Docker template +# Common culprits: SABnzbd, qBittorrent, slskd — check these first +# Once fixed, this script should correct 0 files per run (pure failsafe) +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID and aliases HOST*_MEDIA_PERMISSION_SHARES → MEDIA_PERMISSION_SHARES +# Each server only applies permissions to the shares it owns. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock "wait" — wait if previous run still active (large share scans take time) +# detect_hosts() — correct share list per host via MY_ID aliases +# Empty array guard — warns and exits cleanly if no shares configured +# Folder existence — skips missing shares with warning, continues others +# Separate passes — directories and files chmod'd separately for correctness +# validate_unraid_cmd — notify script validated before use +# Silent by default — only failures produce output, success is silent +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_MEDIA_PERMISSION_SHARES — shares this host applies permissions to +# Aliased by detect_hosts() — script uses MEDIA_PERMISSION_SHARES +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# PERMISSIONS_DIR_MODE — directory permissions (default 755) +# PERMISSIONS_FILE_MODE — file permissions (default 664) +# PERMISSIONS_OWNER — ownership applied to all files (default nobody:users) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# media_shares_permissions.sh — normal run +# media_shares_permissions.sh --dry-run — preview without making changes +# media_shares_permissions.sh --log — verbose output +# media_shares_permissions.sh --status — show config and exit +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" @@ -25,95 +81,159 @@ if [[ "$EUID" -ne 0 ]]; then exit 1 fi -success "Running as root" +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- -if [[ "$SHOW_STATUS" == true ]]; then - echo "" - echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_PERMS Mode: $PERMISSIONS_MODE" - echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER" - echo "$ICON_PERMS Shares: ${#MEDIA_PERMISSION_SHARES[@]}" - echo "$ICON_GEAR Dry Run: $DRY_RUN" - echo "━━━━━━━━━━━━━━━━━━━━━━━" +acquire_lock "wait" + +# detect_hosts() sets MY_ID and aliases HOST*_MEDIA_PERMISSION_SHARES +detect_hosts + +# Empty array guard +if [[ ${#MEDIA_PERMISSION_SHARES[@]} -eq 0 ]]; then + warn "MEDIA_PERMISSION_SHARES is empty for $MY_ID — nothing to do" + warn "Check HOST*_MEDIA_PERMISSION_SHARES in master_host*.conf" exit 0 fi [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" -acquire_lock "wait" +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_PERMS Dir mode: ${PERMISSIONS_DIR_MODE:-755}" + echo "$ICON_PERMS File mode: ${PERMISSIONS_FILE_MODE:-664}" + echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER" + echo "$ICON_PERMS Shares: ${#MEDIA_PERMISSION_SHARES[@]}" + echo "" + for share in "${MEDIA_PERMISSION_SHARES[@]}"; do + local_status="missing" + [[ -d "$share" ]] && local_status="exists" + echo " $share — $local_status" + done + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_PERMS Media Permissions ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Apply Permissions ━━━ +# ============================================================================================== echo "" -echo "━━━ $ICON_PERMS Media Permissions ━━━" -echo "$ICON_PERMS Mode: $PERMISSIONS_MODE" -echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER" -echo "$ICON_PERMS Shares: ${#MEDIA_PERMISSION_SHARES[@]}" +echo "━━━ $ICON_PERMS Media Permissions — $MY_ID ━━━" +log "Dir mode: ${PERMISSIONS_DIR_MODE:-755}" +log "File mode: ${PERMISSIONS_FILE_MODE:-664}" +log "Owner: $PERMISSIONS_OWNER" +log "Shares: ${#MEDIA_PERMISSION_SHARES[@]}" echo "" START=$(date +%s) FAILED=() UPDATED=() SKIPPED=() +TOTAL_DIRS_FIXED=0 +TOTAL_FILES_FIXED=0 for SHARE in "${MEDIA_PERMISSION_SHARES[@]}"; do SHARE_NAME=$(basename "$SHARE") if [[ ! -d "$SHARE" ]]; then - warn "$ICON_PERMS $SHARE_NAME not found — skipping" + warn "$SHARE_NAME not found — skipping" SKIPPED+=("$SHARE_NAME") continue fi if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would apply $PERMISSIONS_MODE $PERMISSIONS_OWNER to $SHARE" + # Count what would be changed without making changes + DIR_COUNT=$(find "$SHARE" -type d ! -perm "${PERMISSIONS_DIR_MODE:-755}" \ + 2>/dev/null | wc -l) + FILE_COUNT=$(find "$SHARE" -type f ! -perm "${PERMISSIONS_FILE_MODE:-664}" \ + 2>/dev/null | wc -l) + OWNER_COUNT=$(find "$SHARE" ! -user nobody -o ! -group users \ + 2>/dev/null | wc -l) + warn "DRY RUN — $SHARE_NAME: $DIR_COUNT dirs, $FILE_COUNT files, $OWNER_COUNT ownership fixes needed" continue fi - info "$ICON_PERMS Updating $SHARE_NAME..." + log "Updating $SHARE_NAME..." - CHMOD_OK=true + CHMOD_DIR_OK=true + CHMOD_FILE_OK=true CHOWN_OK=true - chmod -R "$PERMISSIONS_MODE" "$SHARE" 2>/dev/null || CHMOD_OK=false + # Count files with wrong ownership before fixing (diagnostic) + WRONG_OWNER=$(find "$SHARE" \( ! -user nobody -o ! -group users \) \ + 2>/dev/null | wc -l) + + # Apply ownership first — affects all files and directories chown -R "$PERMISSIONS_OWNER" "$SHARE" 2>/dev/null || CHOWN_OK=false - if [[ "$CHMOD_OK" == true && "$CHOWN_OK" == true ]]; then - echo "$ICON_UNLOCKED $SHARE_NAME — permissions applied" + # Apply directory permissions — separate pass for correctness + # Directories need execute bit — different from files + find "$SHARE" -type d -exec chmod "${PERMISSIONS_DIR_MODE:-755}" {} + \ + 2>/dev/null || CHMOD_DIR_OK=false + + # Apply file permissions — no execute bit on media files + find "$SHARE" -type f -exec chmod "${PERMISSIONS_FILE_MODE:-664}" {} + \ + 2>/dev/null || CHMOD_FILE_OK=false + + if [[ "$CHMOD_DIR_OK" == true && \ + "$CHMOD_FILE_OK" == true && \ + "$CHOWN_OK" == true ]]; then + log "$ICON_UNLOCKED $SHARE_NAME — permissions applied" UPDATED+=("$SHARE_NAME") + # Log diagnostic if many files had wrong ownership + if [[ "$WRONG_OWNER" -gt 0 ]]; then + warn "$SHARE_NAME — corrected $WRONG_OWNER file(s) with wrong ownership" + warn "If this is high, check container PUID/PGID settings (should be PUID=99 PGID=100)" + fi + TOTAL_DIRS_FIXED=$(( TOTAL_DIRS_FIXED + 1 )) + TOTAL_FILES_FIXED=$(( TOTAL_FILES_FIXED + WRONG_OWNER )) else - error "$SHARE_NAME — permissions failed (chmod=$CHMOD_OK chown=$CHOWN_OK)" + error "$SHARE_NAME — permissions failed" + error " chown: $CHOWN_OK chmod dirs: $CHMOD_DIR_OK chmod files: $CHMOD_FILE_OK" FAILED+=("$SHARE_NAME") fi done END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY MEDIA PERMISSIONS SUMMARY ━━━━━" -echo "$ICON_PERMS Mode: $PERMISSIONS_MODE" -echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -echo "" -[[ ${#UPDATED[@]} -gt 0 ]] && echo " $ICON_UNLOCKED Updated: ${#UPDATED[@]}" -[[ ${#SKIPPED[@]} -gt 0 ]] && echo " $ICON_WARN Skipped: ${#SKIPPED[@]}" -[[ ${#FAILED[@]} -gt 0 ]] && echo " $ICON_ERROR Failed: ${#FAILED[@]} — ${FAILED[*]}" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_PERMS Dir mode: ${PERMISSIONS_DIR_MODE:-755}" +echo "$ICON_PERMS File mode: ${PERMISSIONS_FILE_MODE:-664}" +echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" +[[ ${#UPDATED[@]} -gt 0 ]] && log "Updated: ${#UPDATED[@]} shares" +[[ ${#SKIPPED[@]} -gt 0 ]] && warn "Skipped: ${SKIPPED[*]} (not found)" +[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" + +# Diagnostic — high correction count indicates container PUID/PGID issue +if [[ "$TOTAL_FILES_FIXED" -gt 50 ]]; then + warn "$TOTAL_FILES_FIXED files had wrong ownership this run" + warn "High count suggests a container is not set to PUID=99 PGID=100" + warn "Common culprits: SABnzbd, qBittorrent, slskd — check container env vars" +fi + if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no changes made" + warn "DRY RUN — no changes made" elif [[ ${#FAILED[@]} -gt 0 ]]; then - echo "$ICON_ERROR Status: $ICON_ERROR SOME SHARES FAILED" - notify "Media permissions failed on $(hostname) — ${FAILED[*]}" "Media Permissions" "warning" + echo "$ICON_ERROR Status: SOME SHARES FAILED" + notify "Media permissions failed on $(hostname) — ${FAILED[*]}" \ + "Media Permissions" "warning" else - echo "$ICON_DONE Status: $ICON_SUCCESS DONE" - notify "Media permissions applied on $(hostname) — ${#UPDATED[@]} shares updated" "Media Permissions" "normal" + log "$ICON_DONE Status: done — ${#UPDATED[@]} shares updated" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" diff --git a/Media/radarr_cleanup.sh b/Media/radarr_cleanup.sh index a04f7a5..6ddbf5c 100644 --- a/Media/radarr_cleanup.sh +++ b/Media/radarr_cleanup.sh @@ -1,67 +1,109 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Radarr Cleanup Script -------------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= Radarr Cleanup ============================================= +# ============================================================================================== # Removes orphaned movie files from the library that Radarr no longer tracks. # Uses the Radarr API to build a complete list of tracked movie file paths then compares -# against what exists on disk — anything not tracked and older than RADARR_ORPHAN_AGE +# against what exists on disk — anything untracked and older than RADARR_ORPHAN_AGE # days is considered an orphan and deleted. # -# File classification: +# ── FILE CLASSIFICATION ─────────────────────────────────────────────────────────────────────── # TRACKED — Radarr API knows about this exact file path → leave it alone # PROTECTED — matches RADARR_PROTECTED_PATTERNS → never delete (artwork, subtitles, .nfo) # ORPHAN — video file, not tracked, older than RADARR_ORPHAN_AGE days → delete # JUNK — not a video extension, not protected → delete regardless of age # RECENT — not tracked, under RADARR_ORPHAN_AGE days old → skip (may be mid-import) # -# Why protected patterns matter: -# Radarr generates movie artwork (*.jpg), metadata (*.nfo) and manages subtitles -# (*.srt, *.sub, *.ass) but does not include these in its tracked file API response. -# Without protection these would be classified as orphans and deleted. +# ── WHY PROTECTED PATTERNS MATTER ───────────────────────────────────────────────────────────── +# Radarr generates movie artwork (*.jpg), metadata (*.nfo) and manages subtitles (*.srt, +# *.sub, *.ass) but does NOT include these in its tracked file API response. +# Without protection these would be classified as orphans and deleted — breaking +# Radarr and Emby metadata display. # +# ── SAFETY LAYERS — ALL MUST PASS BEFORE ANY FILE IS TOUCHED ───────────────────────────────── +# 1. Container must be running and not starting/unhealthy +# 2. API must be reachable +# 3. API version must match tested major version in master.conf +# 4. Movie count must be > 0 +# 5. Tracked file count must be > 0 +# 6. Deletion size must be < RADARR_MAX_DELETE_GB — or --i-know-what-im-doing required +# +# ── POST-DELETION ───────────────────────────────────────────────────────────────────────────── +# After files are deleted notify_emby_scan() triggers Emby "Clean Missing Files" task. +# Emby immediately removes ghost entries — no user-facing file-not-found errors. +# +# ── OVERRIDE FLAGS ──────────────────────────────────────────────────────────────────────────── +# --i-know-what-im-doing required when deletion exceeds RADARR_MAX_DELETE_GB +# --skip-strike-list bypasses RADARR_ORPHAN_AGE age check +# NUCLEAR MODE — both active: age + size bypass, deletes on first pass +# ⚠️ User accepts full responsibility — no recovery possible after deletion +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── # HOST1 Radarr manages Movies. HOST2 Radarr manages Anime_Movies. -# detect_hosts() selects the correct URL, API key, and root path at runtime. -# All configuration in Master.conf under Arr Cleanup section. -# Supports --dry-run to preview what would be deleted without making changes. -# ----------------------------------------------------------------------------------------------- +# detect_hosts() sets MY_ID and aliases RADARR_URL, RADARR_API_KEY, RADARR_MOVIES_ROOT. +# No manual HOST1/HOST2 comparisons — MY_ID routes correctly on any server. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock "wait" — large scans take time, wait for previous run to finish +# jq + curl validation — exits if either tool missing +# DOCKER_TIMEOUT — container checks protected against daemon hangs +# 6 safety layers — all must pass before any file is touched +# notify_emby_scan() — triggers Emby clean after deletion +# validate_unraid_cmd — notify script validated before use +# Silent by default — orphans/junk warn(), clean library logs silently +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_RADARR_URL / HOST*_RADARR_API_KEY / HOST*_RADARR_MOVIES_ROOT +# HOST*_RADARR_PATH_MAP — container path → host path translation +# All aliased by detect_hosts() — script uses unprefixed names +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# RADARR_ORPHAN_AGE — days before untracked file eligible for deletion +# RADARR_MAX_DELETE_GB — require --i-know-what-im-doing above this +# RADARR_EXTENSIONS — video file extensions considered for orphan classification +# RADARR_PROTECTED_PATTERNS — file patterns never deleted +# RADARR_VERSION_MAJOR — expected Radarr major version for API safety check +# ARR_CLEANUP_STATS — stats file path (read by coffee report) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# radarr_cleanup.sh — normal run +# radarr_cleanup.sh --dry-run — preview, no deletions +# radarr_cleanup.sh --log — verbose output +# radarr_cleanup.sh --status — show config and exit +# radarr_cleanup.sh --i-know-what-im-doing — bypass size threshold +# radarr_cleanup.sh --i-know-what-im-doing --skip-strike-list — NUCLEAR MODE +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" -# Check for --i-know-what-im-doing and --skip-strike-list flags before parse_args +# ── Special flag pre-processing ─────────────────────────────────────────────────────────────── I_KNOW=false SKIP_STRIKES=false FILTERED_ARGS=() for arg in "$@"; do - if [[ "$arg" == "--i-know-what-im-doing" ]]; then - I_KNOW=true - elif [[ "$arg" == "--skip-strike-list" ]]; then - SKIP_STRIKES=true - else - FILTERED_ARGS+=("$arg") - fi + case "$arg" in + --i-know-what-im-doing) I_KNOW=true ;; + --skip-strike-list) SKIP_STRIKES=true ;; + *) FILTERED_ARGS+=("$arg") ;; + esac done parse_args "${FILTERED_ARGS[@]}" -# Nuclear mode disclaimer +# ── Nuclear mode warning ────────────────────────────────────────────────────────────────────── if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" != true ]]; then echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "⚠️ WARNING — NUCLEAR MODE ACTIVE" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo " Flags: --i-know-what-im-doing --skip-strike-list" - echo " Strike system: BYPASSED — deletes on first pass" - echo " Size threshold: BYPASSED — no GB limit" - echo " Data recovery: NOT POSSIBLE after deletion" + echo " Flags: --i-know-what-im-doing --skip-strike-list" + echo " Strike system: BYPASSED — deletes on first pass" + echo " Size threshold: BYPASSED — no GB limit" + echo " Data recovery: NOT POSSIBLE after deletion" echo "" - echo " The script author takes no responsibility for data" - echo " loss when both flags are used together. This is a" - echo " 100% intentional action by the user." - echo "" - echo " Review the dry run output before proceeding." + echo " Review --dry-run output before proceeding." echo " You have 10 seconds to cancel (Ctrl+C)..." echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" sleep 10 @@ -69,9 +111,9 @@ if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" != echo "" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" @@ -80,8 +122,6 @@ if [[ "$EUID" -ne 0 ]]; then exit 1 fi -success "Running as root" - if ! command -v curl >/dev/null 2>&1; then error "curl not found — required for Radarr API calls" exit 1 @@ -93,33 +133,25 @@ if ! command -v jq >/dev/null 2>&1; then exit 1 fi -# Select correct Radarr instance based on which server is running this script +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + +acquire_lock "wait" + +# detect_hosts() sets MY_ID and aliases RADARR_URL, RADARR_API_KEY, RADARR_MOVIES_ROOT detect_hosts -if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - RADARR_URL="$HOST1_RADARR_URL" - RADARR_API_KEY="$HOST1_RADARR_API_KEY" - RADARR_MOVIES_ROOT="$HOST1_RADARR_MOVIES_ROOT" -else - RADARR_URL="$HOST2_RADARR_URL" - RADARR_API_KEY="$HOST2_RADARR_API_KEY" - RADARR_MOVIES_ROOT="$HOST2_RADARR_MOVIES_ROOT" -fi +DOCKER_TIMEOUT=15 +RADARR_CONTAINER="Radarr" -# Load path map for this host +# Build path map from MY_ID's Radarr path map declare -A ARR_PATH_MAP -if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - for key in "${!HOST1_RADARR_PATH_MAP[@]}"; do - ARR_PATH_MAP["$key"]="${HOST1_RADARR_PATH_MAP[$key]}" - done -else - for key in "${!HOST2_RADARR_PATH_MAP[@]}"; do - ARR_PATH_MAP["$key"]="${HOST2_RADARR_PATH_MAP[$key]}" - done -fi - -info "Radarr instance: $LOCAL_SERVER_NAME → $RADARR_URL" -info "Movies root: $RADARR_MOVIES_ROOT" +local_path_map_var="${MY_ID}_RADARR_PATH_MAP" +eval "for key in \"\${!${local_path_map_var}[@]}\"; do + ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\" +done" require_var RADARR_URL require_var RADARR_API_KEY @@ -127,35 +159,77 @@ require_var RADARR_MOVIES_ROOT if [[ ! -d "$RADARR_MOVIES_ROOT" ]]; then error "Movies root not found: $RADARR_MOVIES_ROOT" - notify "Radarr cleanup failed on $(hostname) — movies root not found: $RADARR_MOVIES_ROOT" "Radarr Cleanup" "warning" + notify "Radarr cleanup failed on $(hostname) — movies root not found: $RADARR_MOVIES_ROOT" \ + "Radarr Cleanup" "warning" exit 1 fi -acquire_lock "wait" +log "Radarr URL: $RADARR_URL" +log "Movies root: $RADARR_MOVIES_ROOT" -success "Config validated" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted" +[[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing active" +[[ "$SKIP_STRIKES" == true ]] && warn "OVERRIDE — --skip-strike-list active — age check bypassed" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_GEAR Radarr URL: $RADARR_URL" - echo "$ICON_GEAR Movies root: $RADARR_MOVIES_ROOT" - echo "$ICON_TIME Orphan age: ${RADARR_ORPHAN_AGE} days" - echo "$ICON_GEAR Extensions: ${RADARR_EXTENSIONS[*]}" - echo "$ICON_GEAR Protected patterns: ${RADARR_PROTECTED_PATTERNS[*]}" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_GEAR Radarr URL: $RADARR_URL" + echo "$ICON_GEAR Movies root: $RADARR_MOVIES_ROOT" + echo "$ICON_TIME Orphan age: ${RADARR_ORPHAN_AGE} days" + echo "$ICON_GEAR Max delete: ${RADARR_MAX_DELETE_GB}GB (requires --i-know-what-im-doing)" + echo "$ICON_GEAR Radarr ver: v${RADARR_VERSION_MAJOR} expected" + echo "$ICON_GEAR Extensions: ${RADARR_EXTENSIONS[*]}" + echo "$ICON_GEAR Protected patterns: ${RADARR_PROTECTED_PATTERNS[*]}" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_GEAR I know: $I_KNOW" + echo "$ICON_GEAR Skip strikes: $SKIP_STRIKES" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted" +# ============================================================================================== +# ━━━ Safety Layer 1 — Container Health ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_SHIELD Safety Checks ━━━" -# ----------------------------------------------------------------------------------------------- -# HELPERS -# ----------------------------------------------------------------------------------------------- +CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ + '{{.State.Running}}' "$RADARR_CONTAINER" 2>/dev/null) +if [[ "$CONTAINER_RUNNING" != "true" ]]; then + error "$RADARR_CONTAINER is not running — aborting" + notify "Radarr cleanup aborted on $(hostname) — container not running" \ + "Radarr Cleanup" "warning" + exit 1 +fi + +CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ + '{{.State.Health.Status}}' "$RADARR_CONTAINER" 2>/dev/null) +case "$CONTAINER_HEALTH" in + healthy) log "$RADARR_CONTAINER is healthy" ;; + "") log "$RADARR_CONTAINER has no health check — proceeding" ;; + starting) + error "$RADARR_CONTAINER is still starting — aborting" + notify "Radarr cleanup aborted on $(hostname) — container still starting" \ + "Radarr Cleanup" "warning" + exit 1 ;; + unhealthy) + error "$RADARR_CONTAINER is unhealthy — aborting" + notify "Radarr cleanup aborted on $(hostname) — container unhealthy" \ + "Radarr Cleanup" "warning" + exit 1 ;; + *) warn "$RADARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;; +esac + +log "Safety layer 1 passed — container healthy" + +# ============================================================================================== +# ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== radarr_api() { local endpoint="$1" @@ -171,10 +245,9 @@ radarr_api() { body=$(echo "$response" | head -n -1) if [[ "$http_code" != "200" ]]; then - error "Radarr API returned HTTP $http_code for endpoint: $endpoint" + error "Radarr API HTTP $http_code for: $endpoint" return 1 fi - echo "$body" } @@ -199,36 +272,55 @@ is_protected_file() { return 1 } -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SYNC Fetching Radarr Tracked Files ━━━ -# ----------------------------------------------------------------------------------------------- +format_bytes() { + local bytes=$1 + if (( bytes > 1073741824 )); then + awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}" + elif (( bytes > 1048576 )); then + awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}" + else + echo "${bytes}B" + fi +} + +# ============================================================================================== +# ━━━ Fetch Radarr Tracked Files ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_SYNC Fetching Radarr Tracked Files ━━━" -check_api "$RADARR_URL" "Radarr" || { +# Safety Layer 2 — API reachability +if ! check_api "$RADARR_URL" "Radarr" 10; then notify "Radarr cleanup aborted on $(hostname) — API unreachable" "Radarr Cleanup" "warning" exit 1 -} +fi -info "Querying Radarr API: $RADARR_URL" +# Safety Layer 3 — API version check +check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" "Radarr" || exit 1 -# Step 1 — get all movies to extract IDs +log "Querying Radarr API: $RADARR_URL" + +# Fetch all movies MOVIES_RESPONSE=$(radarr_api "movie") || { - error "Failed to fetch movies from Radarr — check URL and API key" - notify "Radarr cleanup failed on $(hostname) — API unreachable" "Radarr Cleanup" "warning" + error "Failed to fetch movies from Radarr" + notify "Radarr cleanup failed on $(hostname) — could not fetch movies" \ + "Radarr Cleanup" "warning" exit 1 } MOVIE_IDS=$(echo "$MOVIES_RESPONSE" | jq -r '.[].id' 2>/dev/null) -MOVIE_COUNT=$(echo "$MOVIE_IDS" | grep -c . 2>/dev/null || echo 0) -info "Movies in Radarr: $MOVIE_COUNT" +MOVIE_COUNT=$(echo "$MOVIE_IDS" | grep -c "." 2>/dev/null || echo 0) +# Safety Layer 4 — movie count > 0 if [[ "$MOVIE_COUNT" -eq 0 ]]; then - warn "No movies returned from Radarr — aborting to prevent mass deletion" - notify "Radarr cleanup aborted on $(hostname) — no movies returned from API" "Radarr Cleanup" "warning" + error "API returned 0 movies — aborting to prevent mass deletion" + notify "Radarr cleanup aborted on $(hostname) — 0 movies returned" \ + "Radarr Cleanup" "warning" exit 1 fi +log "Found $MOVIE_COUNT movies — fetching movie files..." + TMP_DIR="/tmp/radarr_cleanup_$$" mkdir -p "$TMP_DIR" trap "rm -rf $TMP_DIR" EXIT @@ -236,12 +328,12 @@ trap "rm -rf $TMP_DIR" EXIT TRACKED_FILE="$TMP_DIR/tracked_paths.txt" > "$TRACKED_FILE" -# Step 2 — get movie files per movie ID MOVIE_INDEX=0 while IFS= read -r movie_id; do [[ -z "$movie_id" ]] && continue - ((MOVIE_INDEX++)) - [[ $(( MOVIE_INDEX % 100 )) -eq 0 ]] && info "Fetching files: $MOVIE_INDEX/$MOVIE_COUNT movies..." + (( MOVIE_INDEX++ )) + [[ $(( MOVIE_INDEX % 100 )) -eq 0 ]] && \ + log "Fetching files: $MOVIE_INDEX/$MOVIE_COUNT movies..." MOVIE_FILES=$(radarr_api "moviefile?movieId=${movie_id}" 2>/dev/null) if [[ -n "$MOVIE_FILES" ]]; then while IFS= read -r api_path; do @@ -253,24 +345,34 @@ done <<< "$MOVIE_IDS" sort -u "$TRACKED_FILE" -o "$TRACKED_FILE" +# Build in-memory lookup map — O(1) per lookup vs O(n) grep per file +# Eliminates the main performance bottleneck for large libraries +declare -A TRACKED_MAP +while IFS= read -r _tracked_path; do + [[ -n "$_tracked_path" ]] && TRACKED_MAP["$_tracked_path"]=1 +done < "$TRACKED_FILE" +unset _tracked_path +log "Built in-memory lookup map: ${#TRACKED_MAP[@]} tracked paths" TRACKED_COUNT=$(wc -l < "$TRACKED_FILE") -success "Radarr tracks $TRACKED_COUNT movie files" +# Safety Layer 5 — tracked count > 0 if [[ "$TRACKED_COUNT" -eq 0 ]]; then - warn "No tracked files returned — Radarr may not have scanned yet or library is empty" - warn "Aborting to prevent mass deletion" - notify "Radarr cleanup aborted on $(hostname) — no tracked files returned from API" "Radarr Cleanup" "warning" + error "API returned 0 tracked files — aborting to prevent mass deletion" + notify "Radarr cleanup aborted on $(hostname) — 0 tracked files returned" \ + "Radarr Cleanup" "warning" exit 1 fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_CLEAN Scanning Movies Root ━━━ -# ----------------------------------------------------------------------------------------------- +warn "Radarr tracks $TRACKED_COUNT movie files across $MOVIE_COUNT movies" + +# ============================================================================================== +# ━━━ Scan Movies Root ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_CLEAN Scanning Movies Root ━━━" -info "Root: $RADARR_MOVIES_ROOT" -info "Orphan age: ${RADARR_ORPHAN_AGE} days" -info "Protected: ${RADARR_PROTECTED_PATTERNS[*]}" +log "Root: $RADARR_MOVIES_ROOT" +log "Orphan age: ${RADARR_ORPHAN_AGE} days" +log "Protected: ${RADARR_PROTECTED_PATTERNS[*]}" echo "" START=$(date +%s) @@ -283,18 +385,19 @@ JUNK_BYTES=0 AGE_SECONDS=$(( RADARR_ORPHAN_AGE * 86400 )) NOW=$(date +%s) +MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $RADARR_MAX_DELETE_GB * 1073741824}") while IFS= read -r filepath; do [[ -z "$filepath" ]] && continue - if grep -qF "$filepath" "$TRACKED_FILE" 2>/dev/null; then + if [[ -n "${TRACKED_MAP[$filepath]:-}" ]]; then log "TRACKED: $filepath" continue fi if is_protected_file "$filepath"; then log "$ICON_PROTECTED PROTECTED: $filepath" - ((PROTECTED_COUNT++)) + (( PROTECTED_COUNT++ )) continue fi @@ -306,111 +409,114 @@ while IFS= read -r filepath; do if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]]; then log "RECENT (skipping): $filepath" - ((RECENT_COUNT++)) + (( RECENT_COUNT++ )) continue fi warn "$ICON_TRASH ORPHAN: $filepath" - if [[ "$DRY_RUN" == false ]]; then - rm -f "$filepath" && { - ((ORPHAN_COUNT++)) - ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE)) - } || error "Failed to delete: $filepath" - else - ((ORPHAN_COUNT++)) - ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE)) - fi + (( ORPHAN_COUNT++ )) + ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE )) else log "JUNK: $filepath" - if [[ "$DRY_RUN" == false ]]; then - rm -f "$filepath" && { - ((JUNK_COUNT++)) - JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE)) - } || error "Failed to delete: $filepath" - else - ((JUNK_COUNT++)) - JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE)) - fi + (( JUNK_COUNT++ )) + JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE )) fi done < <( - # Scan all host paths defined in ARR_PATH_MAP — covers all root folders managed by Radarr for host_path in "${ARR_PATH_MAP[@]}" "$RADARR_MOVIES_ROOT"; do [[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null done | sort -u ) -if [[ "$DRY_RUN" == false ]]; then - echo "" - info "Cleaning up empty folders..." - for host_path in "${ARR_PATH_MAP[@]}" "$RADARR_MOVIES_ROOT"; do - [[ -d "$host_path" ]] && \ - find "$host_path" -mindepth 1 -type d -empty -delete 2>/dev/null - done - success "Empty folders removed" -fi - -END=$(date +%s) - -format_bytes() { - local bytes=$1 - if (( bytes > 1073741824 )); then - awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}" - elif (( bytes > 1048576 )); then - awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}" - else - echo "${bytes}B" - fi -} - -ORPHAN_HUMAN=$(format_bytes $ORPHAN_BYTES) -JUNK_HUMAN=$(format_bytes $JUNK_BYTES) -TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES )) -MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", ${RADARR_MAX_DELETE_GB:-1} * 1073741824}") +TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) -# Size threshold check +# ============================================================================================== +# ━━━ Safety Layer 6 — Deletion Size Threshold ━━━ +# ============================================================================================== if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}") if [[ "$I_KNOW" != true ]]; then echo "" - error "Deletion would exceed ${RADARR_MAX_DELETE_GB:-1}GB threshold — $TOTAL_HUMAN would be deleted" - error "Review the ORPHAN lines above carefully before proceeding" - error "If this is expected, rerun with: --i-know-what-im-doing" - error "To also bypass age check and delete on first pass: add --skip-strike-list" - notify "Radarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} deletion requires --i-know-what-im-doing" "Radarr Cleanup" "warning" + error "Deletion would exceed ${RADARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted" + error "Review ORPHAN lines above carefully before proceeding" + error "Rerun with: --i-know-what-im-doing" + error "To also bypass age check: add --skip-strike-list" + notify "Radarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \ + "Radarr Cleanup" "warning" exit 1 else - warn "OVERRIDE — deletion is ${TOTAL_HUMAN} — proceeding because --i-know-what-im-doing" + warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing" fi fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ── Execute Deletions ───────────────────────────────────────────────────────────────────────── +if [[ "$DRY_RUN" == false ]]; then + while IFS= read -r filepath; do + [[ -z "$filepath" ]] && continue + [[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue + is_protected_file "$filepath" && continue + + FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0) + FILE_AGE=$(( NOW - FILE_MTIME )) + + if is_video_file "$filepath"; then + [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \ + [[ "$SKIP_STRIKES" != true ]] && continue + fi + + rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath" + + done < <( + for host_path in "${ARR_PATH_MAP[@]}" "$RADARR_MOVIES_ROOT"; do + [[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null + done | sort -u + ) + + log "Cleaning up empty folders..." + for host_path in "${ARR_PATH_MAP[@]}" "$RADARR_MOVIES_ROOT"; do + [[ -d "$host_path" ]] && \ + find "$host_path" -mindepth 1 -type d -empty -delete 2>/dev/null + done + log "Empty folders removed" +fi + +END=$(date +%s) + +ORPHAN_HUMAN=$(format_bytes "$ORPHAN_BYTES") +JUNK_HUMAN=$(format_bytes "$JUNK_BYTES") + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY RADARR CLEANUP SUMMARY ━━━━━" -echo "$ICON_SYNC Tracked by Radarr: $TRACKED_COUNT files" -echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles, metadata)" -echo "$ICON_TRASH Orphans removed: $ORPHAN_COUNT files ($ORPHAN_HUMAN)" -echo "$ICON_TRASH Junk removed: $JUNK_COUNT files ($JUNK_HUMAN)" -echo "$ICON_TIME Recent skipped: $RECENT_COUNT files (under ${RADARR_ORPHAN_AGE} days)" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_SYNC Tracked: $TRACKED_COUNT files ($MOVIE_COUNT movies)" +echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles, metadata)" +echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)" +echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)" +echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${RADARR_ORPHAN_AGE} days)" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" + if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no files deleted" + warn "DRY RUN — no files deleted" elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then - echo "$ICON_DONE Status: $ICON_SUCCESS CLEAN — nothing to remove" - notify "Radarr cleanup complete on $(hostname) — library is clean" "Radarr Cleanup" "normal" + log "$ICON_DONE Clean — nothing to remove" else - echo "$ICON_DONE Status: $ICON_SUCCESS DONE — $TOTAL_REMOVED files removed" - notify "Radarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Radarr Cleanup" "normal" + warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" + notify "Radarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \ + "Radarr Cleanup" "warning" + # Notify Emby to clean missing files — removes ghost entries immediately + notify_emby_scan fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Write stats for sunday_morning_coffee_report.sh if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_CLEANUP_STATS:-}" ]]; then - DATE=$(date '+%Y-%m-%d') - echo "${DATE}|radarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \ + echo "$(date '+%Y-%m-%d')|radarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \ >> "$ARR_CLEANUP_STATS" 2>/dev/null || true fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file + +exit 0 \ No newline at end of file diff --git a/Media/sonarr_cleanup.sh b/Media/sonarr_cleanup.sh index f722cbd..6c394c5 100644 --- a/Media/sonarr_cleanup.sh +++ b/Media/sonarr_cleanup.sh @@ -1,67 +1,109 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Sonarr Cleanup Script -------------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= Sonarr Cleanup ============================================= +# ============================================================================================== # Removes orphaned TV episode files from the library that Sonarr no longer tracks. # Uses the Sonarr API to build a complete list of tracked episode file paths then compares -# against what exists on disk — anything not tracked and older than SONARR_ORPHAN_AGE +# against what exists on disk — anything untracked and older than SONARR_ORPHAN_AGE # days is considered an orphan and deleted. # -# File classification: +# ── FILE CLASSIFICATION ─────────────────────────────────────────────────────────────────────── # TRACKED — Sonarr API knows about this exact file path → leave it alone # PROTECTED — matches SONARR_PROTECTED_PATTERNS → never delete (artwork, subtitles, .nfo) # ORPHAN — video file, not tracked, older than SONARR_ORPHAN_AGE days → delete # JUNK — not a video extension, not protected → delete regardless of age # RECENT — not tracked, under SONARR_ORPHAN_AGE days old → skip (may be mid-import) # -# Why protected patterns matter: -# Sonarr generates show artwork (*.jpg), metadata (*.nfo) and manages subtitles -# (*.srt, *.sub, *.ass) but does not include these in its tracked file API response. -# Without protection these would be classified as orphans and deleted. +# ── WHY PROTECTED PATTERNS MATTER ───────────────────────────────────────────────────────────── +# Sonarr generates show artwork (*.jpg), metadata (*.nfo) and manages subtitles (*.srt, +# *.sub, *.ass) but does NOT include these in its tracked file API response. +# Without protection these would be classified as orphans and deleted — breaking +# Sonarr and Emby metadata display. # +# ── SAFETY LAYERS — ALL MUST PASS BEFORE ANY FILE IS TOUCHED ───────────────────────────────── +# 1. Container must be running and not starting/unhealthy +# 2. API must be reachable +# 3. API version must match tested major version in master.conf +# 4. Series count must be > 0 +# 5. Tracked file count must be > 0 +# 6. Deletion size must be < SONARR_MAX_DELETE_GB — or --i-know-what-im-doing required +# +# ── POST-DELETION ───────────────────────────────────────────────────────────────────────────── +# After files are deleted notify_emby_scan() triggers Emby "Clean Missing Files" task. +# Emby immediately removes ghost entries — no user-facing file-not-found errors. +# +# ── OVERRIDE FLAGS ──────────────────────────────────────────────────────────────────────────── +# --i-know-what-im-doing required when deletion exceeds SONARR_MAX_DELETE_GB +# --skip-strike-list bypasses SONARR_ORPHAN_AGE age check +# NUCLEAR MODE — both active: age + size bypass, deletes on first pass +# ⚠️ User accepts full responsibility — no recovery possible after deletion +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── # HOST1 Sonarr manages Tv_Shows. HOST2 Sonarr manages Anime_Shows. -# detect_hosts() selects the correct URL, API key, and root path at runtime. -# All configuration in Master.conf under Arr Cleanup section. -# Supports --dry-run to preview what would be deleted without making changes. -# ----------------------------------------------------------------------------------------------- +# detect_hosts() sets MY_ID and aliases SONARR_URL, SONARR_API_KEY, SONARR_TV_ROOT. +# No manual HOST1/HOST2 comparisons — MY_ID routes correctly on any server. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock "wait" — large scans take time, wait for previous run to finish +# jq + curl validation — exits if either tool missing +# DOCKER_TIMEOUT — container checks protected against daemon hangs +# 6 safety layers — all must pass before any file is touched +# notify_emby_scan() — triggers Emby clean after deletion +# validate_unraid_cmd — notify script validated before use +# Silent by default — orphans/junk warn(), clean library logs silently +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_SONARR_URL / HOST*_SONARR_API_KEY / HOST*_SONARR_TV_ROOT +# HOST*_SONARR_PATH_MAP — container path → host path translation +# All aliased by detect_hosts() — script uses unprefixed names +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# SONARR_ORPHAN_AGE — days before untracked file eligible for deletion +# SONARR_MAX_DELETE_GB — require --i-know-what-im-doing above this +# SONARR_EXTENSIONS — video file extensions considered for orphan classification +# SONARR_PROTECTED_PATTERNS — file patterns never deleted +# SONARR_VERSION_MAJOR — expected Sonarr major version for API safety check +# ARR_CLEANUP_STATS — stats file path (read by coffee report) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# sonarr_cleanup.sh — normal run +# sonarr_cleanup.sh --dry-run — preview, no deletions +# sonarr_cleanup.sh --log — verbose output +# sonarr_cleanup.sh --status — show config and exit +# sonarr_cleanup.sh --i-know-what-im-doing — bypass size threshold +# sonarr_cleanup.sh --i-know-what-im-doing --skip-strike-list — NUCLEAR MODE +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" -# Check for --i-know-what-im-doing and --skip-strike-list flags before parse_args +# ── Special flag pre-processing ─────────────────────────────────────────────────────────────── I_KNOW=false SKIP_STRIKES=false FILTERED_ARGS=() for arg in "$@"; do - if [[ "$arg" == "--i-know-what-im-doing" ]]; then - I_KNOW=true - elif [[ "$arg" == "--skip-strike-list" ]]; then - SKIP_STRIKES=true - else - FILTERED_ARGS+=("$arg") - fi + case "$arg" in + --i-know-what-im-doing) I_KNOW=true ;; + --skip-strike-list) SKIP_STRIKES=true ;; + *) FILTERED_ARGS+=("$arg") ;; + esac done parse_args "${FILTERED_ARGS[@]}" -# Nuclear mode disclaimer +# ── Nuclear mode warning ────────────────────────────────────────────────────────────────────── if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" != true ]]; then echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "⚠️ WARNING — NUCLEAR MODE ACTIVE" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo " Flags: --i-know-what-im-doing --skip-strike-list" - echo " Strike system: BYPASSED — deletes on first pass" - echo " Size threshold: BYPASSED — no GB limit" - echo " Data recovery: NOT POSSIBLE after deletion" + echo " Flags: --i-know-what-im-doing --skip-strike-list" + echo " Strike system: BYPASSED — deletes on first pass" + echo " Size threshold: BYPASSED — no GB limit" + echo " Data recovery: NOT POSSIBLE after deletion" echo "" - echo " The script author takes no responsibility for data" - echo " loss when both flags are used together. This is a" - echo " 100% intentional action by the user." - echo "" - echo " Review the dry run output before proceeding." + echo " Review --dry-run output before proceeding." echo " You have 10 seconds to cancel (Ctrl+C)..." echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" sleep 10 @@ -69,9 +111,9 @@ if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" != echo "" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" @@ -80,8 +122,6 @@ if [[ "$EUID" -ne 0 ]]; then exit 1 fi -success "Running as root" - if ! command -v curl >/dev/null 2>&1; then error "curl not found — required for Sonarr API calls" exit 1 @@ -93,33 +133,25 @@ if ! command -v jq >/dev/null 2>&1; then exit 1 fi -# Select correct Sonarr instance based on which server is running this script +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + +acquire_lock "wait" + +# detect_hosts() sets MY_ID and aliases SONARR_URL, SONARR_API_KEY, SONARR_TV_ROOT detect_hosts -if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - SONARR_URL="$HOST1_SONARR_URL" - SONARR_API_KEY="$HOST1_SONARR_API_KEY" - SONARR_TV_ROOT="$HOST1_SONARR_TV_ROOT" -else - SONARR_URL="$HOST2_SONARR_URL" - SONARR_API_KEY="$HOST2_SONARR_API_KEY" - SONARR_TV_ROOT="$HOST2_SONARR_TV_ROOT" -fi +DOCKER_TIMEOUT=15 +SONARR_CONTAINER="Sonarr" -# Load path map for this host +# Build path map from MY_ID's Sonarr path map declare -A ARR_PATH_MAP -if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - for key in "${!HOST1_SONARR_PATH_MAP[@]}"; do - ARR_PATH_MAP["$key"]="${HOST1_SONARR_PATH_MAP[$key]}" - done -else - for key in "${!HOST2_SONARR_PATH_MAP[@]}"; do - ARR_PATH_MAP["$key"]="${HOST2_SONARR_PATH_MAP[$key]}" - done -fi - -info "Sonarr instance: $LOCAL_SERVER_NAME → $SONARR_URL" -info "TV root: $SONARR_TV_ROOT" +local_path_map_var="${MY_ID}_SONARR_PATH_MAP" +eval "for key in \"\${!${local_path_map_var}[@]}\"; do + ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\" +done" require_var SONARR_URL require_var SONARR_API_KEY @@ -127,35 +159,77 @@ require_var SONARR_TV_ROOT if [[ ! -d "$SONARR_TV_ROOT" ]]; then error "TV root not found: $SONARR_TV_ROOT" - notify "Sonarr cleanup failed on $(hostname) — TV root not found: $SONARR_TV_ROOT" "Sonarr Cleanup" "warning" + notify "Sonarr cleanup failed on $(hostname) — TV root not found: $SONARR_TV_ROOT" \ + "Sonarr Cleanup" "warning" exit 1 fi -acquire_lock "wait" +log "Sonarr URL: $SONARR_URL" +log "TV root: $SONARR_TV_ROOT" -success "Config validated" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted" +[[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing active" +[[ "$SKIP_STRIKES" == true ]] && warn "OVERRIDE — --skip-strike-list active — age check bypassed" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_GEAR Sonarr URL: $SONARR_URL" - echo "$ICON_GEAR TV root: $SONARR_TV_ROOT" - echo "$ICON_TIME Orphan age: ${SONARR_ORPHAN_AGE} days" - echo "$ICON_GEAR Extensions: ${SONARR_EXTENSIONS[*]}" - echo "$ICON_GEAR Protected patterns: ${SONARR_PROTECTED_PATTERNS[*]}" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_GEAR Sonarr URL: $SONARR_URL" + echo "$ICON_GEAR TV root: $SONARR_TV_ROOT" + echo "$ICON_TIME Orphan age: ${SONARR_ORPHAN_AGE} days" + echo "$ICON_GEAR Max delete: ${SONARR_MAX_DELETE_GB}GB (requires --i-know-what-im-doing)" + echo "$ICON_GEAR Sonarr ver: v${SONARR_VERSION_MAJOR} expected" + echo "$ICON_GEAR Extensions: ${SONARR_EXTENSIONS[*]}" + echo "$ICON_GEAR Protected patterns: ${SONARR_PROTECTED_PATTERNS[*]}" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_GEAR I know: $I_KNOW" + echo "$ICON_GEAR Skip strikes: $SKIP_STRIKES" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted" +# ============================================================================================== +# ━━━ Safety Layer 1 — Container Health ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_SHIELD Safety Checks ━━━" -# ----------------------------------------------------------------------------------------------- -# HELPERS -# ----------------------------------------------------------------------------------------------- +CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ + '{{.State.Running}}' "$SONARR_CONTAINER" 2>/dev/null) +if [[ "$CONTAINER_RUNNING" != "true" ]]; then + error "$SONARR_CONTAINER is not running — aborting" + notify "Sonarr cleanup aborted on $(hostname) — container not running" \ + "Sonarr Cleanup" "warning" + exit 1 +fi + +CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ + '{{.State.Health.Status}}' "$SONARR_CONTAINER" 2>/dev/null) +case "$CONTAINER_HEALTH" in + healthy) log "$SONARR_CONTAINER is healthy" ;; + "") log "$SONARR_CONTAINER has no health check — proceeding" ;; + starting) + error "$SONARR_CONTAINER is still starting — aborting" + notify "Sonarr cleanup aborted on $(hostname) — container still starting" \ + "Sonarr Cleanup" "warning" + exit 1 ;; + unhealthy) + error "$SONARR_CONTAINER is unhealthy — aborting" + notify "Sonarr cleanup aborted on $(hostname) — container unhealthy" \ + "Sonarr Cleanup" "warning" + exit 1 ;; + *) warn "$SONARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;; +esac + +log "Safety layer 1 passed — container healthy" + +# ============================================================================================== +# ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== sonarr_api() { local endpoint="$1" @@ -171,10 +245,9 @@ sonarr_api() { body=$(echo "$response" | head -n -1) if [[ "$http_code" != "200" ]]; then - error "Sonarr API returned HTTP $http_code for endpoint: $endpoint" + error "Sonarr API HTTP $http_code for: $endpoint" return 1 fi - echo "$body" } @@ -199,36 +272,55 @@ is_protected_file() { return 1 } -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SYNC Fetching Sonarr Tracked Files ━━━ -# ----------------------------------------------------------------------------------------------- +format_bytes() { + local bytes=$1 + if (( bytes > 1073741824 )); then + awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}" + elif (( bytes > 1048576 )); then + awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}" + else + echo "${bytes}B" + fi +} + +# ============================================================================================== +# ━━━ Fetch Sonarr Tracked Files ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_SYNC Fetching Sonarr Tracked Files ━━━" -check_api "$SONARR_URL" "Sonarr" || { +# Safety Layer 2 — API reachability +if ! check_api "$SONARR_URL" "Sonarr" 10; then notify "Sonarr cleanup aborted on $(hostname) — API unreachable" "Sonarr Cleanup" "warning" exit 1 -} +fi -info "Querying Sonarr API: $SONARR_URL" +# Safety Layer 3 — API version check +check_arr_version "$SONARR_URL" "$SONARR_API_KEY" "v3" "$SONARR_VERSION_MAJOR" "Sonarr" || exit 1 -# Step 1 — get all series to extract IDs +log "Querying Sonarr API: $SONARR_URL" + +# Fetch all series SERIES_RESPONSE=$(sonarr_api "series") || { - error "Failed to fetch series from Sonarr — check URL and API key" - notify "Sonarr cleanup failed on $(hostname) — API unreachable" "Sonarr Cleanup" "warning" + error "Failed to fetch series from Sonarr" + notify "Sonarr cleanup failed on $(hostname) — could not fetch series" \ + "Sonarr Cleanup" "warning" exit 1 } SERIES_IDS=$(echo "$SERIES_RESPONSE" | jq -r '.[].id' 2>/dev/null) -SERIES_COUNT=$(echo "$SERIES_IDS" | grep -c . 2>/dev/null || echo 0) -info "Series in Sonarr: $SERIES_COUNT" +SERIES_COUNT=$(echo "$SERIES_IDS" | grep -c "." 2>/dev/null || echo 0) +# Safety Layer 4 — series count > 0 if [[ "$SERIES_COUNT" -eq 0 ]]; then - warn "No series returned from Sonarr — aborting to prevent mass deletion" - notify "Sonarr cleanup aborted on $(hostname) — no series returned from API" "Sonarr Cleanup" "warning" + error "API returned 0 series — aborting to prevent mass deletion" + notify "Sonarr cleanup aborted on $(hostname) — 0 series returned" \ + "Sonarr Cleanup" "warning" exit 1 fi +log "Found $SERIES_COUNT series — fetching episode files..." + TMP_DIR="/tmp/sonarr_cleanup_$$" mkdir -p "$TMP_DIR" trap "rm -rf $TMP_DIR" EXIT @@ -236,12 +328,12 @@ trap "rm -rf $TMP_DIR" EXIT TRACKED_FILE="$TMP_DIR/tracked_paths.txt" > "$TRACKED_FILE" -# Step 2 — get episode files per series ID SERIES_INDEX=0 while IFS= read -r series_id; do [[ -z "$series_id" ]] && continue - ((SERIES_INDEX++)) - [[ $(( SERIES_INDEX % 50 )) -eq 0 ]] && info "Fetching files: $SERIES_INDEX/$SERIES_COUNT series..." + (( SERIES_INDEX++ )) + [[ $(( SERIES_INDEX % 50 )) -eq 0 ]] && \ + log "Fetching files: $SERIES_INDEX/$SERIES_COUNT series..." SERIES_FILES=$(sonarr_api "episodefile?seriesId=${series_id}" 2>/dev/null) if [[ -n "$SERIES_FILES" ]]; then while IFS= read -r api_path; do @@ -252,25 +344,26 @@ while IFS= read -r series_id; do done <<< "$SERIES_IDS" sort -u "$TRACKED_FILE" -o "$TRACKED_FILE" - TRACKED_COUNT=$(wc -l < "$TRACKED_FILE") -success "Sonarr tracks $TRACKED_COUNT episode files" +# Safety Layer 5 — tracked count > 0 if [[ "$TRACKED_COUNT" -eq 0 ]]; then - warn "No tracked files returned — Sonarr may not have scanned yet or library is empty" - warn "Aborting to prevent mass deletion" - notify "Sonarr cleanup aborted on $(hostname) — no tracked files returned from API" "Sonarr Cleanup" "warning" + error "API returned 0 tracked files — aborting to prevent mass deletion" + notify "Sonarr cleanup aborted on $(hostname) — 0 tracked files returned" \ + "Sonarr Cleanup" "warning" exit 1 fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_CLEAN Scanning TV Root ━━━ -# ----------------------------------------------------------------------------------------------- +warn "Sonarr tracks $TRACKED_COUNT episode files across $SERIES_COUNT series" + +# ============================================================================================== +# ━━━ Scan TV Root ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_CLEAN Scanning TV Root ━━━" -info "Root: $SONARR_TV_ROOT" -info "Orphan age: ${SONARR_ORPHAN_AGE} days" -info "Protected: ${SONARR_PROTECTED_PATTERNS[*]}" +log "Root: $SONARR_TV_ROOT" +log "Orphan age: ${SONARR_ORPHAN_AGE} days" +log "Protected: ${SONARR_PROTECTED_PATTERNS[*]}" echo "" START=$(date +%s) @@ -283,6 +376,7 @@ JUNK_BYTES=0 AGE_SECONDS=$(( SONARR_ORPHAN_AGE * 86400 )) NOW=$(date +%s) +MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $SONARR_MAX_DELETE_GB * 1073741824}") while IFS= read -r filepath; do [[ -z "$filepath" ]] && continue @@ -294,7 +388,7 @@ while IFS= read -r filepath; do if is_protected_file "$filepath"; then log "$ICON_PROTECTED PROTECTED: $filepath" - ((PROTECTED_COUNT++)) + (( PROTECTED_COUNT++ )) continue fi @@ -306,111 +400,114 @@ while IFS= read -r filepath; do if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]]; then log "RECENT (skipping): $filepath" - ((RECENT_COUNT++)) + (( RECENT_COUNT++ )) continue fi warn "$ICON_TRASH ORPHAN: $filepath" - if [[ "$DRY_RUN" == false ]]; then - rm -f "$filepath" && { - ((ORPHAN_COUNT++)) - ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE)) - } || error "Failed to delete: $filepath" - else - ((ORPHAN_COUNT++)) - ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE)) - fi + (( ORPHAN_COUNT++ )) + ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE )) else log "JUNK: $filepath" - if [[ "$DRY_RUN" == false ]]; then - rm -f "$filepath" && { - ((JUNK_COUNT++)) - JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE)) - } || error "Failed to delete: $filepath" - else - ((JUNK_COUNT++)) - JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE)) - fi + (( JUNK_COUNT++ )) + JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE )) fi done < <( - # Scan all host paths defined in ARR_PATH_MAP — covers all root folders managed by Sonarr for host_path in "${ARR_PATH_MAP[@]}" "$SONARR_TV_ROOT"; do [[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null done | sort -u ) -if [[ "$DRY_RUN" == false ]]; then - echo "" - info "Cleaning up empty folders..." - for host_path in "${ARR_PATH_MAP[@]}" "$SONARR_TV_ROOT"; do - [[ -d "$host_path" ]] && \ - find "$host_path" -mindepth 1 -type d -empty -delete 2>/dev/null - done - success "Empty folders removed" -fi - -END=$(date +%s) - -format_bytes() { - local bytes=$1 - if (( bytes > 1073741824 )); then - awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}" - elif (( bytes > 1048576 )); then - awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}" - else - echo "${bytes}B" - fi -} - -ORPHAN_HUMAN=$(format_bytes $ORPHAN_BYTES) -JUNK_HUMAN=$(format_bytes $JUNK_BYTES) -TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES )) -MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", ${SONARR_MAX_DELETE_GB:-1} * 1073741824}") +TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) -# Size threshold check +# ============================================================================================== +# ━━━ Safety Layer 6 — Deletion Size Threshold ━━━ +# ============================================================================================== if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}") if [[ "$I_KNOW" != true ]]; then echo "" - error "Deletion would exceed ${SONARR_MAX_DELETE_GB:-1}GB threshold — $TOTAL_HUMAN would be deleted" - error "Review the ORPHAN lines above carefully before proceeding" - error "If this is expected, rerun with: --i-know-what-im-doing" - error "To also bypass age check and delete on first pass: add --skip-strike-list" - notify "Sonarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} deletion requires --i-know-what-im-doing" "Sonarr Cleanup" "warning" + error "Deletion would exceed ${SONARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted" + error "Review ORPHAN lines above carefully before proceeding" + error "Rerun with: --i-know-what-im-doing" + error "To also bypass age check: add --skip-strike-list" + notify "Sonarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \ + "Sonarr Cleanup" "warning" exit 1 else - warn "OVERRIDE — deletion is ${TOTAL_HUMAN} — proceeding because --i-know-what-im-doing" + warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing" fi fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ── Execute Deletions ───────────────────────────────────────────────────────────────────────── +if [[ "$DRY_RUN" == false ]]; then + while IFS= read -r filepath; do + [[ -z "$filepath" ]] && continue + grep -qF "$filepath" "$TRACKED_FILE" 2>/dev/null && continue + is_protected_file "$filepath" && continue + + FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0) + FILE_AGE=$(( NOW - FILE_MTIME )) + + if is_video_file "$filepath"; then + [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \ + [[ "$SKIP_STRIKES" != true ]] && continue + fi + + rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath" + + done < <( + for host_path in "${ARR_PATH_MAP[@]}" "$SONARR_TV_ROOT"; do + [[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null + done | sort -u + ) + + log "Cleaning up empty folders..." + for host_path in "${ARR_PATH_MAP[@]}" "$SONARR_TV_ROOT"; do + [[ -d "$host_path" ]] && \ + find "$host_path" -mindepth 1 -type d -empty -delete 2>/dev/null + done + log "Empty folders removed" +fi + +END=$(date +%s) + +ORPHAN_HUMAN=$(format_bytes "$ORPHAN_BYTES") +JUNK_HUMAN=$(format_bytes "$JUNK_BYTES") + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY SONARR CLEANUP SUMMARY ━━━━━" -echo "$ICON_SYNC Tracked by Sonarr: $TRACKED_COUNT files" -echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles, metadata)" -echo "$ICON_TRASH Orphans removed: $ORPHAN_COUNT files ($ORPHAN_HUMAN)" -echo "$ICON_TRASH Junk removed: $JUNK_COUNT files ($JUNK_HUMAN)" -echo "$ICON_TIME Recent skipped: $RECENT_COUNT files (under ${SONARR_ORPHAN_AGE} days)" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_SYNC Tracked: $TRACKED_COUNT files ($SERIES_COUNT series)" +echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles, metadata)" +echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)" +echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)" +echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${SONARR_ORPHAN_AGE} days)" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" + if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no files deleted" + warn "DRY RUN — no files deleted" elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then - echo "$ICON_DONE Status: $ICON_SUCCESS CLEAN — nothing to remove" - notify "Sonarr cleanup complete on $(hostname) — library is clean" "Sonarr Cleanup" "normal" + log "$ICON_DONE Clean — nothing to remove" else - echo "$ICON_DONE Status: $ICON_SUCCESS DONE — $TOTAL_REMOVED files removed" - notify "Sonarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Sonarr Cleanup" "normal" + warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" + notify "Sonarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \ + "Sonarr Cleanup" "warning" + # Notify Emby to clean missing files — removes ghost entries immediately + notify_emby_scan fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Write stats for sunday_morning_coffee_report.sh if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_CLEANUP_STATS:-}" ]]; then - DATE=$(date '+%Y-%m-%d') - echo "${DATE}|sonarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \ + echo "$(date '+%Y-%m-%d')|sonarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \ >> "$ARR_CLEANUP_STATS" 2>/dev/null || true fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file + +exit 0 \ No newline at end of file diff --git a/Monitors/README-Monitors.md b/Monitors/README-Monitors.md index 6b58f77..d6c2b32 100644 --- a/Monitors/README-Monitors.md +++ b/Monitors/README-Monitors.md @@ -1,376 +1,981 @@ -# Monitors +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# 📊 MONITORS +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Watch and report only. Scripts in this folder never take action — they observe, measure, and notify. Intervention is handled by other parts of the ecosystem. +**Observe, measure, and report — never act.** Scripts in this folder watch the system +and notify when something needs attention. They do not restart containers, do not +reboot servers, do not delete files. The other folders handle intervention. This folder +handles awareness. ``` -unRAID_Essentials/ — acts on the system (restarts, reboots, stops) -Docker_Essentials/ — acts on containers (watchdog, restarts) -Monitors/ — observes and reports (this folder) +Monitors/ ← observes and reports (this folder) +Docker_Essentials/ ← acts on containers +unRAID_Essentials/ ← acts on the server +Media/ ← maintains the library ``` -All monitor scripts are safe to run at any time. None of them write to flash drives except `bandwidth_monitor.sh` which makes one bounded append per rsync run. All other monitors are read-only operations. +> **All monitor scripts are safe to run at any time.** They are read-only except for +> `bandwidth_monitor.sh` (one bounded append per rsync run) and +> `system_tuning_monitor.sh` (one bounded append every 6 hours). Every other script +> in this folder writes nothing. None of them can break anything. --- -## Scripts +## ━━━ THE PROBLEM THAT BUILT THIS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -### `cert_monitor.sh` +--- -Monitors SSL certificate expiry for all configured domains. +### 🔴 Certs Expiring Silently Despite Renewal + +Certbot runs on schedule. NPM shows the cert as renewed. But the web server was not +reloaded after renewal — the old expired cert is still being served to external clients. +NPM's dashboard shows green. `openssl s_client` from the outside shows a cert that +expired three days ago. Users see certificate errors. You see a healthy NPM status. + +The fix: `cert_monitor.sh` connects directly via `openssl` — not to NPM's API, not +to any internal check, but to the actual TLS handshake the outside world sees. It +catches renewed-but-not-reloaded certs, wrong certs being served, chain issues that +are invisible from the inside. If users would see an error, this script catches it. + +--- + +### 🔴 No Early Warning on Drive Degradation + +A drive accumulates reallocated sectors silently over weeks. The count climbs from 0 +to 12 to 47. Nothing fails. No error in the Docker logs. No alert from unRAID. +The drive is in progressive failure — heading toward a point of no return — and the +only way to know is to actively query SMART data. Which nobody does until something +goes wrong. + +The fix: `smart_health.sh` queries SMART attributes for every drive every week. +Reallocated sectors, pending sectors, uncorrectable sectors, temperature — anything +concerning triggers a notification. Drive failure has a warning curve. Weekly SMART +checks give you time to act on it. + +--- + +### 🔴 rsync Succeeds But the Copy Is Wrong + +`rsync` reports success. Transfer completed. Exit code 0. But a storage hardware +issue on either server silently corrupted a file during the write. The file exists +on both ends. rsync is satisfied — sizes match, timestamps match. Nobody knows the +content is wrong until someone tries to play the file and it fails, or worse, until +a drive failure reveals the backup was corrupt from the start. + +The fix: `backup_verify.sh` independently computes MD5 checksums on both sides of +the rsync mirror and compares them. It does not trust rsync's exit code. It does not +trust that the file exists. It verifies the actual content matches. Silent data +corruption is the failure mode backup systems exist to catch — and the only way to +catch it is to verify independently. + +--- + +### 🔴 inotify Exhaustion Producing Unexplained Symptoms + +inotify watches are silently exhausted. The kernel limit is hit. New file events +stop being delivered — silently. Downloads finish but arr containers don't see them. +Live TV stutter appears because media files aren't being detected as they arrive. +Library updates stop happening. None of these produce clear error messages — the +containers are running, the files are there, nothing is obviously wrong. The symptom +looks like a network problem or a container bug. It is an inotify problem. + +The same issue appears with PHP-FPM worker exhaustion — WebGUI requests start +queueing, timeouts appear, the dashboard feels sluggish. The worker count is at +ceiling and you have no visibility into how often it happens or how close it gets. + +The fix: `system_tuning_monitor.sh` snapshots inotify and PHP-FPM utilisation every +6 hours. The weekly coffee report shows peak, average, and warning count. You see +the trend before it becomes a problem. `inotify_tuning.sh` and `php_fpm_max_children.sh` +fix the limits — this script tells you whether the fix is holding. + +--- + +### 🔴 No Single View of What the Continuous Scripts Are Doing + +Three scripts run continuously as background processes: `system_watchdog.sh`, +`docker_watchdog.sh`, and `failover.sh`. Are they running? How long have they been +up? Are there active strikes against any container? Is failover in a non-NORMAL state? +Is the skip list empty? You have to check each state file individually, parse the +format yourself, and mentally correlate the information. + +The fix: `continuous_scripts_status.sh` — a live dashboard that reads all state files +and presents a single coherent view. Run it any time to know exactly what the watchdogs +are doing, what strikes are active, what the failover state is, and how the system +health looks right now. One command, complete picture. + +--- + +## ━━━ WHAT THIS FOLDER DOES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Four distinct observation roles: + +--- + +### 🔍 Live Connectivity Checks — Active probes against real endpoints + +`cert_monitor.sh` and `backup_verify.sh` go out and test things. They connect, +compute, compare. The result is a direct measurement of the actual state of the +world — not a reading of a cached status. + +--- + +### 📋 State File Aggregation — Synthesising the ecosystem's own data + +`weekly_health_digest.sh` and `continuous_scripts_status.sh` read the state files +the watchdogs and orchestrators have already written. They don't gather new data — +they organise existing data into human-readable summaries. + +--- + +### 📈 Time-Series Snapshots — Tracking trends over time + +`bandwidth_monitor.sh` and `system_tuning_monitor.sh` write one bounded log entry +per interval. Over time this builds a picture of trends — whether inotify pressure +is growing, whether transfer sizes are increasing, whether PHP-FPM is consistently +near ceiling. + +--- + +### 🖥️ Hardware Health — Checking what the hardware itself reports + +`smart_health.sh` and `zfs_memory_snapshot.sh` query the hardware and kernel +directly — drive SMART attributes, ZFS pool state, ARC statistics, kernel memory +pressure. Things that are invisible in application-level monitoring. + +--- + +## ━━━ FLASH DRIVE WRITE POLICY ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +unRAID boots from a USB flash drive. Flash drives have limited write cycles. +Every script in this folder is designed with this constraint in mind. + +| Script | Writes to flash | Detail | +|--------|----------------|--------| +| `cert_monitor.sh` | Never | openssl connections only | +| `smart_health.sh` | Never | smartctl reads only | +| `zfs_memory_snapshot.sh` | Never | writes to `/var/log/` (tmpfs) | +| `backup_verify.sh` | Never | SSH + MD5 comparison only | +| `emby_session_report.sh` | Never | API queries only | +| `weekly_health_digest.sh` | Never | reads existing state files only | +| `continuous_scripts_status.sh` | Never | reads existing state files only | +| `bandwidth_monitor.sh` | One append + one trim per rsync run | Bounded — never exceeds `BANDWIDTH_LOG_RETENTION` lines | +| `system_tuning_monitor.sh` | One append + one trim every 6 hours | Bounded — trimmed to `TUNING_LOG_RETENTION` days | + +The two scripts that write to flash are designed to be minimal and bounded. They +write one line and trim old entries on every write — the files never grow. + +--- + +## ━━━ SCRIPTS AT A GLANCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +| Script | What It Watches | When | +|--------|----------------|------| +| `cert_monitor.sh` | SSL cert expiry via live TLS connection | Weekly — Sunday 9am | +| `smart_health.sh` | Drive SMART health attributes | Weekly — Sunday 7am | +| `zfs_memory_snapshot.sh` | ZFS pool health, ARC stats, memory pressure | Weekly — Sunday 6am | +| `backup_verify.sh` | rsync mirror integrity via MD5 checksums | Weekly — Sunday 10am | +| `bandwidth_monitor.sh` | rsync transfer history and trends | Auto (by rsync.sh) + weekly report | +| `weekly_health_digest.sh` | Full ecosystem health aggregation | Daily (profile controls notify) | +| `emby_session_report.sh` | Emby streaming usage and library stats | Weekly — Sunday 11am | +| `system_tuning_monitor.sh` | inotify + PHP-FPM utilisation trends | Every 6 hours | +| `continuous_scripts_status.sh` | Live dashboard for all background processes | On demand | + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔐 cert_monitor.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +SSL certificate expiry monitoring via live TLS connection. Connects directly to each +domain via `openssl s_client` and reads the certificate actually being served — +not the certificate file, not NPM's API, not any cached status. ```bash -# Scheduled as: 0 9 * * 0 (Sunday 9am weekly) -/mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh +# Scheduled: 0 9 * * 0 (Sunday 9am weekly) ``` -**How it works:** +--- -Connects directly to each domain via `openssl s_client` and reads the certificate the server is actually presenting. This is different from checking your certificate files directly — it catches real-world issues that file-based checks miss: +### ── Why Direct openssl, Not an API ────────────────────────────────────────── -- Certificate renewed but web server not reloaded — old cert still being served -- Wrong certificate being served for a domain -- Certificate chain issues invisible to the cert file itself - -Each domain and subdomain is a separate entry. They have independent certificates — `gmer4lfe.com` and `auth.gmer4lfe.com` may expire on different dates. - -**Notification behavior:** -- Silent when all certs are healthy -- One warning notification for all domains approaching `CERT_WARN_DAYS` -- One critical notification for all domains within `CERT_CRIT_DAYS` -- Notifications batched by severity — not one per domain - -**Configuration:** ```bash -CERT_MONITOR_DOMAINS=( - "Gmer4Lfe.com" - "Gmer4Lfe.us" - # "auth.Gmer4Lfe.com" # add subdomains as separate entries +# ───────────────────────────────────────────────────────────────────────────── +# API-based cert checks ask the certificate manager if the cert is valid. +# openssl checks ask the server what cert it is actually serving. +# These are not the same question and the answers can differ. +# +# Direct openssl catches: +# - Cert renewed in NPM but web server not reloaded (old cert still serving) +# - Wrong cert being served (subdomain pointing to wrong vhost) +# - Chain issues visible externally but not internally +# - NPM reporting healthy while clients see an expired cert +# +# If a user would see a certificate error in their browser, this script +# catches it. If the cert manager says it's fine but the server is serving +# an old cert, this script catches that too. +``` + +--- + +### ── Results Per Domain ─────────────────────────────────────────────────────── + +``` +HEALTHY > CERT_WARN_DAYS remaining → silent ✅ +WARNING ≤ CERT_WARN_DAYS remaining → notification +CRITICAL ≤ CERT_CRIT_DAYS remaining → urgent notification +FAILED could not connect or parse cert → notification +``` + +Notifications are batched by severity — one message lists all domains at WARNING, +a separate message lists all domains at CRITICAL. Not one notification per domain. + +--- + +### ── Configuration ──────────────────────────────────────────────────────────── + +```bash +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# Each server monitors its own domains — detect_hosts() aliases the correct list. +# Each domain and subdomain is a separate entry — they have independent certs. +# +HOST1_CERT_MONITOR_DOMAINS=( + "Gmer4Lfe.com" # primary domain — checks the actual TLS handshake + "Gmer4Lfe.us" # secondary domain + # "auth.Gmer4Lfe.com" # add subdomains separately — they have separate certs + # "cloud.Gmer4Lfe.com" ) -CERT_WARN_DAYS=30 # warn when this many days remaining -CERT_CRIT_DAYS=7 # critical when this many days remaining -CERT_TIMEOUT=10 # seconds before giving up per domain + +# master.conf + CERT_WARN_DAYS=30 # warn this many days before expiry — time to investigate + CERT_CRIT_DAYS=7 # critical alert — action needed now + CERT_TIMEOUT=10 # seconds per domain before declaring FAILED ``` --- -### `smart_health.sh` - -Checks SMART health attributes for all drives in the system. +### ── Usage ─────────────────────────────────────────────────────────────────── ```bash -# Scheduled as: 0 7 * * 0 (Sunday 7am weekly) -/mnt/user/appdata/unraid_scripts/Monitors/smart_health.sh +cert_monitor.sh # normal run — silent if all healthy +cert_monitor.sh --dry-run # check all certs and show results, no notification sent +cert_monitor.sh --log # verbose output per domain +cert_monitor.sh --status # show domain list and thresholds ``` -**What it checks per drive:** +--- -| Attribute | Threshold | Meaning | -|-----------|-----------|---------| -| Reallocated_Sector_Ct | > 0 = warning | Bad sectors remapped — drive showing wear | -| Current_Pending_Sector | > 0 = warning | Sectors awaiting reallocation | -| Offline_Uncorrectable | > 0 = critical | Sectors that could not be corrected | -| Temperature_Celsius | SMART_TEMP_WARN/CRIT | Drive running hot | -| Power_On_Hours | informational | Drive age estimate | -| Overall health status | PASSED/FAILED | Drive's own self-assessment | +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 💾 smart_health.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Drive discovery is automatic — `/dev/sd*` and `/dev/nvme*` are scanned on every run. No drive list to maintain. +Drive SMART health monitoring. Queries SMART attributes for every drive on the system +weekly. Catches drive degradation before it becomes drive failure — most spinning +drives go through a warning period before dying outright. -**Why ignore the boot USB:** - -unRAID boots from a USB flash drive that typically appears as `sda`. Flash drives either don't support SMART or report meaningless values. Add it to `SMART_IGNORE_DRIVES` to keep it out of the report. - -**Notification behavior:** -- Silent when all drives are healthy -- One warning notification listing all drives with concerning attributes -- One critical notification if any drive has uncorrectable sectors - -**Configuration:** ```bash -SMART_TEMP_WARN=45 # degrees C -SMART_TEMP_CRIT=55 # degrees C -SMART_IGNORE_DRIVES=( - "sda" # boot USB — not meaningful to check +# Scheduled: 0 7 * * 0 (Sunday 7am weekly) +``` + +--- + +### ── What It Monitors Per Drive ────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# The attributes that actually indicate drive health problems: +# +Reallocated_Sector_Ct any > 0 = warning + The drive found a bad sector and swapped in a spare. + 0 spare = healthy. 1 spare = warning. More = escalating concern. + This is the first sign a drive is wearing out. + +Current_Pending_Sector any > 0 = warning + Sectors the drive suspects are bad but hasn't confirmed yet. + May recover on next read. May escalate to Reallocated. Watch it. + +Offline_Uncorrectable any > 0 = critical + Sectors the drive tried to correct during offline tests and couldn't. + No spares. No recovery. Data loss risk. + +Temperature_Celsius vs thresholds from dynamix.cfg + High sustained temperature shortens drive life significantly. + Script reads unRAID's own configured thresholds — no duplication needed. + +Overall SMART status PASSED / FAILED + The drive's own self-assessment. FAILED = get data off this drive now. + +Power_On_Hours informational + Drive age estimate — useful context for evaluating other attributes. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Drive Discovery + Temperature Source ──────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Drives discovered automatically — /dev/sd* and /dev/nvme* on every run. +# NVMe drives use different attribute names — detected and handled automatically. +# No drive list to maintain in configuration. +# +# Temperature thresholds read from /boot/config/plugins/dynamix/dynamix.cfg: +# hot, max, hotssd, maxssd — unRAID's own configured values. +# Falls back to SMART_TEMP_WARN / SMART_TEMP_CRIT from master.conf if not found. +# This means smart_health.sh and unRAID's dashboard use the same thresholds. +``` + +--- + +### ── Configuration ──────────────────────────────────────────────────────────── + +```bash +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# Drives to skip — typically the boot USB flash drive. +# unRAID boots from USB which appears as sda. Flash drives either don't +# support SMART or report meaningless values. Skip it. +# +HOST1_SMART_IGNORE_DRIVES=( + "sda" # boot USB flash drive — no meaningful SMART data ) + +# master.conf + SMART_TEMP_WARN=45 # °C — fallback if dynamix.cfg not found + SMART_TEMP_CRIT=55 # °C — fallback ``` --- -### `zfs_memory_snapshot.sh` - -Weekly ZFS pool health and memory diagnostic report. +### ── Usage ─────────────────────────────────────────────────────────────────── ```bash -# Scheduled as: 0 6 * * 0 (Sunday 6am weekly) -/mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh +smart_health.sh # normal run — silent if all drives healthy +smart_health.sh --dry-run # show which drives would be checked, no smartctl +smart_health.sh --log # verbose — full attribute dump per drive +smart_health.sh --status # show config and ignore list ``` -**What it reports:** +--- -- Pool status — ONLINE/DEGRADED/FAULTED per pool -- Pool overview — size, allocated, free, capacity, health -- ARC statistics — max, current, metadata usage, utilization % -- Memory status — total, free, available RAM vs thresholds -- Top N Docker containers by memory usage -- Kernel pressure snapshot via vmstat +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🗄️ zfs_memory_snapshot.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -**Informational only.** This script reports what it finds. `system_watchdog.sh` handles threshold-based intervention — ARC reclaim, reboot decisions, memory pressure response. The snapshot gives you the weekly picture; the watchdog handles emergencies. - -**Output is written to both console and `ZFS_REPORT_LOG`** — the log file lets you compare pool health week over week without having to remember what last week's numbers were. - -**Pool ignore list:** - -Pools expected to run at high capacity can be excluded from health reporting. They remain fully monitored by unRAID — this only affects what appears in the weekly report. +Weekly ZFS pool health and memory diagnostic report. Combines ZFS pool status, ARC +statistics, Docker memory usage, and kernel memory pressure into a single snapshot. +Informational only — `system_watchdog.sh` handles threshold-based intervention. ```bash -ZFS_REPORT_IGNORE_POOLS=( - "disk10" # high usage expected +# Scheduled: 0 6 * * 0 (Sunday 6am weekly — before other Sunday scripts) +``` + +--- + +### ── What It Reports ───────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# ZFS pool health: +# Status (ONLINE/DEGRADED/FAULTED) per pool +# Size, allocated, free, capacity, health per pool +# Pools in ZFS_REPORT_IGNORE_POOLS are excluded from reporting +# (they remain fully monitored by unRAID — this only affects the report) +# +# ARC statistics: +# Current ARC size vs max — what percentage of max is being used +# Metadata ARC — how much ARC is occupied by metadata vs data +# ARC hit rate — cache effectiveness +# Warns if ARC utilisation exceeds ZFS_REPORT_ARC_WARN_PCT +# +# Memory status: +# Total, free, available RAM +# Warns if free < ZFS_REPORT_FREE_WARN_GB +# Warns if available < ZFS_REPORT_AVAIL_WARN_GB +# +# Docker memory: +# Top ZFS_REPORT_DOCKER_TOP containers by memory usage +# Useful for spotting containers approaching their watchdog limits +# +# Kernel pressure: +# vmstat snapshot — 3 samples showing CPU and memory activity +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Output Goes to Console and Log ───────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Output is written to both console (for User Scripts output log) and ZFS_REPORT_LOG. +# The log file lets you compare pool health week over week without having to +# remember what last week's numbers were. Open it to see the trend. +# In dry-run mode — console only, nothing written to log. +# + ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log" # tmpfs — resets on reboot +``` + +--- + +### ── Configuration ──────────────────────────────────────────────────────────── + +```bash +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# Single-disk ZFS array members can be excluded from the report. +# They remain fully monitored by unRAID — this only affects the weekly snapshot. +# Each disk in a JBOD ZFS configuration appears as its own pool — there can be many. +# +HOST1_ZFS_REPORT_IGNORE_POOLS=( + "disk10" # JBOD member — high usage expected, exclude from report noise "disk9" "disk8" "disk6" "disk5" ) -``` -**Configuration:** -```bash -ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log" -ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC utilization above this % -ZFS_REPORT_FREE_WARN_GB=10 # warn if free RAM below this GB -ZFS_REPORT_AVAIL_WARN_GB=20 # warn if available RAM below this GB -ZFS_REPORT_DOCKER_TOP=10 # top N Docker memory users to show +# master.conf + ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC using more than 90% of its max + ZFS_REPORT_FREE_WARN_GB=10 # warn if free RAM below 10GB + ZFS_REPORT_AVAIL_WARN_GB=20 # warn if available RAM below 20GB + ZFS_REPORT_DOCKER_TOP=10 # top 10 Docker containers by memory ``` --- -### `backup_verify.sh` - -Verifies the rsync mirror is healthy by comparing random file checksums between local and remote servers. +### ── Usage ─────────────────────────────────────────────────────────────────── ```bash -# Scheduled as: 0 10 * * 0 (Sunday 10am weekly) -/mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh -``` - -**Why this matters:** - -`rsync.sh` copies files successfully. But does the copy match the original? `backup_verify.sh` answers that question by independently computing MD5 checksums on both sides and comparing them. It catches: - -- Silent data corruption during transfer -- Files that transferred but were corrupted at rest -- Partial transfers that rsync reported as success -- Storage hardware issues on either server - -**How it works:** - -1. Randomly samples `BACKUP_VERIFY_SAMPLE` files per share (files larger than `BACKUP_VERIFY_MIN_SIZE`) -2. Computes MD5 checksum locally -3. SSHes to remote and computes MD5 checksum there -4. Compares results - -**Result per file:** - -| Result | Meaning | -|--------|---------| -| MATCH | Checksums identical — file correctly mirrored | -| MISMATCH | File exists on both but checksums differ — sync may have failed | -| MISSING | File exists locally but not on remote — not yet synced or deleted | - -**Configuration:** -```bash -# Leave empty to use DAILY_SYNC_SHARES automatically -BACKUP_VERIFY_SHARES=( - # /mnt/user/Movies - # /mnt/user/Tv_Shows -) -BACKUP_VERIFY_SAMPLE=10 # files sampled per share per run -BACKUP_VERIFY_MIN_SIZE=1M # skip files smaller than this -``` - -If `BACKUP_VERIFY_SHARES` is empty the script automatically uses `DAILY_SYNC_SHARES` — no additional configuration needed for the standard setup. - -Uses the existing SSH keys already configured for rsync — no additional setup required. - ---- - -### `bandwidth_monitor.sh` - -Logs rsync transfer history and generates weekly summary reports. Called automatically by `rsync.sh` — you do not need to schedule the logging mode manually. - -```bash -# Log mode — called automatically by rsync.sh after each successful sync -# No manual scheduling needed - -# Report mode — run manually or schedule weekly -# Scheduled as: 0 11 * * 0 (Sunday 11am weekly) -/mnt/user/appdata/unraid_scripts/Monitors/bandwidth_monitor.sh --report -``` - -**Two modes:** - -**`--log-transfer profile duration status`** — called by `rsync.sh` after each sync. Appends one line to the log file and trims entries older than `BANDWIDTH_LOG_RETENTION` days. You never call this manually. - -**`--report`** — reads the log file and generates a weekly summary showing per-profile run counts, average durations, last 7 days activity, and failure counts. - -**Log format:** -``` -YYYY-MM-DD|HH:MM|profile|duration_seconds|status -2026-04-14|01:23|arrs_stack|287|success -2026-04-14|01:31|critical-data|143|success -2026-04-14|02:15|movies|1847|failed -``` - -**Why this format:** - -The log never parses rsync output. Earlier designs tried to extract bytes transferred from rsync's human-readable output — that approach breaks silently when rsync updates and changes its output format. The current format captures what's reliably available: profile, duration, and success/failure. This is version-proof and survives any rsync update. - -**Flash drive design:** - -The log lives on `/boot/` so it survives reboots. Each rsync run makes exactly one append and one trim — the file never grows beyond `BANDWIDTH_LOG_RETENTION` lines. Minimal flash wear. - -**Configuration:** -```bash -BANDWIDTH_LOG="/boot/config/bandwidth_history.db" -BANDWIDTH_LOG_RETENTION=90 # days — file stays bounded -BANDWIDTH_WARN_GB=50 # flag days exceeding this in reports - # (note: current log tracks duration not bytes) +zfs_memory_snapshot.sh # normal run — writes to log + console +zfs_memory_snapshot.sh --dry-run # console only, no log write +zfs_memory_snapshot.sh --log # verbose output +zfs_memory_snapshot.sh --status # show config and pool ignore list ``` --- -### `weekly_health_digest.sh` +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ✅ backup_verify.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Aggregates system health data from across the entire ecosystem into a single digest report. +rsync mirror integrity verification via independent MD5 checksums. Randomly samples +files from each share, computes checksums on both sides, and compares. Does not trust +rsync's exit code. Verifies the actual content matches. ```bash -# Scheduled as: 0 8 * * * (8am daily — profile controls when it notifies) -/mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh -``` - -**The key design:** schedule it daily regardless of profile. The `DIGEST_PROFILE` setting in `Master.conf` controls when a notification is actually sent — not the cron schedule. - -**Three profiles:** - -| Profile | Behavior | Use When | -|---------|----------|----------| -| `always` | Sends every run | You want a daily health summary | -| `smart` | Sends only if something worth reporting | Quiet operation, alerts on issues | -| `weekly` | Sends once per week on `DIGEST_DAY` | Weekly digest, silent other days | - -Switch profiles by changing `DIGEST_PROFILE` in `Master.conf` — no cron changes needed. - -**Data sources — reads only, no writes:** - -| Source | What it reads | -|--------|---------------| -| `/tmp/transcode_state.db` | Ramdisk symlink and usage | -| `/tmp/container_watchdog_state.db` | Active container watchdog strikes | -| `/tmp/system_watchdog_state.db` | Active system watchdog strikes | -| `/boot/config/failover_state.db` | Current failover state | -| `/boot/config/system_watchdog_failed.db` | Container skip list | -| `/boot/config/bandwidth_history.db` | Recent transfer totals | -| Live `openssl` connection | SSL cert days remaining per domain | - -**Smart profile triggers:** -```bash -# Set true to include this check in smart mode's "worth reporting" decision -DIGEST_SMART_ON_WATCHDOG=true # any active watchdog strikes -DIGEST_SMART_ON_FAILOVER=true # failover state is not NORMAL -DIGEST_SMART_ON_CERT_WARN=true # any cert under CERT_WARN_DAYS -DIGEST_SMART_ON_BANDWIDTH=true # any transfer exceeded BANDWIDTH_WARN_GB -``` - -**Configuration:** -```bash -DIGEST_PROFILE="weekly" # always | smart | weekly -DIGEST_DAY="Sunday" # for weekly profile — must match date +%A output +# Scheduled: 0 10 * * 0 (Sunday 10am weekly) ``` --- -### `emby_session_report.sh` - -Weekly Emby usage report via the Emby API. +### ── Why Verify Independently ───────────────────────────────────────────────── ```bash -# Scheduled as: 0 11 * * 0 (Sunday 11am weekly) -/mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh -``` - -**What it reports:** -- Active streams at time of run -- Stream breakdown — total, Live TV, transcoding, direct play -- Library counts — movies, episodes, songs -- Current ramdisk transcode usage and symlink state - -**No persistent writes** — queries the Emby API fresh on every run. No log files, no state. Run it any time for a current snapshot. - -**Requires an Emby API key:** -1. Open Emby Settings → API Keys -2. Generate a new key -3. Paste it into `Master.conf` as `EMBY_API_KEY` - -**Configuration:** -```bash -EMBY_URL="http://localhost:8096" -EMBY_API_KEY="" # get from Emby Settings → API Keys -EMBY_REPORT_DAYS=7 # report period in days -EMBY_REPORT_TOP_N=10 # top N content items to show +# ───────────────────────────────────────────────────────────────────────────── +# rsync reports success when: +# - The transfer completed without network errors +# - The file sizes and modification times match +# +# rsync does NOT detect: +# - Silent corruption during the transfer (bitflip in transit) +# - Corruption written to storage at rest (faulty drive sector) +# - Files that matched size/mtime but had wrong content (rare, happens) +# +# These failure modes produce exit code 0. rsync says "done." +# backup_verify.sh checks whether "done" means "correct." +# +# Sample size is intentionally small (10 files per share, default) — +# this is a spot check, not an exhaustive verify. It catches systematic +# problems and hardware issues while running in minutes, not hours. +# ───────────────────────────────────────────────────────────────────────────── ``` --- -## Flash Drive Write Policy +### ── Results Per File ───────────────────────────────────────────────────────── -unRAID boots from a USB flash drive. Flash drives have limited write cycles. The Monitors folder is designed with this in mind: +``` +MATCH checksums identical on both sides → correctly mirrored ✅ +MISMATCH file exists on both, checksums differ → sync failure or corruption +MISSING file exists locally but not on remote → not yet synced or deleted +``` -| Script | Writes to flash | Notes | -|--------|----------------|-------| -| `cert_monitor.sh` | Never | Read-only openssl checks | -| `smart_health.sh` | Never | Read-only smartctl checks | -| `zfs_memory_snapshot.sh` | Never | Writes to `/var/log/` (RAM disk) | -| `backup_verify.sh` | Never | SSH + MD5 comparison only | -| `bandwidth_monitor.sh` | One append + one trim per rsync run | Bounded — never exceeds retention days | -| `weekly_health_digest.sh` | Never | Reads existing state files only | -| `emby_session_report.sh` | Never | API queries only | - -The only flash write in the entire Monitors folder is `bandwidth_monitor.sh` — and it's designed to be minimal and bounded. +All MISMATCHes and any significant number of MISSINGs trigger a notification. --- -## Recommended Schedule +### ── Pre-flight Checks Before Any SSH ─────────────────────────────────────── ```bash +# ───────────────────────────────────────────────────────────────────────────── +# Two checks before computing a single checksum: +# +# check_connectivity() — is the remote Tailscale IP reachable? +# Without this: all files would show as MISSING (SSH timeout per file). +# False alarm from a network hiccup. +# +# check_remote_array() — is the remote /mnt/user mounted? +# Without this: remote files don't exist yet (array not started). +# All files would show as MISSING. False catastrophic alarm. +# +# Both must pass before any checksums are computed. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Configuration ──────────────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── + BACKUP_VERIFY_SAMPLE=10 # random files sampled per share per run + BACKUP_VERIFY_MIN_SIZE="1M" # skip files smaller than this (checksums waste + # time on tiny files with low corruption risk) + +# master_host*.conf +# ───────────────────────────────────────────────────────────────────────────── +# Leave empty to use HOST*_DAILY_SYNC_SHARES automatically. +# No additional configuration needed for the standard setup. +# + HOST1_BACKUP_VERIFY_SHARES=( + # empty — uses HOST1_DAILY_SYNC_SHARES automatically + # "/mnt/user/Movies" # override here to check specific shares only + ) +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +backup_verify.sh # normal run — silent if all samples match +backup_verify.sh --dry-run # show which files would be sampled, no checksums +backup_verify.sh --log # verbose — show checksum comparison per file +backup_verify.sh --status # show share list and sample configuration +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 📡 bandwidth_monitor.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +rsync transfer history logging and weekly summary reporting. Called automatically by +`rsync.sh` after each sync — no manual scheduling needed for the logging part. The +report can be run manually or on a weekly schedule. + +```bash +# Log mode: called automatically by rsync.sh — no manual scheduling needed +# Report mode — scheduled: 0 11 * * 0 (Sunday 11am weekly) +``` + +--- + +### ── Two Modes ──────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# --log-transfer profile duration_seconds status bytes_transferred +# Called by rsync.sh after each sync completes. +# Appends one line to BANDWIDTH_LOG. +# Trims entries older than BANDWIDTH_LOG_RETENTION days. +# Flags transfers exceeding BANDWIDTH_WARN_GB for report highlighting. +# You never call this manually. +# +# --report (or no arguments) +# Reads the accumulated log and generates a weekly summary. +# Per-profile breakdown: run count, total transferred, average duration, failures. +# Last 7 days activity timeline. +# Flags any days or transfers that exceeded BANDWIDTH_WARN_GB. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Log Format — Version-Proof Design ────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# One line per transfer: +# YYYY-MM-DD|HH:MM|profile|duration_seconds|status|bytes_transferred +# +# Example entries: +# 2026-04-14|01:23|critical-data|143|success|2847362048 +# 2026-04-14|01:31|arrs_stack|287|success|891234567 +# 2026-04-14|02:15|movies|1847|failed|0 +# +# Why this format: +# Earlier designs parsed rsync's human-readable output for bytes transferred. +# rsync changes its output format between versions — those parsers break silently. +# Status (success/failed) and duration are always available regardless of rsync version. +# Bytes captured from rsync --stats output via awk — version-stable field names. +# This format survives any rsync update with no changes. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Configuration ──────────────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── + BANDWIDTH_LOG="/boot/config/bandwidth_history.db" # survives reboots + BANDWIDTH_LOG_RETENTION=90 # days — file stays bounded, never grows unbounded + BANDWIDTH_WARN_GB=50 # flag transfers or daily totals exceeding this +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +bandwidth_monitor.sh # generate report (default mode) +bandwidth_monitor.sh --report # generate report (explicit) +bandwidth_monitor.sh --status # show config and log statistics +bandwidth_monitor.sh --log # verbose output + +# This is called automatically by rsync.sh — never call manually: +bandwidth_monitor.sh --log-transfer "profile" 287 "success" 891234567 +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 📋 weekly_health_digest.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Full ecosystem health aggregation. Reads state files from across the entire system — +watchdog strikes, failover state, skip list, bandwidth history, transcode stats, cert +status — and compiles them into a single digest report. Reads only, writes nothing. + +```bash +# Scheduled: 0 8 * * * (8am daily — DIGEST_PROFILE controls when it notifies) +``` + +--- + +### ── The Key Design — Schedule Daily, Profile Controls Behaviour ───────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# The cron schedule never changes — always daily at 8am. +# DIGEST_PROFILE controls what actually happens on each run. +# Switch profiles by changing one line in master.conf — no cron edit needed. +# + DIGEST_PROFILE="weekly" # always | smart | weekly +``` + +--- + +### ── Three Profiles ─────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# always — sends every run regardless of findings +# Use: daily digest of everything, even when healthy +# Result: daily notification with full system snapshot +# +# smart — sends only when something worth reporting is found +# Use: quiet operation, alerts on issues +# Result: silent on clean days, notification when something needs attention +# What triggers "worth reporting" is configurable via DIGEST_SMART_ON_* toggles +# +# weekly — sends once per week on DIGEST_DAY, silent all other days +# Use: weekly summary without daily noise +# Result: one notification on Sunday, silent Monday-Saturday +# ───────────────────────────────────────────────────────────────────────────── + + DIGEST_DAY="Sunday" # for weekly profile — must match `date +%A` output + +# Smart profile triggers — each independently toggleable + DIGEST_SMART_ON_WATCHDOG=true # send if any active watchdog strikes + DIGEST_SMART_ON_FAILOVER=true # send if failover state is not NORMAL + DIGEST_SMART_ON_CERT_WARN=true # send if any cert within CERT_WARN_DAYS + DIGEST_SMART_ON_BANDWIDTH=true # send if any transfer exceeded BANDWIDTH_WARN_GB +``` + +--- + +### ── Data Sources — Reads Only ──────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Every source is a read — this script writes nothing, changes nothing. +# + FAILOVER_STATE_FILE # current failover state (NORMAL/FAILOVER/etc.) + SYS_WATCHDOG_FAILED_FILE # container skip list — anything here needs attention + WATCHDOG_STATE_FILE # active container watchdog strikes + SYS_WATCHDOG_STATE_FILE # active system watchdog strikes + BANDWIDTH_LOG # yesterday's transfer history + TRANSCODE_DAILY_LOG # weekly transcode statistics + CERT_MONITOR_DOMAINS # live openssl cert check per domain + RAMDISK_PATH / TRANSCODE_LINK # current transcode location and usage +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +weekly_health_digest.sh # normal run — profile determines if notification sent +weekly_health_digest.sh --dry-run # show digest output, no notification regardless of profile +weekly_health_digest.sh --log # verbose per-section output +weekly_health_digest.sh --status # show profile, day, and trigger configuration +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🎬 emby_session_report.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Emby usage report via the Emby API. Queries activity logs and session history to +produce a weekly summary of streaming activity. No persistent state — queries fresh +on every run. + +```bash +# Scheduled: 0 11 * * 0 (Sunday 11am weekly) +``` + +--- + +### ── What It Reports ───────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── + Server info: name, version, uptime + Active sessions: current streams, direct play vs transcode breakdown + Library stats: movie count, episode count, song count + Activity history: play events from the last EMBY_REPORT_DAYS days + Top content: most played items in the period (top EMBY_REPORT_TOP_N) + Most active users: who watched the most in the period + Transcode ratio: how often transcoding was needed vs direct play + notifies if > 80% of streams are transcoding (config issue) + Ramdisk status: current transcode location and usage +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Configuration ──────────────────────────────────────────────────────────── + +```bash +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# detect_hosts() aliases these from HOST*_ prefixed vars. +# Each server reports on its own Emby instance automatically. +# + HOST1_EMBY_URL="http://192.168.50.2:8096" + HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829" +# +# To get an API key: +# Emby UI → Settings → API Keys → New API Key +# Give it a name (e.g. "unraid_scripts") — copy the key + +# master.conf + EMBY_REPORT_DAYS=7 # report period in days + EMBY_REPORT_TOP_N=10 # top N content items to show +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +emby_session_report.sh # generate report +emby_session_report.sh --dry-run # test API connectivity only, no notification +emby_session_report.sh --log # verbose per-section output +emby_session_report.sh --status # show config and Emby connection status +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ⚙️ system_tuning_monitor.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +inotify and PHP-FPM utilisation tracking. Snapshots current usage every 6 hours and +writes to a bounded log. The weekly coffee report reads this log for peak/average/ +warning counts over the week. Silent when healthy — warns when thresholds are hit. + +```bash +# Scheduled: 0 */6 * * * (every 6 hours) +``` + +--- + +### ── What It Tracks ─────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# inotify instances: +# Current in use vs kernel limit (INOTIFY_MAX_INSTANCES) +# Utilisation % — warns above INOTIFY_WARN_PCT (default 80%) +# Top 5 consumers by instance count +# +# What inotify exhaustion looks like at the application level: +# Downloads complete but arrs don't detect them +# Live TV stutter as media files aren't seen arriving +# Library updates stop happening +# No clear error anywhere — things just stop working +# +# php-fpm workers: +# Active workers vs PHP_MAX_CHILDREN limit +# Utilisation % — warns above PHP_FPM_WARN_PCT (default 80%) +# +# What PHP-FPM exhaustion looks like: +# unRAID WebGUI becomes slow or times out +# Dashboard requests queue behind each other +# Settings saves hang +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Log Format ────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# One line per snapshot — trimmed to TUNING_LOG_RETENTION days on each write: +# DATE|TIME|INOTIFY_USED|INOTIFY_LIMIT|INOTIFY_PCT|INOTIFY_WARN| +# PHPFPM_ACTIVE|PHPFPM_MAX|PHPFPM_PCT|PHPFPM_WARN +# +# INOTIFY_WARN and PHPFPM_WARN are 1/0 flags — 1 means threshold was exceeded +# during this snapshot. The weekly coffee report counts warnings over the week. +``` + +--- + +### ── Configuration ──────────────────────────────────────────────────────────── + +```bash +# master.conf + INOTIFY_WARN_PCT=80 # warn if inotify instances above 80% of limit + PHP_FPM_WARN_PCT=80 # warn if php-fpm workers above 80% of max + TUNING_MONITOR_LOG="$DATA_DIR/tuning_monitor.db" + TUNING_LOG_RETENTION=30 # days — file stays bounded +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🖥️ continuous_scripts_status.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Live dashboard for all continuously running background scripts. Run any time to get +an immediate picture of the ecosystem's operational state. No schedule — on demand only. + +--- + +### ── What It Shows ──────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# For each continuous script (system_watchdog, docker_watchdog, failover): +# Running state, PID, uptime, approximate cycle count +# +# system_watchdog: +# Rootfs usage, free RAM, ZFS ARC size, load average, zombie count, CPU temp +# Active strikes (any check with non-zero strike count) +# Recent watchdog-triggered reboots +# +# docker_watchdog: +# Running vs stopped vs unhealthy container counts +# Required containers — which are running, which are stopped +# Memory-monitored containers — current usage vs hard limits +# Active container strikes +# Recent restart history (per container) +# Skip list — anything here needs human attention +# +# failover: +# Current state (NORMAL/FAILOVER/NO_INTERNET/DARK) +# Which tier containers are active +# Outage duration if in FAILOVER +# Remote server Tailscale visibility (ping result) +# DDNS containers running on this server +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Run any time — no cron, no schedule. This is an on-demand status tool. +# ───────────────────────────────────────────────────────────────────────────── +continuous_scripts_status.sh # live dashboard +continuous_scripts_status.sh --log # verbose output with full state file dumps +``` + +--- + +## ━━━ RECOMMENDED SCHEDULE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +```bash +# ───────────────────────────────────────────────────────────────────────────── # Daily -0 8 * * * weekly_health_digest.sh # profile controls when it notifies +# ───────────────────────────────────────────────────────────────────────────── +0 8 * * * weekly_health_digest.sh # DIGEST_PROFILE controls notify frequency -# Weekly — Sunday morning block -0 6 * * 0 zfs_memory_snapshot.sh -0 7 * * 0 smart_health.sh -0 9 * * 0 cert_monitor.sh -0 10 * * 0 backup_verify.sh -0 11 * * 0 emby_session_report.sh -0 11 * * 0 bandwidth_monitor.sh --report +# ───────────────────────────────────────────────────────────────────────────── +# Every 6 hours — background snapshot +# ───────────────────────────────────────────────────────────────────────────── +0 */6 * * * system_tuning_monitor.sh # inotify + php-fpm utilisation tracking +# ───────────────────────────────────────────────────────────────────────────── +# Sunday morning block — runs after nightly maintenance completes (~3am) +# By 6am the weekly restarts, log clears, and media maintenance have finished. +# Monitors see a freshly maintained system. +# ───────────────────────────────────────────────────────────────────────────── +0 6 * * 0 zfs_memory_snapshot.sh # ZFS + memory — first, before everything +0 7 * * 0 smart_health.sh # drive SMART health +0 9 * * 0 cert_monitor.sh # SSL cert expiry +0 10 * * 0 backup_verify.sh # rsync mirror integrity +0 11 * * 0 emby_session_report.sh # Emby streaming usage +0 11 * * 0 bandwidth_monitor.sh --report # rsync transfer summary + +# ───────────────────────────────────────────────────────────────────────────── # Automatic — no scheduling needed -# bandwidth_monitor.sh --log-transfer is called by rsync.sh after each sync +# ───────────────────────────────────────────────────────────────────────────── +# bandwidth_monitor.sh --log-transfer is called by rsync.sh after each sync. +# continuous_scripts_status.sh is run on demand — no cron entry. ``` -The Sunday morning block runs after the nightly maintenance window — by the time the monitors run, the weekly restarts, log clears, and media management jobs have completed. The health snapshot reflects a freshly maintained system. - --- -## --dry-run Support +## ━━━ ADDING A NEW MONITOR ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -All monitor scripts support `--dry-run`. In dry-run mode: - -- Checks run and results are shown -- No notifications are sent -- No files are written - -Useful for testing configuration changes before scheduling: +The monitor pattern is consistent across all scripts in this folder: ```bash -/mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh --dry-run -/mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh --dry-run -/mnt/user/appdata/unraid_scripts/Monitors/smart_health.sh --dry-run +# ───────────────────────────────────────────────────────────────────────────── +# All monitor scripts share these properties: +# +# SILENT_MODE=false — output is the point (monitors are meant to produce output) +# No action taken — observe only, intervention is handled elsewhere +# acquire_lock — prevent concurrent runs producing duplicate output +# detect_hosts() — correct configuration per server via MY_ID +# validate_unraid_cmd — notify script validated before use +# --dry-run support — run checks without sending notifications +# --status support — show configuration and exit +# --log support — verbose per-item output +# Silent on healthy — no notification when everything passes +# +# To add a new monitor: +# 1. Source load_config.sh — gives you all common functions +# 2. Set SILENT_MODE=false — this is a monitor, output is expected +# 3. Call detect_hosts() — correct per-host configuration +# 4. Call acquire_lock — prevent duplicate runs +# 5. Do your checks — read, connect, compare +# 6. notify() only on problems — silent on healthy results +# 7. Support --dry-run (no notify) and --status (show config, exit) +# ───────────────────────────────────────────────────────────────────────────── ``` \ No newline at end of file diff --git a/Monitors/backup_verify.sh b/Monitors/backup_verify.sh index ac87237..5fb92ad 100644 --- a/Monitors/backup_verify.sh +++ b/Monitors/backup_verify.sh @@ -1,79 +1,152 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Backup Verify ---------------------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= Backup Verify ============================================== +# ============================================================================================== # Verifies the rsync mirror is healthy by comparing random file samples between # local and remote servers using MD5 checksums. # -# Randomly samples BACKUP_VERIFY_SAMPLE files per share, computes checksums locally, -# then computes the same checksums on the remote via SSH and compares results. +# ── WHAT IT DOES ────────────────────────────────────────────────────────────────────────────── +# Randomly samples BACKUP_VERIFY_SAMPLE files per share above BACKUP_VERIFY_MIN_SIZE, +# computes MD5 checksums locally, then computes the same checksums on the remote via SSH +# and compares results. Catches silent corruption or incomplete syncs that rsync itself +# would not detect. # -# Results per file: -# MATCH — checksums identical, file is correctly mirrored -# MISMATCH — file exists on both but checksums differ — sync may have failed -# MISSING — file exists locally but not on remote — not yet synced or deleted +# ── RESULTS PER FILE ────────────────────────────────────────────────────────────────────────── +# MATCH — checksums identical, file is correctly mirrored ✅ +# MISMATCH — file exists on both but checksums differ — sync may have partially failed +# MISSING — file exists locally but not on remote — not yet synced or deleted on remote # -# Silent when all files match. Notifies on any mismatch or missing file. -# Uses existing SSH keys — no additional configuration needed beyond share list. +# ── SHARE SELECTION ─────────────────────────────────────────────────────────────────────────── +# Uses HOST*_BACKUP_VERIFY_SHARES if defined, falls back to HOST*_DAILY_SYNC_SHARES. +# Both aliased by detect_hosts() — no manual HOST1/HOST2 selection needed. # -# All configuration in Master.conf under Backup Verify section. -# Supports --dry-run to show what would be checked without running checksums. -# ----------------------------------------------------------------------------------------------- +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock — prevents concurrent runs producing duplicate/conflicting results +# check_connectivity() — verifies remote reachable before attempting SSH calls +# check_remote_array() — verifies remote array mounted before checksums +# remote array down = all files "missing" = false alarm ✅ +# version parity — verifies both servers on compatible unRAID before trusting results +# SSH_TIMEOUT — all SSH calls protected against hangs +# validate_unraid_cmd — notify script validated before use +# Silent by default — only issues produce output, all-match runs are silent +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_BACKUP_VERIFY_SHARES — override share list (empty = use DAILY_SYNC_SHARES) +# HOST*_DAILY_SYNC_SHARES — fallback share list +# All aliased by detect_hosts() +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# BACKUP_VERIFY_SAMPLE — random files to check per share (default 10) +# BACKUP_VERIFY_MIN_SIZE — minimum file size to include in sample (default 1M) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# backup_verify.sh — normal run +# backup_verify.sh --dry-run — show sample selection only, no checksums +# backup_verify.sh --log — verbose output +# backup_verify.sh --status — show config and exit +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- +SSH_TIMEOUT=15 + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + +acquire_lock + +# detect_hosts() sets MY_ID and aliases BACKUP_VERIFY_SHARES + DAILY_SYNC_SHARES detect_hosts resolve_remote_ip -# Use BACKUP_VERIFY_SHARES if defined, fall back to DAILY_SYNC_SHARES +# Share selection — configured list or fallback to daily sync shares if [[ ${#BACKUP_VERIFY_SHARES[@]} -gt 0 ]]; then VERIFY_SHARES=("${BACKUP_VERIFY_SHARES[@]}") - info "Using BACKUP_VERIFY_SHARES (${#VERIFY_SHARES[@]} shares)" + log "Using BACKUP_VERIFY_SHARES (${#VERIFY_SHARES[@]} shares)" else VERIFY_SHARES=("${DAILY_SYNC_SHARES[@]}") - info "BACKUP_VERIFY_SHARES not set — using DAILY_SYNC_SHARES (${#VERIFY_SHARES[@]} shares)" + log "BACKUP_VERIFY_SHARES not set — using DAILY_SYNC_SHARES (${#VERIFY_SHARES[@]} shares)" fi if [[ ${#VERIFY_SHARES[@]} -eq 0 ]]; then - warn "No shares configured — nothing to verify" - exit 0 -fi - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- -if [[ "$SHOW_STATUS" == true ]]; then - echo "" - echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_VERIFY Shares: ${#VERIFY_SHARES[@]}" - echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share" - echo "$ICON_VERIFY Min size: $BACKUP_VERIFY_MIN_SIZE" - echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)" - echo "$ICON_GEAR Dry Run: $DRY_RUN" - echo "━━━━━━━━━━━━━━━━━━━━━━━" + warn "No shares configured for $MY_ID — nothing to verify" + warn "Check HOST*_BACKUP_VERIFY_SHARES or HOST*_DAILY_SYNC_SHARES in master_host*.conf" exit 0 fi [[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing sample selection only, no checksums computed" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_VERIFY Backup Verification ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_HOST Remote: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)" + echo "$ICON_VERIFY Shares: ${#VERIFY_SHARES[@]}" + echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share" + echo "$ICON_VERIFY Min size: $BACKUP_VERIFY_MIN_SIZE" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "" + echo " Shares to verify:" + for share in "${VERIFY_SHARES[@]}"; do + echo " $share" + done + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Pre-flight ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_SHIELD Pre-flight ━━━" + +# Connectivity — no point making 100+ SSH calls if remote is unreachable +check_connectivity + +# Version parity — mismatched unRAID could cause md5sum path differences +check_unraid_version_parity || { + warn "Version parity check failed — proceeding with caution" + warn "Checksum results may be unreliable if md5sum path changed between versions" +} + +# Remote array — if array is down all files appear "missing" = false alarm +if ! check_remote_array; then + error "Remote array not mounted on $REMOTE_SERVER_NAME" + error "All files would appear as MISSING — aborting to prevent false alarm" + notify "Backup verify aborted on $(hostname) — remote array not mounted on $REMOTE_SERVER_NAME" \ + "Backup Verify" "warning" + exit 1 +fi + +log "Pre-flight passed ✅" + +# ============================================================================================== +# ━━━ Backup Verification ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_VERIFY Backup Verification — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)" -echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share (min size: $BACKUP_VERIFY_MIN_SIZE)" +echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → $REMOTE_ID ($REMOTE_SERVER_NAME)" +echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share (min: $BACKUP_VERIFY_MIN_SIZE)" echo "" START=$(date +%s) @@ -93,69 +166,74 @@ for share in "${VERIFY_SHARES[@]}"; do continue fi - # Find files above minimum size and randomly sample - SAMPLE_FILES=$(find "$share" -type f -size +"$BACKUP_VERIFY_MIN_SIZE" 2>/dev/null | \ - shuf | head -n "$BACKUP_VERIFY_SAMPLE") + # Sample random files above minimum size + mapfile -t SAMPLE_FILES < <( + find "$share" -type f -size +"$BACKUP_VERIFY_MIN_SIZE" 2>/dev/null | \ + shuf | head -n "$BACKUP_VERIFY_SAMPLE" + ) - SAMPLE_COUNT=$(echo "$SAMPLE_FILES" | grep -c "." 2>/dev/null || echo 0) - - if [[ "$SAMPLE_COUNT" -eq 0 ]]; then - info "No files found above $BACKUP_VERIFY_MIN_SIZE — skipping" + if [[ ${#SAMPLE_FILES[@]} -eq 0 ]]; then + log "$SHARE_NAME — no files found above $BACKUP_VERIFY_MIN_SIZE" echo "" continue fi - info "Sampled $SAMPLE_COUNT files" + log "$SHARE_NAME — sampled ${#SAMPLE_FILES[@]} files" if [[ "$DRY_RUN" == true ]]; then - echo "$SAMPLE_FILES" | while IFS= read -r f; do + for f in "${SAMPLE_FILES[@]}"; do warn "DRY RUN — would check: $(basename "$f")" done echo "" continue fi + SHARE_MATCH=0 SHARE_MISMATCH=0 SHARE_MISSING=0 - SHARE_MATCH=0 - while IFS= read -r local_file; do + for local_file in "${SAMPLE_FILES[@]}"; do [[ -z "$local_file" ]] && continue - # Compute local checksum + # Local checksum local_md5=$(md5sum "$local_file" 2>/dev/null | awk '{print $1}') - if [[ -z "$local_md5" ]]; then - warn "Could not checksum: $local_file — skipping" + warn "Could not checksum locally: $(basename "$local_file") — skipping" continue fi - # Compute remote checksum via SSH - remote_md5=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ + # Remote checksum via SSH — timeout protected + remote_md5=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" \ + -o StrictHostKeyChecking=no \ + root@"$REMOTE_SERVER" \ "md5sum '$local_file' 2>/dev/null | awk '{print \$1}'" 2>/dev/null) - ((TOTAL_CHECKED++)) + (( TOTAL_CHECKED++ )) if [[ -z "$remote_md5" ]]; then warn "$ICON_ERROR MISSING: $(basename "$local_file")" - ((SHARE_MISSING++)) - ((TOTAL_MISSING++)) + (( SHARE_MISSING++ )) + (( TOTAL_MISSING++ )) elif [[ "$local_md5" == "$remote_md5" ]]; then log "MATCH: $(basename "$local_file")" - ((SHARE_MATCH++)) - ((TOTAL_MATCH++)) + (( SHARE_MATCH++ )) + (( TOTAL_MATCH++ )) else - error "$ICON_ERROR MISMATCH: $(basename "$local_file")" - ((SHARE_MISMATCH++)) - ((TOTAL_MISMATCH++)) + error "MISMATCH: $(basename "$local_file")" + error " local: $local_md5" + error " remote: $remote_md5" + (( SHARE_MISMATCH++ )) + (( TOTAL_MISMATCH++ )) fi + done - done <<< "$SAMPLE_FILES" - - echo " $ICON_SUCCESS Match: $SHARE_MATCH $ICON_WARN Missing: $SHARE_MISSING $ICON_ERROR Mismatch: $SHARE_MISMATCH" - + # Per-share result — only visible if issues found if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 ]]; then + warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH" SHARES_WITH_ISSUES+=("$SHARE_NAME") + else + log "$SHARE_NAME — all $SHARE_MATCH files match ✅" fi echo "" @@ -163,25 +241,32 @@ done END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "━━━━━ $ICON_SUMMARY BACKUP VERIFY SUMMARY ━━━━━" -echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME" +echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_HOST Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)" echo "$ICON_VERIFY Checked: $TOTAL_CHECKED files" -echo "$ICON_SUCCESS Match: $TOTAL_MATCH" -echo "$ICON_WARN Missing: $TOTAL_MISSING" -echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" -if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no checksums computed" -elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then - echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention" - notify "Backup verify failed on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" "Backup Verify" "warning" -else - echo "$ICON_DONE Status: $ICON_SUCCESS ALL FILES MATCH" - notify "Backup verify passed on $(hostname) → $REMOTE_SERVER_NAME — $TOTAL_CHECKED files checked across ${#VERIFY_SHARES[@]} shares" "Backup Verify" "normal" +if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then + echo "$ICON_SUCCESS Match: $TOTAL_MATCH" + warn "Missing: $TOTAL_MISSING" + [[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH" fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — no checksums computed" +elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then + echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention: ${SHARES_WITH_ISSUES[*]}" + notify "Backup verify FAILED on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" \ + "Backup Verify" "warning" +else + log "$ICON_DONE Status: all $TOTAL_CHECKED files match across ${#VERIFY_SHARES[@]} shares ✅" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ "$TOTAL_MISMATCH" -gt 0 ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/Monitors/bandwidth_monitor.sh b/Monitors/bandwidth_monitor.sh index 8608ef1..8cc222c 100644 --- a/Monitors/bandwidth_monitor.sh +++ b/Monitors/bandwidth_monitor.sh @@ -1,38 +1,63 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Bandwidth Monitor ------------------------------------------ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= Bandwidth Monitor ========================================== +# ============================================================================================== # Logs rsync transfer history and generates weekly summary reports. # Designed for minimal flash drive impact — one bounded write per rsync run. # -# Two modes: -# --log-transfer "profile" duration status — called by rsync.sh after each sync -# appends one line, trims old entries -# --report (or no args) — generates summary from log +# ── TWO MODES ───────────────────────────────────────────────────────────────────────────────── # -# Log format — one line per transfer, version-proof, never needs rsync output parsing: -# YYYY-MM-DD|HH:MM|profile|duration_seconds|status +# --log-transfer "profile" duration_seconds status +# Called automatically by rsync.sh after each sync completes. +# Appends one line to the log and trims entries older than BANDWIDTH_LOG_RETENTION. +# Flags syncs exceeding BANDWIDTH_WARN_GB in the log for weekly report highlighting. # -# Log file stays bounded to BANDWIDTH_LOG_RETENTION days — old entries trimmed on every write. +# --report (or no args) +# Generates a summary from the accumulated log. +# Shows per-profile breakdown, last 7 days, and overall totals. +# This is a monitor script — SILENT_MODE=false — output is the point. +# +# ── LOG FORMAT ──────────────────────────────────────────────────────────────────────────────── +# One line per transfer — version-proof, never needs rsync output parsing: +# YYYY-MM-DD|HH:MM|profile|duration_seconds|status|bytes_transferred +# +# Log file stays bounded to BANDWIDTH_LOG_RETENTION days — trimmed on every write. # Minimal flash drive impact: one append + one trim per rsync run. # -# All configuration in Master.conf under Bandwidth Monitor section. -# ----------------------------------------------------------------------------------------------- +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock "wait" — prevents log corruption from concurrent rsync completions +# validate_unraid_cmd — notify script validated before use +# Atomic log write — temp file + mv prevents partial writes on trim +# Log existence check — creates log directory if needed, exits cleanly if unwritable +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# BANDWIDTH_LOG — log file path +# BANDWIDTH_LOG_RETENTION — days before old entries are purged (default 90) +# BANDWIDTH_WARN_GB — flag syncs larger than this in report (default 50) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# bandwidth_monitor.sh — generate report +# bandwidth_monitor.sh --report — generate report (explicit) +# bandwidth_monitor.sh --log-transfer profile secs ok — log a transfer (called by rsync.sh) +# bandwidth_monitor.sh --status — show config and exit +# bandwidth_monitor.sh --log — verbose output +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" + +# Monitor script — output is the point +SILENT_MODE=false parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# Parse mode from PARSED_ARGS -# ----------------------------------------------------------------------------------------------- +# ── Parse mode from PARSED_ARGS ─────────────────────────────────────────────────────────────── LOG_TRANSFER_MODE=false TRANSFER_PROFILE="" TRANSFER_DURATION=0 TRANSFER_STATUS="success" +TRANSFER_BYTES=0 for arg in "${PARSED_ARGS[@]}"; do case "$arg" in @@ -42,79 +67,131 @@ for arg in "${PARSED_ARGS[@]}"; do if [[ "$LOG_TRANSFER_MODE" == true ]]; then if [[ -z "$TRANSFER_PROFILE" ]]; then TRANSFER_PROFILE="$arg" - elif [[ "$TRANSFER_DURATION" -eq 0 ]]; then + elif [[ "$TRANSFER_DURATION" -eq 0 && "$arg" =~ ^[0-9]+$ ]]; then TRANSFER_DURATION="$arg" - else + elif [[ "$arg" == "success" || "$arg" == "failed" ]]; then TRANSFER_STATUS="$arg" + elif [[ "$arg" =~ ^[0-9]+$ ]]; then + TRANSFER_BYTES="$arg" fi fi ;; esac done -# Ensure log directory and file exist +# ── Ensure log file exists and is writable ──────────────────────────────────────────────────── mkdir -p "$(dirname "$BANDWIDTH_LOG")" touch "$BANDWIDTH_LOG" 2>/dev/null || { error "Cannot write to bandwidth log: $BANDWIDTH_LOG" exit 1 } -# ----------------------------------------------------------------------------------------------- -# LOG TRANSFER MODE +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + +# detect_hosts() sets MY_ID for report header +detect_hosts + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_BANDWIDTH Log file: $BANDWIDTH_LOG" + echo "$ICON_BANDWIDTH Retention: ${BANDWIDTH_LOG_RETENTION} days" + echo "$ICON_BANDWIDTH Warn GB: ${BANDWIDTH_WARN_GB}GB" + local entry_count=0 + [[ -f "$BANDWIDTH_LOG" ]] && entry_count=$(wc -l < "$BANDWIDTH_LOG") + echo "$ICON_BANDWIDTH Log entries: $entry_count" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Log Transfer Mode ━━━ +# ============================================================================================== # Called by rsync.sh after each sync — appends one line and trims old entries. -# Usage: bandwidth_monitor.sh --log-transfer "profile" duration_seconds status -# ----------------------------------------------------------------------------------------------- +# Uses "wait" lock — if two rsync jobs finish simultaneously, wait and write in order. +# Usage: bandwidth_monitor.sh --log-transfer "profile" duration_seconds status [bytes] + if [[ "$LOG_TRANSFER_MODE" == true ]]; then - [[ -z "$TRANSFER_PROFILE" ]] && error "No profile specified for --log-transfer" && exit 1 + [[ -z "$TRANSFER_PROFILE" ]] && { error "No profile specified for --log-transfer"; exit 1; } + + acquire_lock "wait" TODAY=$(date '+%Y-%m-%d') NOW=$(date '+%H:%M') DURATION_FMT=$(format_duration "$TRANSFER_DURATION") - # Append entry - echo "${TODAY}|${NOW}|${TRANSFER_PROFILE}|${TRANSFER_DURATION}|${TRANSFER_STATUS}" >> "$BANDWIDTH_LOG" - log "$ICON_BANDWIDTH Logged: $TRANSFER_PROFILE — ${DURATION_FMT} — $TRANSFER_STATUS" + # Check if transfer exceeds warn threshold + WARN_FLAG="" + if [[ -n "$TRANSFER_BYTES" && "$TRANSFER_BYTES" -gt 0 ]]; then + WARN_BYTES=$(awk "BEGIN {printf \"%d\", $BANDWIDTH_WARN_GB * 1073741824}") + [[ "$TRANSFER_BYTES" -gt "$WARN_BYTES" ]] && WARN_FLAG="LARGE" + fi - # Trim entries older than retention period — keeps file bounded + # Append entry — format: date|time|profile|duration|status|bytes|warn_flag + echo "${TODAY}|${NOW}|${TRANSFER_PROFILE}|${TRANSFER_DURATION}|${TRANSFER_STATUS}|${TRANSFER_BYTES}|${WARN_FLAG}" \ + >> "$BANDWIDTH_LOG" + log "$ICON_BANDWIDTH Logged: $TRANSFER_PROFILE — ${DURATION_FMT} — $TRANSFER_STATUS${WARN_FLAG:+ [$WARN_FLAG]}" + + # Trim entries older than retention — atomic write via temp file CUTOFF=$(date -d "${BANDWIDTH_LOG_RETENTION} days ago" '+%Y-%m-%d') TEMP_FILE="${BANDWIDTH_LOG}.tmp" awk -F'|' -v cutoff="$CUTOFF" '$1 >= cutoff' "$BANDWIDTH_LOG" > "$TEMP_FILE" && \ mv "$TEMP_FILE" "$BANDWIDTH_LOG" - log "$ICON_BANDWIDTH Log trimmed — retaining entries from $CUTOFF onwards" + log "$ICON_BANDWIDTH Log trimmed — retaining entries from $CUTOFF" exit 0 fi -# ----------------------------------------------------------------------------------------------- -# REPORT MODE — generate summary from log -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Report Mode ━━━ +# ============================================================================================== +acquire_lock "wait" + echo "" echo "━━━ $ICON_BANDWIDTH Bandwidth Report — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)" echo "" if [[ ! -s "$BANDWIDTH_LOG" ]]; then warn "No bandwidth data yet — log is empty" - warn "Data accumulates as rsync jobs complete" + warn "Data accumulates as rsync jobs complete via rsync.sh" exit 0 fi START=$(date +%s) -# Date range +# ── Log overview ────────────────────────────────────────────────────────────────────────────── OLDEST=$(awk -F'|' 'NR==1{print $1}' "$BANDWIDTH_LOG") NEWEST=$(awk -F'|' 'END{print $1}' "$BANDWIDTH_LOG") ENTRY_COUNT=$(wc -l < "$BANDWIDTH_LOG") SUCCESS_COUNT=$(awk -F'|' '$5=="success"' "$BANDWIDTH_LOG" | wc -l) FAILED_COUNT=$(awk -F'|' '$5=="failed"' "$BANDWIDTH_LOG" | wc -l) +LARGE_COUNT=$(awk -F'|' '$7=="LARGE"' "$BANDWIDTH_LOG" | wc -l) info "Log covers: $OLDEST → $NEWEST ($ENTRY_COUNT runs)" echo "" -# ── Per-profile breakdown ──────────────────────────────────────────────────────────────────── +# ── Per-profile breakdown ───────────────────────────────────────────────────────────────────── echo "━━━ $ICON_BANDWIDTH Per-Profile Summary ━━━" awk -F'|' '{ runs[$3]++ duration[$3] += $4 if ($5 == "failed") fails[$3]++ + if ($7 == "LARGE") large[$3]++ } END { for (profile in runs) { @@ -122,21 +199,22 @@ END { mins = int(avg / 60) secs = int(avg % 60) fail_count = (profile in fails) ? fails[profile] : 0 - printf " %-20s %3d runs avg %dm%ds failed: %d\n", \ - profile, runs[profile], mins, secs, fail_count + large_count = (profile in large) ? large[profile] : 0 + large_str = (large_count > 0) ? " ⚠️ " large_count " large" : "" + printf " %-22s %3d runs avg %dm%ds failed: %d%s\n", \ + profile, runs[profile], mins, secs, fail_count, large_str } }' "$BANDWIDTH_LOG" | sort - echo "" -# ── Last 7 days ────────────────────────────────────────────────────────────────────────────── +# ── Last 7 days ─────────────────────────────────────────────────────────────────────────────── echo "━━━ $ICON_BANDWIDTH Last 7 Days ━━━" for i in 6 5 4 3 2 1 0; do day=$(date -d "$i days ago" '+%Y-%m-%d') day_name=$(date -d "$i days ago" '+%a') day_runs=$(awk -F'|' -v d="$day" '$1==d' "$BANDWIDTH_LOG" | wc -l) - day_success=$(awk -F'|' -v d="$day" '$1==d && $5=="success"' "$BANDWIDTH_LOG" | wc -l) day_failed=$(awk -F'|' -v d="$day" '$1==d && $5=="failed"' "$BANDWIDTH_LOG" | wc -l) + day_large=$(awk -F'|' -v d="$day" '$1==d && $7=="LARGE"' "$BANDWIDTH_LOG" | wc -l) day_duration=$(awk -F'|' -v d="$day" '$1==d{sum+=$4} END{print sum+0}' "$BANDWIDTH_LOG") day_duration_fmt=$(format_duration "$day_duration") @@ -144,25 +222,47 @@ for i in 6 5 4 3 2 1 0; do echo " $ICON_TIME $day ($day_name) — no syncs" elif [[ "$day_failed" -gt 0 ]]; then echo " $ICON_WARN $day ($day_name) — $day_runs runs / ${day_duration_fmt} total / $ICON_ERROR $day_failed failed" + elif [[ "$day_large" -gt 0 ]]; then + echo " $ICON_WARN $day ($day_name) — $day_runs runs / ${day_duration_fmt} total / ⚠️ $day_large large" else echo " $ICON_DONE $day ($day_name) — $day_runs runs / ${day_duration_fmt} total" fi done - echo "" -# ── Totals ─────────────────────────────────────────────────────────────────────────────────── +# ── Large transfers ─────────────────────────────────────────────────────────────────────────── +if [[ "$LARGE_COUNT" -gt 0 ]]; then + echo "━━━ $ICON_WARN Large Transfers (>${BANDWIDTH_WARN_GB}GB) ━━━" + awk -F'|' '$7=="LARGE" { + bytes=$6+0 + gb=bytes/1073741824 + printf " %s %s %-20s %.1fGB\n", $1, $2, $3, gb + }' "$BANDWIDTH_LOG" | tail -10 + echo "" +fi + +# ── Totals ──────────────────────────────────────────────────────────────────────────────────── TOTAL_DURATION=$(awk -F'|' '{sum+=$4} END{print sum+0}' "$BANDWIDTH_LOG") TOTAL_DURATION_FMT=$(format_duration "$TOTAL_DURATION") END=$(date +%s) echo "━━━━━ $ICON_SUMMARY BANDWIDTH SUMMARY ━━━━━" -echo "$ICON_BANDWIDTH Total runs: $ENTRY_COUNT ($SUCCESS_COUNT success / $FAILED_COUNT failed)" -echo "$ICON_TIME Total time: $TOTAL_DURATION_FMT" -echo "$ICON_TIME Log period: $OLDEST → $NEWEST" -echo "$ICON_GEAR Retention: ${BANDWIDTH_LOG_RETENTION} days" -echo "$ICON_TIME Generated in: $(format_duration $((END - START)))" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_BANDWIDTH Runs: $ENTRY_COUNT ($SUCCESS_COUNT success / $FAILED_COUNT failed)" +[[ "$LARGE_COUNT" -gt 0 ]] && \ + warn "Large: $LARGE_COUNT transfer(s) exceeded ${BANDWIDTH_WARN_GB}GB" +echo "$ICON_TIME Total: $TOTAL_DURATION_FMT" +echo "$ICON_TIME Period: $OLDEST → $NEWEST" +echo "$ICON_GEAR Retention: ${BANDWIDTH_LOG_RETENTION} days" +echo "$ICON_TIME Generated: $(format_duration $(( END - START )))" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -notify "Bandwidth report on $(hostname) — $ENTRY_COUNT rsync runs ($SUCCESS_COUNT success / $FAILED_COUNT failed) over ${BANDWIDTH_LOG_RETENTION} day window" "Bandwidth Monitor" "normal" \ No newline at end of file +# Only notify if there are failures or large transfers worth flagging +if [[ "$FAILED_COUNT" -gt 0 ]]; then + notify "Bandwidth report on $(hostname) — $FAILED_COUNT failed sync(s) in ${BANDWIDTH_LOG_RETENTION} day window" \ + "Bandwidth Monitor" "warning" +elif [[ "$LARGE_COUNT" -gt 0 ]]; then + notify "Bandwidth report on $(hostname) — $LARGE_COUNT large transfer(s) exceeded ${BANDWIDTH_WARN_GB}GB" \ + "Bandwidth Monitor" "normal" +fi \ No newline at end of file diff --git a/Monitors/cert_monitor.sh b/Monitors/cert_monitor.sh index bf9cf41..19226b9 100644 --- a/Monitors/cert_monitor.sh +++ b/Monitors/cert_monitor.sh @@ -1,73 +1,130 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Certificate Monitor ---------------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= Certificate Monitor ======================================== +# ============================================================================================== # Monitors SSL certificate expiry for all configured domains by connecting directly # via openssl — no dependency on NPM or any other service. Reads the actual certificate # the server is presenting to the outside world. # -# This approach catches real-world cert issues that API-based checks miss: -# - Cert renewed but server not reloaded -# - Wrong cert being served -# - Cert chain issues +# ── WHY DIRECT OPENSSL ──────────────────────────────────────────────────────────────────────── +# Catches real-world cert issues that API-based checks miss: +# - Cert renewed in NPM but server not reloaded (old cert still serving) +# - Wrong cert being served to external clients +# - Cert chain issues not visible from the internal network +# - NPM reporting healthy while the world sees an expired cert # -# Each domain and subdomain is a separate entry — they have independent certs. -# Silent when all certs are healthy. Notifies when any approach warning threshold. -# Notifications batched per severity — one message for warnings, one for criticals. +# ── BEHAVIOUR ───────────────────────────────────────────────────────────────────────────────── +# Each domain is checked independently — they have independent certs. +# Results per domain: +# HEALTHY — > CERT_WARN_DAYS remaining — silent ✅ +# WARNING — <= CERT_WARN_DAYS remaining — notifies +# CRITICAL — <= CERT_CRIT_DAYS remaining — notifies with urgency +# FAILED — could not connect or parse cert — notifies # -# All configuration in Master.conf under Certificate Monitor section. -# Supports --dry-run to check certs and show results without sending notifications. -# ----------------------------------------------------------------------------------------------- +# Notifications batched per severity — one message per severity level, not per domain. +# This is a monitor script — SILENT_MODE=false — output is the point. +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID and aliases HOST*_CERT_MONITOR_DOMAINS → CERT_MONITOR_DOMAINS. +# Each server monitors its own domains — HOST1 monitors Gmer4Lfe.com etc. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock — prevents concurrent runs +# detect_hosts() — correct domain list per host via MY_ID aliases +# Empty array guard — warns and exits cleanly if no domains configured +# CERT_TIMEOUT — openssl connects are time-limited per domain +# validate_unraid_cmd — openssl and notify validated before use +# Silent healthy certs — only problems produce visible output +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_CERT_MONITOR_DOMAINS — domains checked by this host +# Aliased by detect_hosts() — script uses CERT_MONITOR_DOMAINS +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# CERT_WARN_DAYS — warn when cert expires within this many days (default 30) +# CERT_CRIT_DAYS — critical alert within this many days (default 7) +# CERT_TIMEOUT — seconds per domain before giving up (default 10) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# cert_monitor.sh — normal run +# cert_monitor.sh --dry-run — check certs and show results, no notifications +# cert_monitor.sh --log — verbose output +# cert_monitor.sh --status — show config and exit +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" + +# Monitor script — output is the point +SILENT_MODE=false parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" -if ! command -v openssl >/dev/null 2>&1; then - error "openssl not found — required for certificate checks" +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" exit 1 fi -success "openssl available" +# Validate openssl — required for all cert checks +validate_unraid_cmd \ + "$(command -v openssl 2>/dev/null || echo /usr/bin/openssl)" \ + "version" "OpenSSL" \ + "openssl" || { error "openssl not found — required for certificate checks"; exit 1; } +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + +acquire_lock + +# detect_hosts() sets MY_ID and aliases HOST*_CERT_MONITOR_DOMAINS +detect_hosts + +# Empty array guard if [[ ${#CERT_MONITOR_DOMAINS[@]} -eq 0 ]]; then - warn "CERT_MONITOR_DOMAINS is empty in Master.conf — add your domains to enable monitoring" + warn "CERT_MONITOR_DOMAINS is empty for $MY_ID" + warn "Check HOST*_CERT_MONITOR_DOMAINS in master_host*.conf" exit 0 fi -info "$ICON_CERT Domains to check: ${#CERT_MONITOR_DOMAINS[@]}" +log "Domains to check: ${#CERT_MONITOR_DOMAINS[@]}" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — results shown but no notifications sent" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_CERT Domains: ${CERT_MONITOR_DOMAINS[*]}" - echo "$ICON_WARN Warn at: ${CERT_WARN_DAYS} days remaining" - echo "$ICON_ERROR Crit at: ${CERT_CRIT_DAYS} days remaining" - echo "$ICON_TIME Timeout: ${CERT_TIMEOUT}s per domain" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_CERT Domains: ${CERT_MONITOR_DOMAINS[*]}" + echo "$ICON_WARN Warn at: ${CERT_WARN_DAYS} days remaining" + echo "$ICON_ERROR Crit at: ${CERT_CRIT_DAYS} days remaining" + echo "$ICON_TIME Timeout: ${CERT_TIMEOUT}s per domain" + echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — results shown but no notifications sent" - -# ----------------------------------------------------------------------------------------------- -# CERT CHECK FUNCTION +# ============================================================================================== +# ── CERT CHECK FUNCTION ─────────────────────────────────────────────────────────────────────── +# ============================================================================================== # Connects to domain:443 via openssl, extracts expiry date, calculates days remaining. -# Returns 0=healthy 1=warning 2=critical 3=failed -# ----------------------------------------------------------------------------------------------- +# Returns: +# 0 = healthy (> CERT_WARN_DAYS) +# 1 = warning (<= CERT_WARN_DAYS) +# 2 = critical (<= CERT_CRIT_DAYS) +# 3 = failed (could not connect or parse) + check_cert() { local domain="$1" local port="${2:-443}" @@ -79,7 +136,7 @@ check_cert() { 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) if [[ -z "$expiry_str" ]]; then - error "$ICON_CERT $domain — could not retrieve certificate" + error "$ICON_CERT $domain — could not retrieve certificate (unreachable or no TLS)" return 3 fi @@ -103,18 +160,19 @@ check_cert() { warn "$ICON_CERT $domain — WARNING: ${days_remaining} days remaining (expires $expiry_display)" return 1 else - success "$ICON_CERT $domain — OK: ${days_remaining} days remaining (expires $expiry_display)" + log "$ICON_CERT $domain — OK: ${days_remaining} days remaining (expires $expiry_display)" return 0 fi } -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_CERT Certificate Monitor ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Certificate Monitor ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_CERT Certificate Monitor — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -echo "$ICON_WARN Warn threshold: ${CERT_WARN_DAYS} days" -echo "$ICON_ERROR Crit threshold: ${CERT_CRIT_DAYS} days" +echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)" +log "Warn threshold: ${CERT_WARN_DAYS} days" +log "Crit threshold: ${CERT_CRIT_DAYS} days" echo "" START=$(date +%s) @@ -126,7 +184,6 @@ declare -A DOMAIN_STATUS for domain in "${CERT_MONITOR_DOMAINS[@]}"; do [[ -z "$domain" ]] && continue - echo "━━━ $ICON_CERT $domain ━━━" check_cert "$domain" result=$? case $result in @@ -135,46 +192,57 @@ for domain in "${CERT_MONITOR_DOMAINS[@]}"; do 2) CRITICAL+=("$domain"); DOMAIN_STATUS["$domain"]="CRIT" ;; 3) FAILED+=("$domain"); DOMAIN_STATUS["$domain"]="FAIL" ;; esac - echo "" done END=$(date +%s) +# ── Send notifications — batched per severity ───────────────────────────────────────────────── if [[ "$DRY_RUN" == false ]]; then [[ ${#CRITICAL[@]} -gt 0 ]] && \ - notify "Certificate CRITICAL on $(hostname) — expiring within ${CERT_CRIT_DAYS} days: ${CRITICAL[*]}" "Certificate Monitor" "warning" + notify "Certificate CRITICAL on $(hostname) — expiring within ${CERT_CRIT_DAYS} days: ${CRITICAL[*]}" \ + "Certificate Monitor" "warning" [[ ${#WARNING[@]} -gt 0 ]] && \ - notify "Certificate WARNING on $(hostname) — expiring within ${CERT_WARN_DAYS} days: ${WARNING[*]}" "Certificate Monitor" "warning" + notify "Certificate WARNING on $(hostname) — expiring within ${CERT_WARN_DAYS} days: ${WARNING[*]}" \ + "Certificate Monitor" "warning" [[ ${#FAILED[@]} -gt 0 ]] && \ - notify "Certificate check FAILED on $(hostname) — could not reach: ${FAILED[*]}" "Certificate Monitor" "warning" + notify "Certificate check FAILED on $(hostname) — could not reach: ${FAILED[*]}" \ + "Certificate Monitor" "warning" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== +echo "" echo "━━━━━ $ICON_SUMMARY CERTIFICATE MONITOR SUMMARY ━━━━━" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" -echo " $ICON_SUCCESS Healthy: ${#HEALTHY[@]} $ICON_WARN Warning: ${#WARNING[@]} $ICON_ERROR Critical: ${#CRITICAL[@]} Failed: ${#FAILED[@]}" +echo " $ICON_SUCCESS Healthy: ${#HEALTHY[@]}" +[[ ${#WARNING[@]} -gt 0 ]] && warn "Warning: ${#WARNING[@]} — renewal recommended" +[[ ${#CRITICAL[@]} -gt 0 ]] && echo "$ICON_ERROR Critical: ${#CRITICAL[@]} — ACTION REQUIRED" +[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${#FAILED[@]} — unreachable" echo "" + +# Per-domain results — only show problems, healthy ones stay in log() for domain in "${CERT_MONITOR_DOMAINS[@]}"; do [[ -z "$domain" ]] && continue case "${DOMAIN_STATUS[$domain]:-UNKN}" in - OK) echo " $ICON_SUCCESS $domain" ;; - WARN) echo " $ICON_WARN $domain" ;; - CRIT) echo " $ICON_ERROR $domain" ;; - FAIL) echo " $ICON_ERROR $domain (unreachable)" ;; + OK) log " $ICON_SUCCESS $domain — healthy" ;; + WARN) warn " $ICON_WARN $domain — warning" ;; + CRIT) echo " $ICON_ERROR $domain — CRITICAL" ;; + FAIL) echo " $ICON_ERROR $domain — unreachable" ;; esac done echo "" + if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no notifications sent" + warn "DRY RUN — no notifications sent" elif [[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]]; then echo "$ICON_ERROR Status: ACTION REQUIRED" elif [[ ${#WARNING[@]} -gt 0 ]]; then - echo "$ICON_WARN Status: WARNINGS — renewal recommended" + warn "Status: WARNINGS — renewal recommended" else - echo "$ICON_DONE Status: $ICON_SUCCESS ALL CERTS HEALTHY" + log "$ICON_DONE Status: all ${#HEALTHY[@]} certs healthy ✅" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" diff --git a/Monitors/continuous_scripts_status.sh b/Monitors/continuous_scripts_status.sh index 494b44c..06a2a63 100644 --- a/Monitors/continuous_scripts_status.sh +++ b/Monitors/continuous_scripts_status.sh @@ -1,35 +1,58 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# ----------------------------- Continuous Scripts Status -------------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ========================= Continuous Scripts Status ========================================== +# ============================================================================================== # Live status dashboard for all continuously running scripts in the ecosystem. # Run manually anytime — no schedule, no cron. # -# Covers all scripts started by array_start.sh that run until array stops: -# system_watchdog.sh — system health monitor -# docker_watchdog.sh — container health monitor -# failover.sh — mutual failover monitor +# ── WHAT IT SHOWS ───────────────────────────────────────────────────────────────────────────── +# For each continuous script (system_watchdog, docker_watchdog, failover): +# Running state, PID, uptime, approximate cycle count +# Active strikes and skip list +# Recent restart history +# Live health snapshot # -# Shows for each: -# Running state, PID, uptime, current cycle -# Active strikes, skip list, recent actions -# Live system/container health snapshot -# Failover state, tier status, Tailscale connectivity +# system_watchdog — rootfs, RAM, ZFS ARC, load, zombie count, CPU temp +# docker_watchdog — running/stopped/unhealthy containers, required containers, +# monitored memory containers, recent restart history +# failover — current state, tier status, remote Tailscale visibility # -# If a script is mid-cycle state files are read as-is — reflects last completed cycle. -# Run: bash Monitors/continuous_scripts_status.sh -# ----------------------------------------------------------------------------------------------- +# If a script is mid-cycle, state files are read as-is — reflects last completed cycle. +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays. +# Required containers, tier delays, and Tailscale checks use MY_ID/REMOTE_ID correctly. +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# continuous_scripts_status.sh — show dashboard +# continuous_scripts_status.sh --log — verbose output +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" + +# Dashboard script — output is the point +SILENT_MODE=false parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# HELPERS -# ----------------------------------------------------------------------------------------------- +DOCKER_TIMEOUT=15 + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays +detect_hosts + +# ============================================================================================== +# ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== get_lock_pid() { local script_name="$1" @@ -51,11 +74,10 @@ get_lock_name() { fi } -is_watchdog_running() { +is_script_running() { local script_name="$1" - local pid + local pid locked_name pid=$(get_lock_pid "$script_name") - local locked_name locked_name=$(get_lock_name "$script_name") [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$locked_name" == "$script_name" ]] } @@ -73,6 +95,7 @@ get_lock_age() { fi } +# Human readable uptime — days/hours/mins format_uptime() { local seconds=$1 local days=$(( seconds / 86400 )) @@ -87,34 +110,29 @@ format_uptime() { fi } -get_strikes() { - local state_file="$1" - local key="$2" - grep -E "^${key}:" "$state_file" 2>/dev/null | cut -d: -f2 -} +divider() { printf '%.0s─' {1..57}; echo; } +section() { echo ""; echo " $1"; divider; } -divider() { printf '%.0s─' {1..55}; echo; } -header() { echo ""; echo " $1"; divider; } - -# ----------------------------------------------------------------------------------------------- -# ━━━ HEADER ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Header ━━━ +# ============================================================================================== clear echo "" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo " 🛡️ WATCHDOG STATUS — $(date '+%A, %B %-d at %-I:%M%p')" -echo " 🖥️ $(hostname)" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " $ICON_HOST $MY_ID — $LOCAL_SERVER_NAME" +echo " $ICON_HOST Remote: $REMOTE_ID — $REMOTE_SERVER_NAME" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== # ━━━ System Watchdog ━━━ -# ----------------------------------------------------------------------------------------------- -header "⚙️ SYSTEM WATCHDOG" +# ============================================================================================== +section "⚙️ SYSTEM WATCHDOG" SYS_PID=$(get_lock_pid "system_watchdog") SYS_RUNNING=false -if is_watchdog_running "system_watchdog"; then +if is_script_running "system_watchdog"; then SYS_RUNNING=true SYS_AGE=$(get_lock_age "system_watchdog") SYS_UPTIME=$(format_uptime "$SYS_AGE") @@ -128,7 +146,7 @@ fi echo "" -# System watchdog strikes +# System strikes if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then ACTIVE_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$") if [[ -n "$ACTIVE_STRIKES" ]]; then @@ -149,8 +167,8 @@ if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then TOTAL_REBOOTS=$(grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0) TOTAL_REBOOTS="${TOTAL_REBOOTS//[^0-9]/}" TOTAL_REBOOTS="${TOTAL_REBOOTS:-0}" - WEEK_EPOCH=$(date -d "7 days ago" +%s) - WEEK_REBOOTS=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \ + WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S') + WEEK_REBOOTS=$(awk -v cutoff="$WEEK_CUTOFF" '$0 >= cutoff' \ "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l) echo " 🔄 Watchdog reboots: $WEEK_REBOOTS this week / $TOTAL_REBOOTS total" fi @@ -159,7 +177,7 @@ fi if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE") echo "" - echo " ⛔ Skip list ($SKIP_COUNT containers — manual intervention needed):" + echo " ⛔ Skip list ($SKIP_COUNT — manual intervention needed):" while IFS= read -r container; do [[ -z "$container" ]] && continue echo " → $container" @@ -168,17 +186,15 @@ else echo " ✅ Skip list: empty" fi -# Current system health snapshot +# Live system health snapshot echo "" echo " 📊 Current system state:" -# rootfs ROOTFS_PCT=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %') [[ "${ROOTFS_PCT:-0}" -ge "${SYS_WATCHDOG_ROOTFS_PCT:-95}" ]] && \ ROOTFS_ICON="⚠️ " || ROOTFS_ICON="✅" echo " ${ROOTFS_ICON} rootfs: ${ROOTFS_PCT}% (threshold: ${SYS_WATCHDOG_ROOTFS_PCT}%)" -# RAM MEM_AVAIL_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) MEM_FREE_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_AVAIL_KB / 1048576}") MEM_TOTAL_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo) @@ -186,7 +202,6 @@ MEM_TOTAL_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo) MEM_ICON="⚠️ " || MEM_ICON="✅" echo " ${MEM_ICON} RAM: ${MEM_FREE_GB}GB free / ${MEM_TOTAL_GB}GB total (threshold: ${SYS_WATCHDOG_MEM_GB}GB free)" -# ARC if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats) @@ -197,7 +212,6 @@ if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then echo " ${ARC_ICON} ZFS ARC: ${ARC_GB}GB (${ARC_PCT}% of max, threshold: ${SYS_WATCHDOG_ARC_PINNED_PCT}%)" fi -# Load LOAD=$(awk '{print $1}' /proc/loadavg) CORES=$(nproc) LOAD_THRESH=$(( CORES * ${SYS_WATCHDOG_LOAD_MULTIPLIER:-3} )) @@ -205,17 +219,13 @@ LOAD_INT=$(printf "%.0f" "$LOAD") [[ "$LOAD_INT" -ge "$LOAD_THRESH" ]] && LOAD_ICON="⚠️ " || LOAD_ICON="✅" echo " ${LOAD_ICON} Load avg: $LOAD (threshold: ${LOAD_THRESH} = ${SYS_WATCHDOG_LOAD_MULTIPLIER}x ${CORES} cores)" -# Zombies ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0) ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}" ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}" -ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}" -ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}" [[ "$ZOMBIE_COUNT" -ge "${SYS_WATCHDOG_ZOMBIE_LIMIT:-50}" ]] && \ ZOMBIE_ICON="⚠️ " || ZOMBIE_ICON="✅" echo " ${ZOMBIE_ICON} Zombies: $ZOMBIE_COUNT (threshold: ${SYS_WATCHDOG_ZOMBIE_LIMIT})" -# CPU temp if command -v sensors >/dev/null 2>&1; then CPU_TEMP=$(sensors 2>/dev/null | \ grep -i "Package id 0\|Tctl\|CPU Temp" | \ @@ -228,15 +238,15 @@ if command -v sensors >/dev/null 2>&1; then fi fi -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== # ━━━ Docker Watchdog ━━━ -# ----------------------------------------------------------------------------------------------- -header "🐳 DOCKER WATCHDOG" +# ============================================================================================== +section "🐳 DOCKER WATCHDOG" DOCKER_PID=$(get_lock_pid "docker_watchdog") DOCKER_RUNNING=false -if is_watchdog_running "docker_watchdog"; then +if is_script_running "docker_watchdog"; then DOCKER_RUNNING=true DOCKER_AGE=$(get_lock_age "docker_watchdog") DOCKER_UPTIME=$(format_uptime "$DOCKER_AGE") @@ -250,7 +260,7 @@ fi echo "" -# Container watchdog strikes +# Container strikes if [[ -f "$WATCHDOG_STATE_FILE" ]]; then ACTIVE_CONTAINER_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$") if [[ -n "$ACTIVE_CONTAINER_STRIKES" ]]; then @@ -264,16 +274,15 @@ if [[ -f "$WATCHDOG_STATE_FILE" ]]; then fi fi -# Container restart history this week +# Container restart history if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then - WEEK_EPOCH=$(date -d "7 days ago" +%s 2>/dev/null || date -v-7d +%s 2>/dev/null) - WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_EPOCH" \ - 'NR>0 {if ($2 >= cutoff) count++} END {print count+0}' \ - "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null) + WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S') + WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \ + '$2 >= cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l) if [[ "${WEEK_RESTARTS:-0}" -gt 0 ]]; then echo "" echo " 🔄 Container restarts this week: $WEEK_RESTARTS" - awk -F'|' -v cutoff="$WEEK_EPOCH" \ + awk -F'|' -v cutoff="$WEEK_CUTOFF" \ '$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \ sort | uniq -c | sort -rn | head -5 | \ while read -r count name; do @@ -284,31 +293,34 @@ if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then fi fi -# Docker container overview +# Container overview echo "" echo " 📦 Container overview:" if command -v docker >/dev/null 2>&1; then - RUNNING=$(docker ps -q 2>/dev/null | wc -l) - TOTAL=$(docker ps -aq 2>/dev/null | wc -l) - UNHEALTHY=$(docker ps --filter health=unhealthy -q 2>/dev/null | wc -l) + RUNNING=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l) + TOTAL=$(timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l) + UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \ + --filter health=unhealthy -q 2>/dev/null | wc -l) - # Filter intentionally stopped containers from the stopped list + # Stopped containers — filter intentionally ignored ones STOPPED_FILTERED=() while IFS= read -r name; do [[ -z "$name" ]] && continue - SKIP=false + local SKIP=false for ignore in "${WATCHDOG_SCAN_IGNORE[@]:-}"; do [[ "$name" == "$ignore" ]] && SKIP=true && break done [[ "$SKIP" == false ]] && STOPPED_FILTERED+=("$name") - done < <(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null) - STOPPED_COUNT="${#STOPPED_FILTERED[@]}" + done < <(timeout "$DOCKER_TIMEOUT" docker ps -af "status=exited" \ + --format "{{.Names}}" 2>/dev/null) + STOPPED_COUNT="${#STOPPED_FILTERED[@]}" echo " Running: $RUNNING / $TOTAL total" [[ "$UNHEALTHY" -gt 0 ]] && echo " ⚠️ Unhealthy: $UNHEALTHY" + if [[ "$STOPPED_COUNT" -gt 0 ]]; then - echo " ⚠️ Stopped containers (unexpected):" + echo " ⚠️ Stopped (unexpected):" for name in "${STOPPED_FILTERED[@]}"; do echo " → $name" done @@ -316,40 +328,35 @@ if command -v docker >/dev/null 2>&1; then echo " ✅ All containers running" fi - # Check required containers - detect_hosts 2>/dev/null - if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - REQUIRED=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}") - else - REQUIRED=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}") - fi - + # Required containers — aliased by detect_hosts() → WATCHDOG_REQUIRED_CONTAINERS REQUIRED_ISSUES=0 - if [[ ${#REQUIRED[@]} -gt 0 ]]; then + if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then echo "" echo " 🔐 Required containers:" - for container in "${REQUIRED[@]}"; do + for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do [[ -z "$container" ]] && continue - STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "not found") + STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ + '{{.State.Running}}' "$container" 2>/dev/null || echo "not found") if [[ "$STATUS" == "true" ]]; then echo " ✅ $container" else echo " ❌ $container — $STATUS" - ((REQUIRED_ISSUES++)) + (( REQUIRED_ISSUES++ )) fi done fi - # Tier 1 monitored containers from WATCHDOG_CONTAINERS + # Memory-monitored containers — aliased by detect_hosts() → WATCHDOG_CONTAINERS if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then echo "" echo " 📊 Monitored containers (memory):" for container in "${!WATCHDOG_CONTAINERS[@]}"; do LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}" LIMIT_GB=$(awk "BEGIN {printf \"%.0f\", $LIMIT_MB / 1024}") - USAGE=$(docker stats --no-stream --format "{{.MemUsage}}" "$container" \ - 2>/dev/null | awk '{print $1}') - STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "not found") + USAGE=$(timeout "$DOCKER_TIMEOUT" docker stats --no-stream \ + --format "{{.MemUsage}}" "$container" 2>/dev/null | awk '{print $1}') + STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ + '{{.State.Running}}' "$container" 2>/dev/null || echo "not found") if [[ "$STATUS" == "true" ]]; then echo " ✅ $container: ${USAGE:-?} (limit: ${LIMIT_GB}GB)" else @@ -361,22 +368,22 @@ else echo " Docker not available" fi -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== # ━━━ Failover ━━━ -# ----------------------------------------------------------------------------------------------- -header "🔀 FAILOVER" +# ============================================================================================== +section "🔀 FAILOVER" FAILOVER_PID=$(get_lock_pid "failover") FAILOVER_RUNNING=false -if is_watchdog_running "failover"; then +if is_script_running "failover"; then FAILOVER_RUNNING=true FAILOVER_AGE=$(get_lock_age "failover") FAILOVER_UPTIME=$(format_uptime "$FAILOVER_AGE") echo " ✅ Running │ PID: $FAILOVER_PID │ Uptime: $FAILOVER_UPTIME" else if [[ "${FAILOVER_ENABLED:-true}" == false ]]; then - echo " ⏸️ Disabled — FAILOVER_ENABLED=false in Master.conf" + echo " ⏸️ Disabled — FAILOVER_ENABLED=false in master.conf" else echo " ❌ NOT RUNNING — failover.sh is not active" echo " Start via: bash Orchestrators/array_start.sh" @@ -387,60 +394,55 @@ echo "" # Failover state FAILOVER_STATE="UNKNOWN" -FAILOVER_LAST_CHANGE="" FAILOVER_STATE_SECONDS=0 if [[ -f "$FAILOVER_STATE_FILE" ]]; then FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) - FAILOVER_LAST_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) - FAILOVER_LAST_EPOCH=$(grep "^last_change_epoch=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) - if [[ -n "$FAILOVER_LAST_EPOCH" ]]; then + FAILOVER_LAST_EPOCH=$(grep "^failover_start=" "$FAILOVER_STATE_FILE" \ + 2>/dev/null | cut -d= -f2) + if [[ -n "$FAILOVER_LAST_EPOCH" && "$FAILOVER_LAST_EPOCH" -gt 0 ]]; then FAILOVER_STATE_SECONDS=$(( $(date +%s) - FAILOVER_LAST_EPOCH )) fi fi STATE_DURATION=$(format_uptime "${FAILOVER_STATE_SECONDS:-0}") +# Tier delays via REMOTE_ID — same logic as failover.sh +REMOTE_TIER2_VAR="${REMOTE_ID}_TIER2_DELAY" +REMOTE_TIER3_VAR="${REMOTE_ID}_TIER3_DELAY" +REMOTE_TIER4_VAR="${REMOTE_ID}_TIER4_DELAY" +TIER2_DELAY="${!REMOTE_TIER2_VAR:-240}" +TIER3_DELAY="${!REMOTE_TIER3_VAR:-720}" +TIER4_DELAY="${!REMOTE_TIER4_VAR:-1440}" + case "$FAILOVER_STATE" in NORMAL) echo " ✅ State: NORMAL" - echo " 📅 In NORMAL state for: $STATE_DURATION" ;; FAILOVER) - echo " ⚠️ State: FAILOVER — remote server down" + echo " ⚠️ State: FAILOVER — $REMOTE_SERVER_NAME is down" echo " ⏱️ Duration: $STATE_DURATION" - # Show which tiers are active - TIER1_DELAY=0 - if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - TIER2_DELAY=$HOST2_TIER2_DELAY - TIER3_DELAY=$HOST2_TIER3_DELAY - TIER4_DELAY=$HOST2_TIER4_DELAY - else - TIER2_DELAY=$HOST1_TIER2_DELAY - TIER3_DELAY=$HOST1_TIER3_DELAY - TIER4_DELAY=$HOST1_TIER4_DELAY - fi FAILOVER_MINS=$(( FAILOVER_STATE_SECONDS / 60 )) echo "" echo " 🔄 Tier status:" - echo " Tier 1 (immediate): ✅ active" + echo " Tier 1 (immediate): ✅ active" if (( FAILOVER_MINS >= TIER2_DELAY )); then - echo " Tier 2 (${TIER2_DELAY}min): ✅ active" + echo " Tier 2 (${TIER2_DELAY}min): ✅ active" else REMAINING=$(( TIER2_DELAY - FAILOVER_MINS )) - echo " Tier 2 (${TIER2_DELAY}min): ⏳ activates in ${REMAINING}min" + echo " Tier 2 (${TIER2_DELAY}min): ⏳ in ${REMAINING}min" fi if (( FAILOVER_MINS >= TIER3_DELAY )); then - echo " Tier 3 (${TIER3_DELAY}min): ✅ active" + echo " Tier 3 (${TIER3_DELAY}min): ✅ active" else REMAINING=$(( TIER3_DELAY - FAILOVER_MINS )) - echo " Tier 3 (${TIER3_DELAY}min): ⏳ activates in ${REMAINING}min" + echo " Tier 3 (${TIER3_DELAY}min): ⏳ in ${REMAINING}min" fi if (( FAILOVER_MINS >= TIER4_DELAY )); then - echo " Tier 4 (${TIER4_DELAY}min): ✅ active" + echo " Tier 4 (${TIER4_DELAY}min): ✅ active" else REMAINING=$(( TIER4_DELAY - FAILOVER_MINS )) - echo " Tier 4 (${TIER4_DELAY}min): ⏳ activates in ${REMAINING}min" + echo " Tier 4 (${TIER4_DELAY}min): ⏳ in ${REMAINING}min" fi ;; NO_INTERNET) @@ -448,7 +450,7 @@ case "$FAILOVER_STATE" in echo " ⏱️ Down for: $STATE_DURATION" ;; DARK) - echo " ❌ State: DARK — remote down AND no internet" + echo " ❌ State: DARK — $REMOTE_SERVER_NAME down AND no internet" echo " ⏱️ Duration: $STATE_DURATION" ;; *) @@ -456,19 +458,18 @@ case "$FAILOVER_STATE" in ;; esac -# Tailscale remote visibility +# Tailscale remote visibility — uses REMOTE_SERVER_NAME from detect_hosts() echo "" if command -v tailscale >/dev/null 2>&1; then - REMOTE_IP=$(tailscale ip -4 "$HOST2" 2>/dev/null) + REMOTE_IP=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) if [[ -n "$REMOTE_IP" ]]; then - # Try a quick ping to see last seen if ping -c 1 -W 2 "$REMOTE_IP" >/dev/null 2>&1; then - echo " 🌐 Remote: $REMOTE_IP │ reachable ✅" + echo " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_IP — reachable ✅" else - echo " 🌐 Remote: $REMOTE_IP │ not responding ⚠️" + echo " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_IP — not responding ⚠️" fi else - echo " 🌐 Remote: $HOST2 not visible on Tailscale ❌" + echo " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME) not visible on Tailscale ❌" fi else echo " 🌐 Tailscale: not available" @@ -476,28 +477,27 @@ fi echo " 📡 Check interval: ${FAILOVER_CHECK_INTERVAL}s │ Handback strikes: ${FAILOVER_HANDBACK_STRIKES}" -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== # ━━━ Footer ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== echo "" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -# Overall status ISSUES=0 -[[ "$SYS_RUNNING" == false ]] && ((ISSUES++)) -[[ "$DOCKER_RUNNING" == false ]] && ((ISSUES++)) -[[ "$FAILOVER_RUNNING" == false ]] && [[ "${FAILOVER_ENABLED:-true}" != false ]] && ((ISSUES++)) -[[ -n "$ACTIVE_STRIKES" ]] && ((ISSUES++)) -[[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && ((ISSUES++)) -[[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && ((ISSUES++)) -[[ "$FAILOVER_STATE" != "NORMAL" ]] && [[ "$FAILOVER_STATE" != "UNKNOWN" ]] && ((ISSUES++)) +[[ "$SYS_RUNNING" == false ]] && (( ISSUES++ )) +[[ "$DOCKER_RUNNING" == false ]] && (( ISSUES++ )) +[[ "$FAILOVER_RUNNING" == false && "${FAILOVER_ENABLED:-true}" != false ]] && (( ISSUES++ )) +[[ -n "$ACTIVE_STRIKES" ]] && (( ISSUES++ )) +[[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && (( ISSUES++ )) +[[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && (( ISSUES++ )) +[[ "$FAILOVER_STATE" != "NORMAL" && "$FAILOVER_STATE" != "UNKNOWN" ]] && (( ISSUES++ )) if [[ "$ISSUES" -eq 0 ]]; then - echo " ✅ All continuous scripts healthy — no issues detected" + echo " ✅ $MY_ID — all continuous scripts healthy" else echo " ⚠️ $ISSUES issue(s) detected — review above" fi -echo " 🕐 Checked at: $(date '+%H:%M:%S')" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " 🕐 Checked: $(date '+%H:%M:%S')" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" \ No newline at end of file diff --git a/Monitors/emby_session_report.sh b/Monitors/emby_session_report.sh index c62cf56..1728b91 100644 --- a/Monitors/emby_session_report.sh +++ b/Monitors/emby_session_report.sh @@ -1,37 +1,75 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Emby Session Report ---------------------------------------- -# ----------------------------------------------------------------------------------------------- -# Generates a weekly usage report from the Emby media server via its API. -# Queries activity logs, session history and library stats to produce a -# human-readable summary of what was watched, by whom and how. +# ============================================================================================== +# ================================= Emby Session Report ======================================== +# ============================================================================================== +# Generates a usage report from the Emby media server via its API. +# Queries activity logs and session history to produce a summary of what was +# watched, by whom, and how over the configured report period. # -# Report includes: -# Total streams during the report period -# Transcode vs direct play ratio -# Live TV usage -# Top N most watched content -# Most active users -# Peak concurrent streams +# ── REPORT INCLUDES ─────────────────────────────────────────────────────────────────────────── +# Server info — name, version, uptime +# Active sessions — current streams, direct play vs transcode +# Library stats — movie, episode, song counts +# Activity history — play events from the last EMBY_REPORT_DAYS days +# Top content — most played items in the period (top EMBY_REPORT_TOP_N) +# Most active users — who watched the most in the period +# Transcode ratio — how often transcoding was needed vs direct play +# Ramdisk status — current transcode location and usage # +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID and aliases EMBY_URL and EMBY_API_KEY. +# Each server reports on its own Emby instance automatically. +# +# ── BEHAVIOUR ───────────────────────────────────────────────────────────────────────────────── # No persistent writes — queries API fresh each run. -# All configuration in Master.conf under Emby Session Report section. -# Supports --dry-run to test API connectivity without sending notification. -# ----------------------------------------------------------------------------------------------- +# This is a monitor/report script — SILENT_MODE=false — output is the point. +# Silent when healthy (no notification on clean run). +# Notifies only if transcoding is very high (>80% of streams) — may indicate config issue. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock — prevents duplicate reports running simultaneously +# check_api() — verifies Emby reachable before queries +# jq + curl validation — exits if either tool missing +# validate_unraid_cmd — notify script validated before use +# Per-section guards — API failure in one section does not abort others +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_EMBY_URL / HOST*_EMBY_API_KEY +# Aliased by detect_hosts() — script uses EMBY_URL / EMBY_API_KEY +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# EMBY_REPORT_DAYS — days to include in the report period (default 7) +# EMBY_REPORT_TOP_N — number of top content items to show (default 10) +# RAMDISK_PATH — ramdisk mount path (for transcode status) +# TRANSCODE_LINK — symlink path (for transcode location) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# emby_session_report.sh — generate report +# emby_session_report.sh --dry-run — test API connectivity only, no notification +# emby_session_report.sh --log — verbose output +# emby_session_report.sh --status — show config and exit +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" + +# Monitor/report script — output is the point +SILENT_MODE=false parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + if ! command -v curl >/dev/null 2>&1; then error "curl not found — required for Emby API calls" exit 1 @@ -43,28 +81,40 @@ if ! command -v jq >/dev/null 2>&1; then exit 1 fi -# Select correct Emby instance based on which server is running this script +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + +acquire_lock + +# detect_hosts() sets MY_ID and aliases EMBY_URL, EMBY_API_KEY detect_hosts -if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - EMBY_URL="$HOST1_EMBY_URL" - EMBY_API_KEY="$HOST1_EMBY_API_KEY" -else - EMBY_URL="$HOST2_EMBY_URL" - EMBY_API_KEY="$HOST2_EMBY_API_KEY" -fi - -info "Emby instance: $LOCAL_SERVER_NAME → $EMBY_URL" - require_var EMBY_URL require_var EMBY_API_KEY -success "Config validated" -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — API will be queried but no notification sent" +log "Emby: $EMBY_URL" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — API queried but no notification sent" -# ----------------------------------------------------------------------------------------------- -# API HELPER -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_EMBY Emby URL: $EMBY_URL" + echo "$ICON_TIME Period: Last ${EMBY_REPORT_DAYS} days" + echo "$ICON_EMBY Top N: ${EMBY_REPORT_TOP_N} items" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ── API HELPER ──────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== emby_api() { local endpoint="$1" local response http_code body @@ -79,54 +129,66 @@ emby_api() { body=$(echo "$response" | head -n -1) if [[ "$http_code" != "200" ]]; then - error "Emby API returned HTTP $http_code for: $endpoint" + error "Emby API HTTP $http_code for: $endpoint" return 1 fi - echo "$body" } -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_EMBY Emby Session Report ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Emby Session Report ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_EMBY Emby Session Report — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -echo "$ICON_EMBY URL: $EMBY_URL" -echo "$ICON_TIME Period: Last ${EMBY_REPORT_DAYS} days" +echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_TIME Period: Last ${EMBY_REPORT_DAYS} days" echo "" START=$(date +%s) -# Test connectivity -info "Testing Emby API connectivity..." +# ── Connectivity and server info ────────────────────────────────────────────────────────────── +if ! check_api "$EMBY_URL" "Emby" 10; then + notify "Emby report failed on $(hostname) — cannot connect to Emby at $EMBY_URL" \ + "Emby Report" "warning" + exit 1 +fi + SYSTEM_INFO=$(emby_api "System/Info" 2>/dev/null) || { error "Cannot connect to Emby at $EMBY_URL" - notify "Emby report failed on $(hostname) — cannot connect to Emby" "Emby Report" "warning" exit 1 } SERVER_NAME=$(echo "$SYSTEM_INFO" | jq -r '.ServerName // "Unknown"' 2>/dev/null) SERVER_VERSION=$(echo "$SYSTEM_INFO" | jq -r '.Version // "Unknown"' 2>/dev/null) -success "Connected to: $SERVER_NAME (v$SERVER_VERSION)" -echo "" +log "Connected to: $SERVER_NAME (v$SERVER_VERSION)" -# Calculate date range -REPORT_START=$(date -d "${EMBY_REPORT_DAYS} days ago" '+%Y-%m-%dT00:00:00') - -# ── Active Sessions ────────────────────────────────────────────────────────────────────────── +# ── Active Sessions ─────────────────────────────────────────────────────────────────────────── echo "━━━ $ICON_EMBY Active Sessions ━━━" SESSIONS=$(emby_api "Sessions" 2>/dev/null) || { warn "Could not fetch sessions"; SESSIONS="[]"; } -ACTIVE_COUNT=$(echo "$SESSIONS" | jq '[.[] | select(.NowPlayingItem != null)] | length' 2>/dev/null || echo 0) -TRANSCODE_COUNT=$(echo "$SESSIONS" | jq '[.[] | select(.NowPlayingItem != null) | select(.TranscodingInfo != null)] | length' 2>/dev/null || echo 0) -DIRECT_COUNT=$(( ACTIVE_COUNT - TRANSCODE_COUNT )) +ACTIVE_COUNT=$(echo "$SESSIONS" | \ + jq '[.[] | select(.NowPlayingItem != null)] | length' 2>/dev/null || echo 0) +TRANSCODE_NOW=$(echo "$SESSIONS" | \ + jq '[.[] | select(.NowPlayingItem != null) | select(.TranscodingInfo != null)] | length' \ + 2>/dev/null || echo 0) +DIRECT_NOW=$(( ACTIVE_COUNT - TRANSCODE_NOW )) -echo " $ICON_EMBY Active streams: $ACTIVE_COUNT" -echo " $ICON_EMBY Direct play: $DIRECT_COUNT" -echo " $ICON_EMBY Transcoding: $TRANSCODE_COUNT" +echo " $ICON_EMBY Active streams: $ACTIVE_COUNT" +echo " $ICON_EMBY Direct play: $DIRECT_NOW" +echo " $ICON_EMBY Transcoding: $TRANSCODE_NOW" + +if [[ "$ACTIVE_COUNT" -gt 0 ]]; then + echo "" + echo " Now playing:" + echo "$SESSIONS" | jq -r ' + .[] | + select(.NowPlayingItem != null) | + " \(.UserName // "Unknown") → \(.NowPlayingItem.Name // "Unknown") [\(if .TranscodingInfo != null then "transcode" else "direct" end)]" + ' 2>/dev/null || true +fi echo "" -# ── Library Stats ──────────────────────────────────────────────────────────────────────────── +# ── Library Stats ───────────────────────────────────────────────────────────────────────────── echo "━━━ $ICON_EMBY Library ━━━" ITEMS=$(emby_api "Items/Counts" 2>/dev/null) || { warn "Could not fetch library counts"; ITEMS="{}"; } @@ -139,31 +201,105 @@ echo " $ICON_EMBY Episodes: $EPISODE_COUNT" echo " $ICON_EMBY Songs: $SONG_COUNT" echo "" -# ── Ramdisk Status (from state file) ──────────────────────────────────────────────────────── -echo "━━━ $ICON_RAM Transcode Location ━━━" -if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then - RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ') - RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}") - SYMLINK=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown") - echo " $ICON_RAM Ramdisk usage: ${RAMDISK_USED_GB}GB" - echo " $ICON_LINK Symlink target: $SYMLINK" +# ── Activity History ────────────────────────────────────────────────────────────────────────── +# Query activity log for the configured period +echo "━━━ $ICON_EMBY Activity — Last ${EMBY_REPORT_DAYS} Days ━━━" + +REPORT_START=$(date -d "${EMBY_REPORT_DAYS} days ago" '+%Y-%m-%dT00:00:00.000Z') + +ACTIVITY=$(emby_api "System/ActivityLog/Entries?MinDate=${REPORT_START}&Limit=1000" \ + 2>/dev/null) || { warn "Could not fetch activity log"; ACTIVITY="{}"; } + +TOTAL_PLAYS=$(echo "$ACTIVITY" | \ + jq '[.Items // [] | .[] | select(.Type == "VideoPlayback" or .Type == "AudioPlayback")] | length' \ + 2>/dev/null || echo 0) + +TRANSCODE_PLAYS=$(echo "$ACTIVITY" | \ + jq '[.Items // [] | .[] | select(.Type == "VideoPlaybackUnplugged" or + (.Type == "VideoPlayback" and (.Overview // "" | contains("Transcode"))))] | length' \ + 2>/dev/null || echo 0) + +echo " $ICON_EMBY Total play events: $TOTAL_PLAYS" + +if [[ "$TOTAL_PLAYS" -gt 0 ]]; then + TRANSCODE_PCT=$(awk "BEGIN {printf \"%.0f\", ($TRANSCODE_PLAYS / $TOTAL_PLAYS) * 100}") + DIRECT_PCT=$(( 100 - TRANSCODE_PCT )) + echo " $ICON_EMBY Direct play: ~${DIRECT_PCT}%" + echo " $ICON_EMBY Transcoded: ~${TRANSCODE_PCT}%" +fi +echo "" + +# ── Top Content ─────────────────────────────────────────────────────────────────────────────── +echo "━━━ $ICON_EMBY Top ${EMBY_REPORT_TOP_N} Content ━━━" + +TOP_ITEMS=$(emby_api "Items?SortBy=DatePlayed&SortOrder=Descending&Limit=${EMBY_REPORT_TOP_N}&Recursive=true&Fields=Overview&IncludeItemTypes=Movie,Episode" \ + 2>/dev/null) || { warn "Could not fetch top content"; TOP_ITEMS="{}"; } + +TOP_COUNT=$(echo "$TOP_ITEMS" | jq '.Items // [] | length' 2>/dev/null || echo 0) +if [[ "$TOP_COUNT" -gt 0 ]]; then + echo "$TOP_ITEMS" | jq -r ' + .Items // [] | + to_entries[] | + " \(.key + 1). \(.value.Name // "Unknown") [\(.value.Type // "")]" + ' 2>/dev/null || warn "Could not parse top content" else - echo " $ICON_RAM Ramdisk: not mounted" + echo " No recent play history found" +fi +echo "" + +# ── Most Active Users ───────────────────────────────────────────────────────────────────────── +echo "━━━ $ICON_EMBY Most Active Users ━━━" +USERS=$(emby_api "Users" 2>/dev/null) || { warn "Could not fetch users"; USERS="[]"; } + +USER_COUNT=$(echo "$USERS" | jq 'length' 2>/dev/null || echo 0) +echo " $ICON_EMBY Total users: $USER_COUNT" + +if [[ "$USER_COUNT" -gt 0 ]]; then + echo "$USERS" | jq -r ' + sort_by(.LastActivityDate // "0") | + reverse | + .[:5][] | + " \(.Name // "Unknown") — last active: \(.LastActivityDate // "never" | split("T")[0])" + ' 2>/dev/null || true +fi +echo "" + +# ── Ramdisk / Transcode Status ──────────────────────────────────────────────────────────────── +echo "━━━ $ICON_RAM Transcode Status ━━━" +if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then + RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ') + RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}") + SYMLINK=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown") + echo " $ICON_RAM Ramdisk usage: ${RAMDISK_USED_GB}GB / ${RAMDISK_SIZE:-8G}" + echo " $ICON_LINK Symlink target: $SYMLINK" + if [[ "$SYMLINK" == *"ssd"* ]] || [[ "$SYMLINK" == *"cache"* ]]; then + warn "Transcode link pointing at SSD — ramdisk may be full" + fi +else + warn "Ramdisk not mounted at $RAMDISK_PATH" fi echo "" END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "━━━━━ $ICON_SUMMARY EMBY REPORT SUMMARY ━━━━━" -echo "$ICON_EMBY Server: $SERVER_NAME (v$SERVER_VERSION)" -echo "$ICON_EMBY Active: $ACTIVE_COUNT streams ($DIRECT_COUNT direct / $TRANSCODE_COUNT transcode)" -echo "$ICON_EMBY Library: $MOVIE_COUNT movies $EPISODE_COUNT episodes $SONG_COUNT songs" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_EMBY Server: $SERVER_NAME (v$SERVER_VERSION)" +echo "$ICON_EMBY Active: $ACTIVE_COUNT streams ($DIRECT_NOW direct / $TRANSCODE_NOW transcode)" +echo "$ICON_EMBY Library: $MOVIE_COUNT movies $EPISODE_COUNT episodes $SONG_COUNT songs" +echo "$ICON_EMBY Period: $TOTAL_PLAYS play events in last ${EMBY_REPORT_DAYS} days" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +# Only notify on issues — high transcode rate may indicate config problem if [[ "$DRY_RUN" == false ]]; then - notify "Emby report on $(hostname) — $ACTIVE_COUNT active streams ($DIRECT_COUNT direct / $TRANSCODE_COUNT transcode) — Library: $MOVIE_COUNT movies $EPISODE_COUNT episodes" "Emby Report" "normal" -fi \ No newline at end of file + if [[ "$TOTAL_PLAYS" -gt 10 && "${TRANSCODE_PCT:-0}" -gt 80 ]]; then + notify "Emby report on $(hostname) — high transcode rate: ${TRANSCODE_PCT}% of $TOTAL_PLAYS plays — check direct play config" \ + "Emby Report" "warning" + fi +fi + +exit 0 \ No newline at end of file diff --git a/Monitors/smart_health.sh b/Monitors/smart_health.sh index 333e51c..feccb06 100644 --- a/Monitors/smart_health.sh +++ b/Monitors/smart_health.sh @@ -1,35 +1,67 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- SMART Health Monitor --------------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= SMART Health Monitor ======================================= +# ============================================================================================== # Checks SMART health attributes for all drives on the system. # Reads data live from each drive via smartctl — no persistent writes. +# Designed to run weekly as a scheduled report. # -# Monitored attributes: -# Reallocated_Sector_Ct — bad sectors remapped — any > 0 is concerning -# Current_Pending_Sector — sectors waiting for reallocation — any > 0 is concerning -# Offline_Uncorrectable — sectors that could not be corrected — any > 0 is critical -# Temperature_Celsius — drive temperature vs SMART_TEMP_WARN / SMART_TEMP_CRIT -# Power_On_Hours — informational — drive age estimation -# SMART overall status — pass/fail per drive +# ── MONITORED ATTRIBUTES ────────────────────────────────────────────────────────────────────── +# Overall SMART status — PASSED/FAILED — immediate fail = drive is dying +# Reallocated_Sector_Ct — bad sectors remapped — any > 0 is concerning +# Current_Pending_Sector — sectors waiting for reallocation — any > 0 is concerning +# Offline_Uncorrectable — sectors that could not be corrected — any > 0 is critical +# Temperature_Celsius — vs thresholds from dynamix.cfg (or master.conf fallback) +# Power_On_Hours — informational — drive age in days # -# Discovers drives automatically — no configuration needed for drive list. -# SMART_IGNORE_DRIVES allows skipping specific drives (e.g. USB flash drives). +# ── DRIVE DISCOVERY ─────────────────────────────────────────────────────────────────────────── +# Discovers drives automatically via /dev/sd* and /dev/nvme* — no config needed. +# NVMe drives use different attribute names — detected and handled automatically. +# HOST*_SMART_IGNORE_DRIVES skips specific drives (e.g. boot USB flash drive). # -# All configuration in Master.conf under SMART Health section. -# Supports --dry-run to show which drives would be checked without running smartctl. -# ----------------------------------------------------------------------------------------------- +# ── TEMPERATURE THRESHOLDS ──────────────────────────────────────────────────────────────────── +# Reads hot/max/hotssd/maxssd from /boot/config/plugins/dynamix/dynamix.cfg at runtime. +# Uses unRAID's own configured thresholds — no need to duplicate them here. +# Falls back to SMART_TEMP_WARN / SMART_TEMP_CRIT from master.conf if dynamix.cfg not found. +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID and aliases HOST*_SMART_IGNORE_DRIVES → SMART_IGNORE_DRIVES. +# Each server monitors its own drives with its own ignore list. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock — smartctl calls are slow, prevent duplicate runs +# detect_hosts() — correct ignore list per host via MY_ID aliases +# validate_unraid_cmd — smartctl and notify validated before use +# Silent healthy drives — only problems produce output +# Silent healthy run — no notify when all drives pass +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_SMART_IGNORE_DRIVES — drives skipped in SMART monitoring +# Aliased by detect_hosts() — script uses SMART_IGNORE_DRIVES +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# SMART_TEMP_WARN — fallback warn threshold in °C (if dynamix.cfg not found) +# SMART_TEMP_CRIT — fallback crit threshold in °C (if dynamix.cfg not found) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# smart_health.sh — normal run +# smart_health.sh --dry-run — show which drives would be checked +# smart_health.sh --log — verbose output +# smart_health.sh --status — show config and exit +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" + +# Monitor script — output is the point +SILENT_MODE=false parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" @@ -38,35 +70,56 @@ if [[ "$EUID" -ne 0 ]]; then exit 1 fi -success "Running as root" +# Validate smartctl — required for all drive checks +validate_unraid_cmd \ + "$(command -v smartctl 2>/dev/null || echo /usr/bin/smartctl)" \ + "--version" "smartmontools" \ + "smartctl" || { + error "smartctl not found — install smartmontools" + notify "SMART health check failed on $(hostname) — smartmontools not installed" \ + "SMART Health" "warning" + exit 1 + } -if ! command -v smartctl >/dev/null 2>&1; then - error "smartctl not found — install smartmontools" - notify "SMART health check failed on $(hostname) — smartmontools not installed" "SMART Health" "warning" - exit 1 -fi +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" -success "smartctl available" +acquire_lock -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +# detect_hosts() sets MY_ID and aliases HOST*_SMART_IGNORE_DRIVES +detect_hosts + +# Load temperature thresholds from dynamix.cfg — unRAID's own settings +get_unraid_temp_thresholds + +log "HDD warn: ${UNRAID_DISK_HOT}°C crit: ${UNRAID_DISK_MAX}°C" +log "SSD warn: ${UNRAID_SSD_HOT}°C crit: ${UNRAID_SSD_MAX}°C" +log "Ignore: ${SMART_IGNORE_DRIVES[*]:-none}" + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing drive list only, no SMART data read" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_SMART Temp warn: ${SMART_TEMP_WARN}°C" - echo "$ICON_SMART Temp crit: ${SMART_TEMP_CRIT}°C" - echo "$ICON_SMART Ignore drives: ${SMART_IGNORE_DRIVES[*]:-none}" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_SMART HDD warn: ${UNRAID_DISK_HOT}°C" + echo "$ICON_SMART HDD crit: ${UNRAID_DISK_MAX}°C" + echo "$ICON_SMART SSD warn: ${UNRAID_SSD_HOT}°C" + echo "$ICON_SMART SSD crit: ${UNRAID_SSD_MAX}°C" + echo "$ICON_SMART Ignore drives: ${SMART_IGNORE_DRIVES[*]:-none}" + echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "" - - # Show drives that would be checked echo "━━━ Discovered Drives ━━━" for drive in /dev/sd? /dev/nvme?; do [[ ! -e "$drive" ]] && continue drive_name=$(basename "$drive") ignored=false - for ignore in "${SMART_IGNORE_DRIVES[@]}"; do + for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do [[ "$drive_name" == "$ignore" ]] && ignored=true && break done if [[ "$ignored" == true ]]; then @@ -79,23 +132,55 @@ if [[ "$SHOW_STATUS" == true ]]; then exit 0 fi -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing drive list only, no SMART data read" +# ============================================================================================== +# ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== -# ----------------------------------------------------------------------------------------------- -# HELPER — extract SMART attribute value -# Usage: get_smart_attr "/dev/sda" "Reallocated_Sector_Ct" -# ----------------------------------------------------------------------------------------------- +# Extract a named SMART attribute value (column 10 — raw value) get_smart_attr() { local drive="$1" attr="$2" smartctl -A "$drive" 2>/dev/null | \ awk -v attr="$attr" '$2 == attr {print $10}' } -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SMART SMART Health Check ━━━ -# ----------------------------------------------------------------------------------------------- +# Get drive temperature — handles HDD (attribute) and NVMe (different output format) +get_drive_temp() { + local drive="$1" + local temp + + # Standard HDD SMART attribute + temp=$(get_smart_attr "$drive" "Temperature_Celsius") + [[ -n "$temp" ]] && echo "$temp" && return + + # NVMe — temperature in different section + temp=$(smartctl -A "$drive" 2>/dev/null | \ + awk '/Temperature:/{gsub(/[^0-9]/,"",$2); if($2>0) print $2; exit}') + [[ -n "$temp" ]] && echo "$temp" && return + + # Fallback — any temperature line + temp=$(smartctl -A "$drive" 2>/dev/null | \ + awk '/Temp/{gsub(/[^0-9]/,"",$NF); if($NF>0 && $NF<120) print $NF; exit}') + echo "${temp:-}" +} + +# Detect if a drive is SSD/NVMe (rotational=0) +is_ssd() { + local drive="$1" + local dev_name + dev_name=$(basename "$drive" | sed 's/nvme[0-9]/nvme0/') + local rotational="/sys/block/$(basename "$drive")/queue/rotational" + [[ -f "$rotational" ]] && [[ "$(cat "$rotational" 2>/dev/null)" == "0" ]] && return 0 + # NVMe is always SSD + [[ "$drive" == *nvme* ]] && return 0 + return 1 +} + +# ============================================================================================== +# ━━━ SMART Health Check ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_SMART SMART Health Check — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)" echo "" START=$(date +%s) @@ -110,12 +195,12 @@ for drive in /dev/sd? /dev/nvme?; do # Check ignore list ignored=false - for ignore in "${SMART_IGNORE_DRIVES[@]}"; do + for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do [[ "$drive_name" == "$ignore" ]] && ignored=true && break done if [[ "$ignored" == true ]]; then - info "$drive_name — ignored (in SMART_IGNORE_DRIVES)" + log "$drive_name — ignored (SMART_IGNORE_DRIVES)" DRIVES_SKIP+=("$drive_name") continue fi @@ -128,34 +213,40 @@ for drive in /dev/sd? /dev/nvme?; do continue fi - # Check if drive supports SMART + # Check SMART support if ! smartctl -i "$drive" 2>/dev/null | grep -q "SMART support is: Enabled"; then - warn "$drive_name — SMART not enabled or not supported" + warn "$drive_name — SMART not enabled or not supported — skipping" DRIVES_SKIP+=("$drive_name") echo "" continue fi - # Overall SMART status - SMART_STATUS=$(smartctl -H "$drive" 2>/dev/null | grep "overall-health" | awk '{print $NF}') - if [[ "$SMART_STATUS" == "PASSED" ]]; then - success "Overall status: PASSED" - else - error "Overall status: $SMART_STATUS" - fi - - # Key attributes DRIVE_WARN=false DRIVE_CRIT=false + # Overall SMART status + SMART_STATUS=$(smartctl -H "$drive" 2>/dev/null | \ + grep "overall-health" | awk '{print $NF}') + case "${SMART_STATUS:-}" in + PASSED) + log "$drive_name overall status: PASSED" ;; + FAILED*) + error "$drive_name overall status: FAILED — drive may be failing" + DRIVE_CRIT=true ;; + "") + warn "$drive_name overall status: unknown — could not read SMART data" ;; + *) + warn "$drive_name overall status: $SMART_STATUS" ;; + esac + # Reallocated sectors REALLOC=$(get_smart_attr "$drive" "Reallocated_Sector_Ct") if [[ -n "$REALLOC" ]]; then if [[ "$REALLOC" -gt 0 ]]; then - warn "$ICON_SMART Reallocated sectors: $REALLOC — drive showing wear" + warn "$ICON_SMART $drive_name — Reallocated sectors: $REALLOC (drive showing wear)" DRIVE_WARN=true else - success "$ICON_SMART Reallocated sectors: $REALLOC" + log "$drive_name reallocated sectors: 0 ✅" fi fi @@ -163,47 +254,53 @@ for drive in /dev/sd? /dev/nvme?; do PENDING=$(get_smart_attr "$drive" "Current_Pending_Sector") if [[ -n "$PENDING" ]]; then if [[ "$PENDING" -gt 0 ]]; then - warn "$ICON_SMART Pending sectors: $PENDING — sectors awaiting reallocation" + warn "$ICON_SMART $drive_name — Pending sectors: $PENDING (awaiting reallocation)" DRIVE_WARN=true else - success "$ICON_SMART Pending sectors: $PENDING" + log "$drive_name pending sectors: 0 ✅" fi fi - # Uncorrectable sectors + # Uncorrectable sectors — critical threshold UNCORR=$(get_smart_attr "$drive" "Offline_Uncorrectable") if [[ -n "$UNCORR" ]]; then if [[ "$UNCORR" -gt 0 ]]; then - error "$ICON_SMART Uncorrectable sectors: $UNCORR — CRITICAL" + error "$ICON_SMART $drive_name — Uncorrectable sectors: $UNCORR — CRITICAL" DRIVE_CRIT=true else - success "$ICON_SMART Uncorrectable sectors: $UNCORR" + log "$drive_name uncorrectable sectors: 0 ✅" fi fi - # Temperature - TEMP=$(get_smart_attr "$drive" "Temperature_Celsius") - # NVMe uses different attribute name - [[ -z "$TEMP" ]] && TEMP=$(smartctl -A "$drive" 2>/dev/null | \ - awk '/Temperature/{print $2}' | head -1) + # Temperature — use SSD/HDD thresholds from dynamix.cfg + TEMP=$(get_drive_temp "$drive") + if [[ -n "$TEMP" ]] && [[ "$TEMP" =~ ^[0-9]+$ ]]; then + if is_ssd "$drive"; then + WARN_THRESH="$UNRAID_SSD_HOT" + CRIT_THRESH="$UNRAID_SSD_MAX" + DRIVE_TYPE="SSD" + else + WARN_THRESH="$UNRAID_DISK_HOT" + CRIT_THRESH="$UNRAID_DISK_MAX" + DRIVE_TYPE="HDD" + fi - if [[ -n "$TEMP" ]]; then - if [[ "$TEMP" -ge "$SMART_TEMP_CRIT" ]]; then - error "$ICON_SMART Temperature: ${TEMP}°C — CRITICAL (threshold: ${SMART_TEMP_CRIT}°C)" + if [[ "$TEMP" -ge "$CRIT_THRESH" ]]; then + error "$ICON_SMART $drive_name — ${DRIVE_TYPE} temp: ${TEMP}°C — CRITICAL (threshold: ${CRIT_THRESH}°C)" DRIVE_CRIT=true - elif [[ "$TEMP" -ge "$SMART_TEMP_WARN" ]]; then - warn "$ICON_SMART Temperature: ${TEMP}°C — warning (threshold: ${SMART_TEMP_WARN}°C)" + elif [[ "$TEMP" -ge "$WARN_THRESH" ]]; then + warn "$ICON_SMART $drive_name — ${DRIVE_TYPE} temp: ${TEMP}°C — warning (threshold: ${WARN_THRESH}°C)" DRIVE_WARN=true else - success "$ICON_SMART Temperature: ${TEMP}°C" + log "$drive_name temp: ${TEMP}°C ${DRIVE_TYPE} ✅" fi fi - # Power on hours — informational + # Power on hours — informational only POH=$(get_smart_attr "$drive" "Power_On_Hours") if [[ -n "$POH" ]]; then POH_DAYS=$(( POH / 24 )) - info "$ICON_SMART Power on hours: $POH (${POH_DAYS} days)" + log "$drive_name power on hours: $POH (${POH_DAYS} days)" fi # Classify drive @@ -220,30 +317,33 @@ done END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "━━━━━ $ICON_SUMMARY SMART HEALTH SUMMARY ━━━━━" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" -echo " $ICON_SUCCESS Healthy: ${#DRIVES_OK[@]} $ICON_WARN Warning: ${#DRIVES_WARN[@]} $ICON_ERROR Critical: ${#DRIVES_CRIT[@]} skipped: ${#DRIVES_SKIP[@]}" +echo " $ICON_SUCCESS Healthy: ${#DRIVES_OK[@]}" +[[ ${#DRIVES_WARN[@]} -gt 0 ]] && warn "Warning: ${#DRIVES_WARN[@]} — ${DRIVES_WARN[*]}" +[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && echo "$ICON_ERROR Critical: ${#DRIVES_CRIT[@]} — ${DRIVES_CRIT[*]}" +[[ ${#DRIVES_SKIP[@]} -gt 0 ]] && log "Skipped: ${#DRIVES_SKIP[@]} — ${DRIVES_SKIP[*]}" echo "" -[[ ${#DRIVES_OK[@]} -gt 0 ]] && echo " $ICON_SUCCESS ${DRIVES_OK[*]}" -[[ ${#DRIVES_WARN[@]} -gt 0 ]] && echo " $ICON_WARN ${DRIVES_WARN[*]}" -[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && echo " $ICON_ERROR ${DRIVES_CRIT[*]}" - -echo "" if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN" + warn "DRY RUN — no SMART data read" elif [[ ${#DRIVES_CRIT[@]} -gt 0 ]]; then echo "$ICON_ERROR Status: CRITICAL — ${DRIVES_CRIT[*]}" - notify "SMART CRITICAL on $(hostname) — drives need immediate attention: ${DRIVES_CRIT[*]}" "SMART Health" "warning" + notify "SMART CRITICAL on $(hostname) — immediate attention needed: ${DRIVES_CRIT[*]}" \ + "SMART Health" "warning" elif [[ ${#DRIVES_WARN[@]} -gt 0 ]]; then - echo "$ICON_WARN Status: WARNING — ${DRIVES_WARN[*]}" - notify "SMART WARNING on $(hostname) — drives showing wear: ${DRIVES_WARN[*]}" "SMART Health" "warning" + warn "Status: WARNING — ${DRIVES_WARN[*]}" + notify "SMART WARNING on $(hostname) — drives showing wear: ${DRIVES_WARN[*]}" \ + "SMART Health" "warning" else - echo "$ICON_DONE Status: $ICON_SUCCESS ALL DRIVES HEALTHY" - notify "SMART health check passed on $(hostname) — ${#DRIVES_OK[@]} drives healthy" "SMART Health" "normal" + log "$ICON_DONE Status: all ${#DRIVES_OK[@]} drives healthy ✅" fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/Monitors/system_tuning_monitor.sh b/Monitors/system_tuning_monitor.sh index e69de29..d169462 100644 --- a/Monitors/system_tuning_monitor.sh +++ b/Monitors/system_tuning_monitor.sh @@ -0,0 +1,205 @@ +#!/bin/bash +# ============================================================================================== +# ============================= System Tuning Monitor ========================================== +# ============================================================================================== +# Tracks inotify and php-fpm usage over time. +# Snapshots written every 6 hours — read by sunday_morning_coffee_report.sh for weekly summary. +# Schedule: 0 */6 * * * (every 6 hours via User Scripts) +# +# ── WHAT IT TRACKS ──────────────────────────────────────────────────────────────────────────── +# inotify instances: +# Current in use vs kernel limit +# % utilization — warns above INOTIFY_WARN_PCT (default 80%) +# Top 5 consumers by instance count +# Symptom of exhaustion: containers miss file events, downloads not detected, +# Live TV stutter, library not updated +# +# php-fpm workers: +# Active workers vs PHP_MAX_CHILDREN limit +# % utilization — warns above PHP_FPM_WARN_PCT (default 80%) +# Symptom: unRAID WebGUI slowdowns or timeouts under load +# +# ── LOG FORMAT ──────────────────────────────────────────────────────────────────────────────── +# DATE|TIME|INOTIFY_USED|INOTIFY_LIMIT|INOTIFY_PCT|INOTIFY_WARN|PHPFPM_ACTIVE|PHPFPM_MAX|PHPFPM_PCT|PHPFPM_WARN +# Log trimmed to TUNING_LOG_RETENTION days on each write — bounded size. +# +# ── WHAT THE WEEKLY REPORT SHOWS ────────────────────────────────────────────────────────────── +# inotify: peak, average, warning count over the week +# php-fpm: peak workers, average workers, warning count over the week +# +# ── SILENT BY DEFAULT ───────────────────────────────────────────────────────────────────────── +# Background snapshot script — no output when healthy. +# Warns to stderr when thresholds exceeded — visible in User Scripts output log. +# Does NOT notify on every snapshot — only when threshold exceeded. +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# Each server writes to its own DATA_DIR — no collision between servers. +# MY_ID included in warning output for clarity in shared notification channels. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock — prevents overlapping 6-hour snapshots +# root check — /proc/*/fd requires root access +# atomic log write — tmp file + mv prevents partial writes on trim +# validate_unraid — notify script validated before use +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# INOTIFY_WARN_PCT — warn threshold % (default 80) +# PHP_FPM_WARN_PCT — warn threshold % (default 80) +# PHP_MAX_CHILDREN — max php-fpm workers (set by php_fpm_max_children.sh) +# TUNING_MONITOR_LOG — log file path +# TUNING_LOG_RETENTION — days before old entries purged (default 30) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# system_tuning_monitor.sh — normal snapshot run +# system_tuning_monitor.sh --dry-run — measure and show, no log write +# system_tuning_monitor.sh --log — verbose output +# system_tuning_monitor.sh --status — show config and exit +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../load_config.sh" + +parse_args "$@" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root — /proc/*/fd requires root access" + exit 1 +fi + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + +acquire_lock + +# detect_hosts() sets MY_ID — used in warning output +detect_hosts + +DATE=$(date '+%Y-%m-%d') +TIME=$(date '+%H:%M') +INOTIFY_WARN=0 +PHPFPM_WARN=0 + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_GEAR inotify warn: ${INOTIFY_WARN_PCT:-80}%" + echo "$ICON_GEAR php-fpm warn: ${PHP_FPM_WARN_PCT:-80}%" + echo "$ICON_GEAR php-fpm max: ${PHP_MAX_CHILDREN:-250}" + echo "$ICON_GEAR Log file: ${TUNING_MONITOR_LOG:-not set}" + echo "$ICON_GEAR Retention: ${TUNING_LOG_RETENTION:-30} days" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + + if [[ -f "$TUNING_MONITOR_LOG" ]]; then + ENTRY_COUNT=$(wc -l < "$TUNING_MONITOR_LOG") + OLDEST=$(awk -F'|' 'NR==1{print $1}' "$TUNING_MONITOR_LOG") + NEWEST=$(awk -F'|' 'END{print $1}' "$TUNING_MONITOR_LOG") + echo "$ICON_MONITOR Log entries: $ENTRY_COUNT ($OLDEST → $NEWEST)" + else + echo "$ICON_MONITOR Log entries: none yet" + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — measuring only, no log write" + +# ============================================================================================== +# ━━━ inotify ━━━ +# ============================================================================================== +INOTIFY_LIMIT=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 0) +INOTIFY_USED=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l) +INOTIFY_USED="${INOTIFY_USED//[^0-9]/}" +INOTIFY_USED="${INOTIFY_USED:-0}" + +if [[ "$INOTIFY_LIMIT" -gt 0 ]]; then + INOTIFY_PCT=$(( INOTIFY_USED * 100 / INOTIFY_LIMIT )) +else + INOTIFY_PCT=0 +fi + +[[ "$INOTIFY_PCT" -ge "${INOTIFY_WARN_PCT:-80}" ]] && INOTIFY_WARN=1 + +# Top 5 inotify consumers +INOTIFY_TOP=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | \ + awk -F/ '{print $3}' | sort | uniq -c | sort -rn | head -5 | \ + while read -r count pid; do + comm=$(cat "/proc/$pid/comm" 2>/dev/null || echo "?") + echo "${count}×${comm}" + done | tr '\n' ',' | sed 's/,$//') + +if [[ "$INOTIFY_WARN" -eq 1 ]]; then + warn "$MY_ID — inotify: ${INOTIFY_USED}/${INOTIFY_LIMIT} (${INOTIFY_PCT}%) — above ${INOTIFY_WARN_PCT}% threshold" + warn "Top consumers: ${INOTIFY_TOP:-unknown}" + warn "Symptoms: containers missing file events, library not updating, Live TV stutter" + notify "inotify at ${INOTIFY_PCT}% on $(hostname) — ${INOTIFY_USED}/${INOTIFY_LIMIT} in use — top: ${INOTIFY_TOP}" \ + "System Tuning" "warning" +else + log "inotify: ${INOTIFY_USED}/${INOTIFY_LIMIT} (${INOTIFY_PCT}%) ✅" + log "inotify top consumers: ${INOTIFY_TOP:-none}" +fi + +# ============================================================================================== +# ━━━ php-fpm ━━━ +# ============================================================================================== +PHPFPM_MAX="${PHP_MAX_CHILDREN:-250}" + +PHPFPM_ACTIVE=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0) +PHPFPM_ACTIVE="${PHPFPM_ACTIVE//[^0-9]/}" +PHPFPM_ACTIVE="${PHPFPM_ACTIVE:-0}" + +if [[ "$PHPFPM_MAX" -gt 0 ]]; then + PHPFPM_PCT=$(( PHPFPM_ACTIVE * 100 / PHPFPM_MAX )) +else + PHPFPM_PCT=0 +fi + +[[ "$PHPFPM_PCT" -ge "${PHP_FPM_WARN_PCT:-80}" ]] && PHPFPM_WARN=1 + +if [[ "$PHPFPM_WARN" -eq 1 ]]; then + warn "$MY_ID — php-fpm: ${PHPFPM_ACTIVE}/${PHPFPM_MAX} workers (${PHPFPM_PCT}%) — above ${PHP_FPM_WARN_PCT}% threshold" + warn "Symptom: unRAID WebGUI slowdowns or timeouts under load" + notify "php-fpm at ${PHPFPM_PCT}% on $(hostname) — ${PHPFPM_ACTIVE}/${PHPFPM_MAX} workers active" \ + "System Tuning" "warning" +else + log "php-fpm: ${PHPFPM_ACTIVE}/${PHPFPM_MAX} active workers (${PHPFPM_PCT}%) ✅" +fi + +# ============================================================================================== +# ━━━ Write Snapshot ━━━ +# ============================================================================================== +if [[ -z "${TUNING_MONITOR_LOG:-}" ]]; then + warn "TUNING_MONITOR_LOG not set — snapshot not written" + exit 0 +fi + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — snapshot not written" + warn "Would write: ${DATE}|${TIME}|${INOTIFY_USED}|${INOTIFY_LIMIT}|${INOTIFY_PCT}|${INOTIFY_WARN}|${PHPFPM_ACTIVE}|${PHPFPM_MAX}|${PHPFPM_PCT}|${PHPFPM_WARN}" + exit 0 +fi + +mkdir -p "$(dirname "$TUNING_MONITOR_LOG")" + +# Trim old entries — atomic write via temp file +if [[ -f "$TUNING_MONITOR_LOG" ]]; then + CUTOFF=$(date -d "${TUNING_LOG_RETENTION:-30} days ago" '+%Y-%m-%d') + awk -F'|' -v cutoff="$CUTOFF" '$1 >= cutoff' \ + "$TUNING_MONITOR_LOG" > "${TUNING_MONITOR_LOG}.tmp" && \ + mv "${TUNING_MONITOR_LOG}.tmp" "$TUNING_MONITOR_LOG" +fi + +# Append snapshot +echo "${DATE}|${TIME}|${INOTIFY_USED}|${INOTIFY_LIMIT}|${INOTIFY_PCT}|${INOTIFY_WARN}|${PHPFPM_ACTIVE}|${PHPFPM_MAX}|${PHPFPM_PCT}|${PHPFPM_WARN}" \ + >> "$TUNING_MONITOR_LOG" + +log "Snapshot written: inotify ${INOTIFY_PCT}% php-fpm ${PHPFPM_PCT}%" \ No newline at end of file diff --git a/Monitors/weekly_health_digest.sh b/Monitors/weekly_health_digest.sh index a70377c..89a6441 100644 --- a/Monitors/weekly_health_digest.sh +++ b/Monitors/weekly_health_digest.sh @@ -1,50 +1,117 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Health Digest ---------------------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= Health Digest ============================================== +# ============================================================================================== # Aggregates system health data from across the ecosystem into a single digest report. -# Reads existing state files — no new writes to flash drive. +# Reads existing state files — no new writes. +# +# ── THREE PROFILES ──────────────────────────────────────────────────────────────────────────── +# always — sends every run regardless of findings +# schedule daily for a daily digest +# +# smart — sends only if something worth reporting was found +# runs every run but stays silent when all healthy +# DIGEST_SMART_ON_* toggles control what triggers a send # -# Three profiles controlled by DIGEST_PROFILE in Master.conf: -# always — sends every run regardless of findings (schedule daily for daily digest) -# smart — sends only if something worth reporting was found (intelligent filtering) # weekly — sends once per week on DIGEST_DAY regardless of schedule frequency +# run daily, digest only fires on DIGEST_DAY (default Sunday) # -# The cron schedule stays the same regardless of profile — just change DIGEST_PROFILE -# in Master.conf to switch behavior. Run daily, profile controls when it actually notifies. +# The cron schedule stays the same regardless of profile — change DIGEST_PROFILE in +# master.conf to switch behaviour. No cron changes needed. # -# Data sources (reads only — no writes): -# /tmp/transcode_state.db — ramdisk symlink and usage -# /tmp/container_watchdog_state.db — active container strikes -# /tmp/system_watchdog_state.db — active system strikes -# /boot/config/failover_state.db — current failover state -# /boot/config/system_watchdog_failed.db — container skip list -# /boot/config/bandwidth_history.db — recent transfer totals -# SSL certs via openssl (live check) — days remaining per domain +# ── DATA SOURCES (reads only) ───────────────────────────────────────────────────────────────── +# FAILOVER_STATE_FILE — current failover state +# SYS_WATCHDOG_FAILED_FILE — container skip list (manual intervention needed) +# WATCHDOG_STATE_FILE — active container watchdog strikes +# SYS_WATCHDOG_STATE_FILE — active system watchdog strikes +# BANDWIDTH_LOG — yesterday's transfer totals +# TRANSCODE_DAILY_LOG — weekly transcode statistics +# CERT_MONITOR_DOMAINS — live SSL cert check via openssl +# RAMDISK_PATH / TRANSCODE_LINK — current transcode location and usage # -# All configuration in Master.conf under Health Digest section. -# Supports --dry-run to generate report without sending notification. -# ----------------------------------------------------------------------------------------------- +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID and aliases CERT_MONITOR_DOMAINS, RAMDISK_WARN_GB, +# RAMDISK_SIZE, RAMDISK_LOW_GB and all other host-specific vars used in this report. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock — report takes time, prevent duplicate runs +# detect_hosts() — correct vars per host +# validate_unraid_cmd — notify and openssl validated before use +# Per-section guards — missing state file skipped cleanly +# Silent smart profile — completely silent when nothing to report +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# DIGEST_PROFILE — always | smart | weekly +# DIGEST_DAY — day name for weekly profile (e.g. Sunday) +# DIGEST_SMART_ON_WATCHDOG — send on active watchdog strikes +# DIGEST_SMART_ON_FAILOVER — send on non-NORMAL failover state +# DIGEST_SMART_ON_CERT_WARN — send on cert warning +# DIGEST_SMART_ON_BANDWIDTH — send on high bandwidth day +# CERT_WARN_DAYS / CERT_CRIT_DAYS / CERT_TIMEOUT +# BANDWIDTH_WARN_GB +# TRANSCODE_DAILY_LOG +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# weekly_health_digest.sh — normal run +# weekly_health_digest.sh --dry-run — generate report, no notification +# weekly_health_digest.sh --log — verbose output +# weekly_health_digest.sh --status — show config and exit +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" + +# Report/monitor script — output is the point when sending +SILENT_MODE=false parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + +validate_unraid_cmd \ + "$(command -v openssl 2>/dev/null || echo /usr/bin/openssl)" \ + "version" "OpenSSL" \ + "openssl" || warn "openssl not found — SSL cert checks will be skipped" + +acquire_lock + +# detect_hosts() sets MY_ID and aliases all host-specific vars used in this report +detect_hosts -success "Health Digest — profile: $DIGEST_PROFILE" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — report generated but no notification sent" -# ----------------------------------------------------------------------------------------------- -# Profile check — should we send today? -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_DIGEST Profile: $DIGEST_PROFILE" + echo "$ICON_DIGEST Digest day: $DIGEST_DAY" + echo "$ICON_DIGEST Smart triggers: watchdog=$DIGEST_SMART_ON_WATCHDOG failover=$DIGEST_SMART_ON_FAILOVER cert=$DIGEST_SMART_ON_CERT_WARN bandwidth=$DIGEST_SMART_ON_BANDWIDTH" + echo "$ICON_CERT Cert domains: ${CERT_MONITOR_DOMAINS[*]:-none}" + echo "$ICON_BANDWIDTH Bandwidth warn: ${BANDWIDTH_WARN_GB}GB" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ── Profile gate — should we send today? ────────────────────────────────────────────────────── +# ============================================================================================== SHOULD_SEND=false case "$DIGEST_PROFILE" in @@ -56,229 +123,206 @@ case "$DIGEST_PROFILE" in TODAY_NAME=$(date '+%A') if [[ "$TODAY_NAME" == "$DIGEST_DAY" ]]; then SHOULD_SEND=true - log "Profile: weekly — today is $DIGEST_DAY, will send" + log "Profile: weekly — today is $DIGEST_DAY — will send" else - info "Profile: weekly — today is $TODAY_NAME, digest day is $DIGEST_DAY — skipping" + log "Profile: weekly — today is $TODAY_NAME, digest day is $DIGEST_DAY — skipping" exit 0 fi ;; smart) - log "Profile: smart — will evaluate findings before deciding" - SHOULD_SEND=false # determined after gathering data + log "Profile: smart — evaluating findings before deciding" + SHOULD_SEND=false ;; *) - warn "Unknown DIGEST_PROFILE: $DIGEST_PROFILE — defaulting to weekly behavior" + warn "Unknown DIGEST_PROFILE: $DIGEST_PROFILE — defaulting to weekly" TODAY_NAME=$(date '+%A') [[ "$TODAY_NAME" == "$DIGEST_DAY" ]] && SHOULD_SEND=true ;; esac -# ----------------------------------------------------------------------------------------------- -# DATA GATHERING -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_DIGEST Gathering System Data ━━━" +# ============================================================================================== +# ── Data Gathering ──────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +FINDINGS=() # notable but not critical +ISSUES=() # need attention +DIGEST_LINES=() # full report lines -FINDINGS=() # things worth noting -ISSUES=() # things that need attention -DIGEST_LINES=() # full report lines - -# ── Failover State ────────────────────────────────────────────────────────────────────────── +# ── Failover State ──────────────────────────────────────────────────────────────────────────── if [[ -f "$FAILOVER_STATE_FILE" ]]; then FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) - FAILOVER_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) - DIGEST_LINES+=("$ICON_FAILOVER Failover: $FAILOVER_STATE (last change: ${FAILOVER_CHANGE:-unknown})") - - if [[ "$FAILOVER_STATE" != "NORMAL" && -n "$FAILOVER_STATE" ]]; then - ISSUES+=("Failover state: $FAILOVER_STATE") - [[ "$DIGEST_SMART_ON_FAILOVER" == true ]] && SHOULD_SEND=true + if [[ -n "$FAILOVER_STATE" ]]; then + DIGEST_LINES+=("$ICON_FAILOVER Failover: $FAILOVER_STATE") + if [[ "$FAILOVER_STATE" != "NORMAL" ]]; then + ISSUES+=("Failover state: $FAILOVER_STATE") + [[ "$DIGEST_SMART_ON_FAILOVER" == true ]] && SHOULD_SEND=true + fi fi else DIGEST_LINES+=("$ICON_FAILOVER Failover: state file not found") fi -# ── Transcode Ramdisk ─────────────────────────────────────────────────────────────────────── -if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then - RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ') - RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}") - RAMDISK_AVAIL_KB=$(df "$RAMDISK_PATH" --output=avail | tail -1 | tr -d ' ') - RAMDISK_AVAIL_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_AVAIL_KB / 1048576}") - SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown") - DIGEST_LINES+=("$ICON_RAM Transcodes: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB free → $SYMLINK_TARGET") - - # Read weekly transcode stats from daily log if available - TRANSCODE_DAILY_LOG="/boot/config/transcode_daily.db" - if [[ -f "$TRANSCODE_DAILY_LOG" ]]; then - WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d') - - # Peak ramdisk usage this week - WEEK_PEAK=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \ - '$1 >= cutoff {if ($2 > max) max=$2} END {printf "%.2f", max+0}' \ - "$TRANSCODE_DAILY_LOG") - - # Total flips this week - WEEK_FLIPS=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \ - '$1 >= cutoff {sum+=$3} END {print sum+0}' \ - "$TRANSCODE_DAILY_LOG") - - # Total files cleaned this week - WEEK_FILES=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \ - '$1 >= cutoff {sum+=$6} END {print sum+0}' \ - "$TRANSCODE_DAILY_LOG") - - # Ram vs SSD session ratio - WEEK_RAM=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \ - '$1 >= cutoff {sum+=$4} END {print sum+0}' \ - "$TRANSCODE_DAILY_LOG") - WEEK_SSD=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \ - '$1 >= cutoff {sum+=$5} END {print sum+0}' \ - "$TRANSCODE_DAILY_LOG") - - DIGEST_LINES+=("$ICON_RAM Transcode week: peak ${WEEK_PEAK}GB | flips: ${WEEK_FLIPS} | cleaned: ${WEEK_FILES} files") - DIGEST_LINES+=("$ICON_RAM Session storage: ${WEEK_RAM} ramdisk / ${WEEK_SSD} SSD") - - # Warn if peak is getting close to threshold - PEAK_INT=$(printf "%.0f" "$WEEK_PEAK") - WARN_INT=$(printf "%.0f" "$RAMDISK_WARN_GB") - if [[ "$PEAK_INT" -ge "$WARN_INT" ]]; then - DIGEST_LINES+=("$ICON_WARN Transcode peak ${WEEK_PEAK}GB near threshold ${RAMDISK_WARN_GB}GB — consider increasing RAMDISK_SIZE") - fi - fi +# ── Container Skip List ─────────────────────────────────────────────────────────────────────── +if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then + SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE") + SKIP_LIST=$(cat "$SYS_WATCHDOG_FAILED_FILE" | tr '\n' ' ') + DIGEST_LINES+=("$ICON_NOT_RUNNING Skip list: $SKIP_COUNT containers — $SKIP_LIST") + ISSUES+=("Containers on skip list (manual intervention needed): $SKIP_LIST") + SHOULD_SEND=true else - DIGEST_LINES+=("$ICON_RAM Transcodes: ramdisk not mounted") - ISSUES+=("Ramdisk not mounted") + DIGEST_LINES+=("$ICON_RUNNING Skip list: empty ✅") fi -# ── Container Watchdog Strikes ────────────────────────────────────────────────────────────── +# ── Container Watchdog Strikes ──────────────────────────────────────────────────────────────── if [[ -f "$WATCHDOG_STATE_FILE" ]]; then - ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | wc -l) + ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -c "." || echo 0) if [[ "$ACTIVE_STRIKES" -gt 0 ]]; then STRIKE_LIST=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ') DIGEST_LINES+=("$ICON_WATCHDOG Container strikes: $ACTIVE_STRIKES active — $STRIKE_LIST") FINDINGS+=("Container watchdog: $ACTIVE_STRIKES active strikes") [[ "$DIGEST_SMART_ON_WATCHDOG" == true ]] && SHOULD_SEND=true else - DIGEST_LINES+=("$ICON_WATCHDOG Container watchdog: no active strikes") + DIGEST_LINES+=("$ICON_WATCHDOG Container watchdog: no active strikes ✅") fi fi -# ── System Watchdog Strikes ───────────────────────────────────────────────────────────────── +# ── System Watchdog Strikes ─────────────────────────────────────────────────────────────────── if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then - SYS_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | wc -l) + SYS_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -c "." || echo 0) if [[ "$SYS_STRIKES" -gt 0 ]]; then SYS_STRIKE_LIST=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ') DIGEST_LINES+=("$ICON_REBOOT_SMART System strikes: $SYS_STRIKES active — $SYS_STRIKE_LIST") FINDINGS+=("System watchdog: $SYS_STRIKES active strikes") [[ "$DIGEST_SMART_ON_WATCHDOG" == true ]] && SHOULD_SEND=true else - DIGEST_LINES+=("$ICON_REBOOT_SMART System watchdog: no active strikes") + DIGEST_LINES+=("$ICON_REBOOT_SMART System watchdog: no active strikes ✅") fi fi -# ── Container Skip List ───────────────────────────────────────────────────────────────────── -if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then - SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE") - SKIP_LIST=$(cat "$SYS_WATCHDOG_FAILED_FILE" | tr '\n' ' ') - DIGEST_LINES+=("$ICON_NOT_RUNNING Skip list: $SKIP_COUNT containers — $SKIP_LIST") - ISSUES+=("Containers on skip list: $SKIP_LIST") - SHOULD_SEND=true +# ── Transcode Ramdisk ───────────────────────────────────────────────────────────────────────── +if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then + RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ') + RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}") + SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown") + DIGEST_LINES+=("$ICON_RAM Transcodes: ${RAMDISK_USED_GB}GB used → $SYMLINK_TARGET") + + # Weekly transcode stats from TRANSCODE_DAILY_LOG + if [[ -f "${TRANSCODE_DAILY_LOG:-}" ]]; then + WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d') + WEEK_PEAK=$(awk -F'|' -v c="$WEEK_CUTOFF" \ + '$1>=c{if($2>max)max=$2} END{printf "%.2f",max+0}' "$TRANSCODE_DAILY_LOG") + WEEK_FLIPS=$(awk -F'|' -v c="$WEEK_CUTOFF" \ + '$1>=c{sum+=$3} END{print sum+0}' "$TRANSCODE_DAILY_LOG") + WEEK_RAM=$(awk -F'|' -v c="$WEEK_CUTOFF" \ + '$1>=c{sum+=$4} END{print sum+0}' "$TRANSCODE_DAILY_LOG") + WEEK_SSD=$(awk -F'|' -v c="$WEEK_CUTOFF" \ + '$1>=c{sum+=$5} END{print sum+0}' "$TRANSCODE_DAILY_LOG") + WEEK_FILES=$(awk -F'|' -v c="$WEEK_CUTOFF" \ + '$1>=c{sum+=$6} END{print sum+0}' "$TRANSCODE_DAILY_LOG") + + DIGEST_LINES+=("$ICON_RAM Transcode week: peak ${WEEK_PEAK}GB | flips: $WEEK_FLIPS | sessions: ${WEEK_RAM} ramdisk / ${WEEK_SSD} SSD | cleaned: ${WEEK_FILES} files") + + PEAK_INT=$(printf "%.0f" "$WEEK_PEAK") + WARN_INT=$(printf "%.0f" "${RAMDISK_WARN_GB:-6.8}") + if [[ "$PEAK_INT" -ge "$WARN_INT" ]]; then + DIGEST_LINES+=("$ICON_WARN Peak ${WEEK_PEAK}GB near threshold ${RAMDISK_WARN_GB}GB — consider increasing HOST*_RAMDISK_SIZE") + FINDINGS+=("Transcode ramdisk near threshold: ${WEEK_PEAK}GB / ${RAMDISK_WARN_GB}GB") + fi + fi else - DIGEST_LINES+=("$ICON_RUNNING Skip list: empty — all containers healthy") + DIGEST_LINES+=("$ICON_RAM Transcodes: ramdisk not mounted") + ISSUES+=("Ramdisk not mounted at $RAMDISK_PATH") + SHOULD_SEND=true fi -# ── Bandwidth (yesterday's total) ─────────────────────────────────────────────────────────── -if [[ -f "$BANDWIDTH_LOG" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then +# ── Bandwidth ───────────────────────────────────────────────────────────────────────────────── +# Updated for new log format: date|time|profile|duration|status|bytes|warn_flag +if [[ -f "${BANDWIDTH_LOG:-}" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then YESTERDAY=$(date -d "yesterday" '+%Y-%m-%d') - YESTERDAY_BYTES=$(awk -F'|' -v d="$YESTERDAY" '$1==d{sum+=$3} END{print sum+0}' "$BANDWIDTH_LOG") - YESTERDAY_GB=$(awk "BEGIN {printf \"%.2f\", $YESTERDAY_BYTES / 1073741824}") - OVER_WARN=$(awk "BEGIN {print ($YESTERDAY_BYTES > $BANDWIDTH_WARN_GB * 1073741824) ? 1 : 0}") + YESTERDAY_BYTES=$(awk -F'|' -v d="$YESTERDAY" '$1==d{sum+=$6} END{print sum+0}' \ + "$BANDWIDTH_LOG") + YESTERDAY_GB=$(awk "BEGIN {printf \"%.2f\", ${YESTERDAY_BYTES:-0} / 1073741824}") + YESTERDAY_LARGE=$(awk -F'|' -v d="$YESTERDAY" '$1==d && $7=="LARGE"' \ + "$BANDWIDTH_LOG" | wc -l) - if [[ "$OVER_WARN" == "1" ]]; then - DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday's transfers: ${YESTERDAY_GB}GB ← exceeded ${BANDWIDTH_WARN_GB}GB threshold") + if [[ "${YESTERDAY_LARGE:-0}" -gt 0 ]]; then + DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday: ${YESTERDAY_GB}GB — $YESTERDAY_LARGE large transfer(s) exceeded ${BANDWIDTH_WARN_GB}GB") FINDINGS+=("High bandwidth day: ${YESTERDAY_GB}GB transferred") [[ "$DIGEST_SMART_ON_BANDWIDTH" == true ]] && SHOULD_SEND=true else - DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday's transfers: ${YESTERDAY_GB}GB") + DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday: ${YESTERDAY_GB}GB") fi else DIGEST_LINES+=("$ICON_BANDWIDTH Bandwidth: no data yet") fi -# ── SSL Certificates ──────────────────────────────────────────────────────────────────────── +# ── SSL Certificates ────────────────────────────────────────────────────────────────────────── if [[ ${#CERT_MONITOR_DOMAINS[@]} -gt 0 ]] && command -v openssl >/dev/null 2>&1; then CERT_ISSUES=() for domain in "${CERT_MONITOR_DOMAINS[@]}"; do [[ -z "$domain" ]] && continue - expiry_str=$(echo | timeout "$CERT_TIMEOUT" openssl s_client \ + expiry_str=$(echo | timeout "${CERT_TIMEOUT:-10}" openssl s_client \ -connect "${domain}:443" -servername "$domain" \ 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) - if [[ -n "$expiry_str" ]]; then expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null) days_remaining=$(( (expiry_epoch - $(date +%s)) / 86400 )) - - if [[ "$days_remaining" -le "$CERT_CRIT_DAYS" ]]; then + if [[ "$days_remaining" -le "${CERT_CRIT_DAYS:-7}" ]]; then CERT_ISSUES+=("$domain: ${days_remaining}d CRITICAL") SHOULD_SEND=true - elif [[ "$days_remaining" -le "$CERT_WARN_DAYS" ]]; then - CERT_ISSUES+=("$domain: ${days_remaining}d WARNING") + elif [[ "$days_remaining" -le "${CERT_WARN_DAYS:-30}" ]]; then + CERT_ISSUES+=("$domain: ${days_remaining}d warning") [[ "$DIGEST_SMART_ON_CERT_WARN" == true ]] && SHOULD_SEND=true fi fi done - if [[ ${#CERT_ISSUES[@]} -gt 0 ]]; then DIGEST_LINES+=("$ICON_CERT Certificates: ${CERT_ISSUES[*]}") FINDINGS+=("Cert issues: ${CERT_ISSUES[*]}") else - DIGEST_LINES+=("$ICON_CERT Certificates: all healthy") + DIGEST_LINES+=("$ICON_CERT Certificates: all healthy ✅") fi fi -# ── Smart profile final decision ──────────────────────────────────────────────────────────── +# ============================================================================================== +# ── Smart profile — exit silently if nothing to report ──────────────────────────────────────── +# ============================================================================================== if [[ "$DIGEST_PROFILE" == "smart" && "$SHOULD_SEND" == false ]]; then - info "Profile: smart — no findings worth reporting — skipping notification" - echo "" - echo "━━━━━ $ICON_SUMMARY DIGEST SUMMARY ━━━━━" - echo "$ICON_SUCCESS Everything looks healthy — no digest sent (smart profile)" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + log "Profile: smart — no findings worth reporting — silent exit" exit 0 fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_DIGEST Build and Send Digest ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Build and Send Digest ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_DIGEST Health Digest — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)" echo "" -echo "━━━ $ICON_DIGEST Health Digest ━━━" -DIGEST_DATE=$(date '+%Y-%m-%d %H:%M:%S') -DIGEST_HOST=$(hostname) - -# Build notification message -NOTIFY_MSG="Health Digest — $DIGEST_HOST — $DIGEST_DATE" -if [[ ${#ISSUES[@]} -gt 0 ]]; then - NOTIFY_MSG+=" | Issues: ${ISSUES[*]}" -fi -if [[ ${#FINDINGS[@]} -gt 0 ]]; then - NOTIFY_MSG+=" | Findings: ${FINDINGS[*]}" -fi - -# Print full digest to console for line in "${DIGEST_LINES[@]}"; do echo " $line" done echo "" echo "━━━━━ $ICON_SUMMARY DIGEST SUMMARY ━━━━━" -echo "$ICON_DIGEST Profile: $DIGEST_PROFILE" -echo "$ICON_TIME Generated: $DIGEST_DATE" -echo "$ICON_ERROR Issues: ${#ISSUES[@]}" -echo "$ICON_WARN Findings: ${#FINDINGS[@]}" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_DIGEST Profile: $DIGEST_PROFILE" +echo "$ICON_ERROR Issues: ${#ISSUES[@]}" +echo "$ICON_WARN Findings: ${#FINDINGS[@]}" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +# Build notification message +NOTIFY_MSG="Health Digest — $MY_ID ($LOCAL_SERVER_NAME)" +[[ ${#ISSUES[@]} -gt 0 ]] && NOTIFY_MSG+=" | Issues: ${ISSUES[*]}" +[[ ${#FINDINGS[@]} -gt 0 ]] && NOTIFY_MSG+=" | Findings: ${FINDINGS[*]}" +[[ ${#ISSUES[@]} -eq 0 && ${#FINDINGS[@]} -eq 0 ]] && NOTIFY_MSG+=" | All systems healthy" + +NOTIFY_SEV="normal" +[[ ${#ISSUES[@]} -gt 0 ]] && NOTIFY_SEV="warning" + if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — digest generated but not sent" elif [[ "$SHOULD_SEND" == true ]]; then - notify "$NOTIFY_MSG" "Health Digest" "$([[ ${#ISSUES[@]} -gt 0 ]] && echo "warning" || echo "normal")" - success "Digest sent" + notify "$NOTIFY_MSG" "Health Digest" "$NOTIFY_SEV" + log "Digest sent" fi \ No newline at end of file diff --git a/Monitors/zfs_memory_snapshot.sh b/Monitors/zfs_memory_snapshot.sh index 8b51729..8532a81 100644 --- a/Monitors/zfs_memory_snapshot.sh +++ b/Monitors/zfs_memory_snapshot.sh @@ -1,93 +1,133 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- ZFS Memory Snapshot ---------------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= ZFS Memory Snapshot ======================================== +# ============================================================================================== # Weekly ZFS pool health and memory diagnostic report. # Combines ZFS pool status, ARC statistics, memory summary, Docker memory usage # and kernel pressure into a single report. Informational only — no action taken. # system_watchdog.sh handles threshold-based intervention. # +# ── WHAT IT REPORTS ─────────────────────────────────────────────────────────────────────────── +# ZFS pool health — status, state, errors per pool (excluding ignored pools) +# ARC statistics — current size, max, utilization %, metadata pressure +# Memory status — total/free/available RAM vs thresholds +# Docker memory — top ZFS_REPORT_DOCKER_TOP containers by memory usage +# Kernel pressure — vmstat snapshot (3 samples) +# +# ── OUTPUT ──────────────────────────────────────────────────────────────────────────────────── # Output goes to both console and ZFS_REPORT_LOG for later review. +# In dry-run mode — console only, nothing written to log. # Notifies if any warning thresholds are exceeded. +# Silent when all healthy — only problems produce output. # -# Pools listed in ZFS_REPORT_IGNORE_POOLS are excluded from health reporting. -# Useful for pools expected to run at high usage (docker, cache etc.) +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS → ZFS_REPORT_IGNORE_POOLS. +# Each server ignores its own single-disk ZFS array pools — not the peer's. # -# All configuration in Master.conf under ZFS Memory Snapshot section. -# Supports --dry-run (preview only, no log write) and --status. -# ----------------------------------------------------------------------------------------------- +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock — zpool + docker stats are slow, prevent duplicates +# detect_hosts() — correct pool ignore list per host +# validate_unraid_cmd — notify validated before use +# DOCKER_TIMEOUT — docker stats protected against hung daemon +# ZFS not available — skips pool and ARC sections gracefully +# Docker not available — skips container section gracefully +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_ZFS_REPORT_IGNORE_POOLS — pools excluded from health reporting +# Aliased by detect_hosts() — script uses ZFS_REPORT_IGNORE_POOLS +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# ZFS_REPORT_LOG — log file path for weekly reports +# ZFS_REPORT_ARC_WARN_PCT — warn if ARC using more than this % of max +# ZFS_REPORT_FREE_WARN_GB — warn if less than this GB free RAM +# ZFS_REPORT_AVAIL_WARN_GB — warn if less than this GB available RAM +# ZFS_REPORT_DOCKER_TOP — how many top Docker containers to show +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# zfs_memory_snapshot.sh — normal report (writes to log) +# zfs_memory_snapshot.sh --dry-run — console only, no log write +# zfs_memory_snapshot.sh --log — verbose output +# zfs_memory_snapshot.sh --status — show config and exit +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" + +# Monitor/report script — output is the point +SILENT_MODE=false parse_args "$@" +DOCKER_TIMEOUT=15 + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + +acquire_lock + +# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS +detect_hosts + +# Build ignore pool lookup map — O(1) check per pool +declare -A IGNORE_POOL_MAP +for pool in "${ZFS_REPORT_IGNORE_POOLS[@]:-}"; do + [[ -n "$pool" ]] && IGNORE_POOL_MAP["$pool"]=1 +done + +log "Identity: $MY_ID ($LOCAL_SERVER_NAME)" +log "Ignoring pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}" + # Tee output to log file unless dry run if [[ "$DRY_RUN" == false ]]; then mkdir -p "$(dirname "$ZFS_REPORT_LOG")" exec > >(tee -a "$ZFS_REPORT_LOG") 2>&1 fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — output will not be written to log" -if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root" - exit 1 -fi - -success "Running as root" - -# Build ignore list for quick lookup -declare -A IGNORE_POOL_MAP -for pool in "${ZFS_REPORT_IGNORE_POOLS[@]}"; do - [[ -n "$pool" ]] && IGNORE_POOL_MAP["$pool"]=1 -done - -if [[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]]; then - info "Ignoring pools: ${ZFS_REPORT_IGNORE_POOLS[*]}" -fi - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_ZFS Log file: $ZFS_REPORT_LOG" - echo "$ICON_ZFS ARC warn: ${ZFS_REPORT_ARC_WARN_PCT}%" - echo "$ICON_MEM Free RAM warn: ${ZFS_REPORT_FREE_WARN_GB}GB" - echo "$ICON_MEM Avail RAM warn: ${ZFS_REPORT_AVAIL_WARN_GB}GB" - echo "$ICON_CONTAINERS Docker top: $ZFS_REPORT_DOCKER_TOP" - echo "$ICON_ZFS Ignore pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_ZFS Log file: $ZFS_REPORT_LOG" + echo "$ICON_ZFS ARC warn: ${ZFS_REPORT_ARC_WARN_PCT}%" + echo "$ICON_MEM Free RAM warn: ${ZFS_REPORT_FREE_WARN_GB}GB" + echo "$ICON_MEM Avail warn: ${ZFS_REPORT_AVAIL_WARN_GB}GB" + echo "$ICON_CONTAINERS Docker top: $ZFS_REPORT_DOCKER_TOP" + echo "$ICON_ZFS Ignore pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}" + echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — output will not be written to log" - -# ----------------------------------------------------------------------------------------------- -# Tracking -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Report ━━━ +# ============================================================================================== WARNINGS=() START=$(date +%s) -DATE=$(date +"%Y-%m-%d %H:%M:%S") +DATE=$(date '+%Y-%m-%d %H:%M:%S') echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo " $ICON_ZFS ZFS WEEKLY HEALTH REPORT — $DATE" -echo " $ICON_HOST Host: $(hostname)" +echo " $ICON_HOST $MY_ID — $LOCAL_SERVER_NAME" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_ZFS ZFS Pool Health ━━━ -# ----------------------------------------------------------------------------------------------- +# ── ZFS Pool Health ─────────────────────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_ZFS ZFS Pool Health ━━━" @@ -95,13 +135,11 @@ if ! command -v zpool >/dev/null 2>&1; then warn "ZFS not available on this system — skipping pool checks" else # Pool status — filtered to key lines, ignoring specified pools - info "Pool status:" + CURRENT_POOL="" while IFS= read -r line; do - # Extract pool name from "pool: poolname" lines if [[ "$line" =~ ^[[:space:]]*pool:[[:space:]]*(.+) ]]; then CURRENT_POOL="${BASH_REMATCH[1]// /}" fi - # Skip lines belonging to ignored pools [[ -n "${IGNORE_POOL_MAP[$CURRENT_POOL]:-}" ]] && continue echo " $line" done < <(zpool status 2>/dev/null | grep -E "pool:|state:|status:|errors:|scan:") @@ -109,41 +147,36 @@ else echo "" # Pool list — filter out ignored pools - info "Pool overview:" zpool list 2>/dev/null | while IFS= read -r line; do - # Always show header line if [[ "$line" == NAME* ]]; then echo " $line" continue fi - # Extract pool name (first field) pool_name=$(echo "$line" | awk '{print $1}') [[ -n "${IGNORE_POOL_MAP[$pool_name]:-}" ]] && continue echo " $line" done - # Check for unhealthy pools — excluding ignored ones - UNHEALTHY=$(zpool list -H -o name,health 2>/dev/null | while IFS=$'\t' read -r name health; do - [[ -n "${IGNORE_POOL_MAP[$name]:-}" ]] && continue - [[ "$health" != "ONLINE" ]] && echo "$name: $health" - done) + # Check for unhealthy non-ignored pools + UNHEALTHY=$(zpool list -H -o name,health 2>/dev/null | \ + while IFS=$'\t' read -r name health; do + [[ -n "${IGNORE_POOL_MAP[$name]:-}" ]] && continue + [[ "$health" != "ONLINE" ]] && echo "$name: $health" + done) if [[ -n "$UNHEALTHY" ]]; then error "One or more ZFS pools are NOT ONLINE: $UNHEALTHY" WARNINGS+=("ZFS pool unhealthy: $UNHEALTHY") else - success "All monitored ZFS pools are ONLINE" + log "All monitored ZFS pools are ONLINE ✅" fi - # Show ignored pools if [[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]]; then - info "Ignored pools (not reported): ${ZFS_REPORT_IGNORE_POOLS[*]}" + log "Ignored pools: ${ZFS_REPORT_IGNORE_POOLS[*]}" fi fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_ZFS ARC Statistics ━━━ -# ----------------------------------------------------------------------------------------------- +# ── ARC Statistics ──────────────────────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_ZFS ARC Statistics ━━━" @@ -170,14 +203,16 @@ else warn "ARC utilization ${ARC_PCT}% — above ${ZFS_REPORT_ARC_WARN_PCT}% threshold" WARNINGS+=("ARC high: ${ARC_PCT}%") else - success "ARC utilization ${ARC_PCT}% — within threshold (${ZFS_REPORT_ARC_WARN_PCT}%)" + log "ARC utilization ${ARC_PCT}% — within threshold ✅" fi echo "" - info "Metadata pressure:" - META_MRU_GHOST=$(awk '/^mru_ghost_metadata / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0) - META_MFU_GHOST=$(awk '/^mfu_ghost_metadata / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0) - META_MISSES=$(awk '/^demand_metadata_misses / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0) + META_MRU_GHOST=$(awk '/^mru_ghost_metadata / {print $3}' \ + /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0) + META_MFU_GHOST=$(awk '/^mfu_ghost_metadata / {print $3}' \ + /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0) + META_MISSES=$(awk '/^demand_metadata_misses / {print $3}' \ + /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0) MRU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MRU_GHOST / 1073741824}") MFU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MFU_GHOST / 1073741824}") @@ -187,17 +222,15 @@ else echo " $ICON_ZFS Metadata Misses: ${META_MISSES}" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_MEM Memory Status ━━━ -# ----------------------------------------------------------------------------------------------- +# ── Memory Status ───────────────────────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_MEM Memory Status ━━━" -FREE_HUMAN=$(free -h | awk '/Mem:/ {print $4}') +FREE_HUMAN=$(free -h | awk '/Mem:/ {print $4}') AVAIL_HUMAN=$(free -h | awk '/Mem:/ {print $7}') TOTAL_HUMAN=$(free -h | awk '/Mem:/ {print $2}') -FREE_GB=$(free -g | awk '/Mem:/ {print $4}') -AVAIL_GB=$(free -g | awk '/Mem:/ {print $7}') +FREE_GB=$(free -g | awk '/Mem:/ {print $4}') +AVAIL_GB=$(free -g | awk '/Mem:/ {print $7}') echo " $ICON_MEM Total RAM: $TOTAL_HUMAN" echo " $ICON_MEM Free RAM: $FREE_HUMAN" @@ -207,42 +240,38 @@ if [[ "$FREE_GB" -lt "$ZFS_REPORT_FREE_WARN_GB" ]]; then warn "Free RAM ${FREE_HUMAN} — below ${ZFS_REPORT_FREE_WARN_GB}GB threshold" WARNINGS+=("Low free RAM: ${FREE_HUMAN}") else - success "Free RAM ${FREE_HUMAN} — within threshold (${ZFS_REPORT_FREE_WARN_GB}GB)" + log "Free RAM ${FREE_HUMAN} — within threshold ✅" fi if [[ "$AVAIL_GB" -lt "$ZFS_REPORT_AVAIL_WARN_GB" ]]; then warn "Available RAM ${AVAIL_HUMAN} — below ${ZFS_REPORT_AVAIL_WARN_GB}GB threshold" WARNINGS+=("Low available RAM: ${AVAIL_HUMAN}") else - success "Available RAM ${AVAIL_HUMAN} — within threshold (${ZFS_REPORT_AVAIL_WARN_GB}GB)" + log "Available RAM ${AVAIL_HUMAN} — within threshold ✅" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_CONTAINERS Docker Memory ━━━ -# ----------------------------------------------------------------------------------------------- +# ── Docker Memory ───────────────────────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_CONTAINERS Top $ZFS_REPORT_DOCKER_TOP Docker Memory Users ━━━" if ! command -v docker >/dev/null 2>&1; then warn "Docker not available — skipping container memory section" else - docker stats --no-stream \ + timeout "$DOCKER_TIMEOUT" docker stats --no-stream \ --format "table {{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}" \ - 2>/dev/null | head -n $(( ZFS_REPORT_DOCKER_TOP + 1 )) | while IFS= read -r line; do - echo " $line" - done + 2>/dev/null | head -n $(( ZFS_REPORT_DOCKER_TOP + 1 )) | \ + while IFS= read -r line; do + echo " $line" + done fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Kernel Pressure ━━━ -# ----------------------------------------------------------------------------------------------- +# ── Kernel Pressure ─────────────────────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_GEAR Kernel Pressure ━━━" if ! command -v vmstat >/dev/null 2>&1; then warn "vmstat not available — skipping kernel pressure section" else - info "vmstat snapshot (3 samples):" vmstat 1 3 2>/dev/null | while IFS= read -r line; do echo " $line" done @@ -250,23 +279,27 @@ fi END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ── Summary ─────────────────────────────────────────────────────────────────────────────────── echo "" echo "━━━━━ $ICON_SUMMARY ZFS REPORT SUMMARY ━━━━━" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -echo "$ICON_ZFS Log: $ZFS_REPORT_LOG" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" +echo "$ICON_ZFS Log: $ZFS_REPORT_LOG" [[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]] && \ - echo "$ICON_ZFS Ignored: ${ZFS_REPORT_IGNORE_POOLS[*]}" + log "Ignored: ${ZFS_REPORT_IGNORE_POOLS[*]}" echo "" + if [[ ${#WARNINGS[@]} -eq 0 ]]; then - success "Report complete — no warnings" + log "$ICON_DONE All checks within thresholds ✅" else echo "$ICON_WARN Warnings: ${#WARNINGS[@]}" for w in "${WARNINGS[@]}"; do echo " $ICON_WARN $w" done - notify "ZFS weekly report on $(hostname) — ${#WARNINGS[@]} warning(s): ${WARNINGS[*]}" "ZFS Report" "warning" + notify "ZFS weekly report on $(hostname) — ${#WARNINGS[@]} warning(s): ${WARNINGS[*]}" \ + "ZFS Report" "warning" fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ ${#WARNINGS[@]} -gt 0 ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/Notes_To-Do.md b/Notes_To-Do.md new file mode 100644 index 0000000..3647d36 --- /dev/null +++ b/Notes_To-Do.md @@ -0,0 +1,40 @@ +echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc +source ~/.bashrc +claude + + + + +arrs system error, can we delete movies from lidar when dropped from tmdb, 99% of the time its future movies that get dropped +movie Silent Hill 2: The Movie (tmdbid 466226) was removed from TMDb + + + + + +later. +. fix failover strike list timing +. verify silent toggle switches back on good notifications + +. add to partnership, on offboard, remove all of containers that belonged to rmote, example remotes vaultwarden-jayred from my machine and leave my vaultwarden-Gmer4Lfe alone. and it does nothing to remote, thier side will hadle thier pc and remove my stuff from thier pc. + +. add updater to update containers while daily runs, along with a toggle to dissable in master. + +. add a script to check all docker containers and update any that still need it to run after the containers that get synced and updated. + + + + + + + +error starting emby + --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared 'lscr.io/linuxserver/emby' + +0e616c065aebad2604d2120fbf9c6bb449cdfc4021d68f167e4b5bc01741c133 +docker: Error response from daemon: failed to create task for container: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error jailing process inside rootfs: open /proc/self/mountinfo: no such file or directory + +Run 'docker run --help' for more information + +The command failed. +had to use --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode instead, for now, just to get it back online diff --git a/Orchestrators/README-Orchestrators.md b/Orchestrators/README-Orchestrators.md index f0e4f22..94ead42 100644 --- a/Orchestrators/README-Orchestrators.md +++ b/Orchestrators/README-Orchestrators.md @@ -1,449 +1,951 @@ -# Orchestrators +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# 🎯 ORCHESTRATORS +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Sequential job runners that coordinate multiple scripts into a single scheduled operation. +**Sequential job runners that coordinate multiple scripts into single scheduled units.** +Orchestrators contain no business logic — they call other scripts in order, track +pass/fail per job, and produce one clean summary. Configuration lives in `master.conf`. +Adding or removing a job never requires touching the orchestrator script itself. -Orchestrators do not contain business logic — they call other scripts in order, track pass/fail per job, and report a clean summary. All configuration lives in `Master.conf`. Adding or removing a job never requires touching the orchestrator script itself. +> **The User Scripts plugin contains only orchestrators.** Every cron entry, every +> "At Startup of Array" entry, every scheduled operation runs through an orchestrator. +> The individual scripts it calls are never scheduled directly — they run in a defined +> order inside a coordinated window, with a unified summary at the end. --- -## Why Orchestrators - -Without orchestrators, each script runs independently on its own schedule. This works but creates problems: - -- **Race conditions** — two scripts running simultaneously on the same data -- **Order dependency failures** — media cleaner runs before permissions, finds wrong ownership -- **No combined summary** — 6 separate notifications instead of one clean report -- **Scheduling complexity** — many cron entries instead of a few clean ones - -Orchestrators solve this by making a set of related scripts into a single scheduled unit with a defined execution order and a unified summary. +## ━━━ THE PROBLEM THAT BUILT THIS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ --- -## The Orchestrator Model +### 🔴 Race Conditions From Independent Scheduling -The ecosystem is designed so the User Scripts plugin contains only a small number of entries — each one an orchestrator that owns a domain: +Media permissions ran at 01:05. Arr cleanup ran at 01:00. Arr cleanup started +five minutes before permissions were applied — running against files that were still +owned by root, silently failing to delete the ones it should have cleaned up. Both +scripts reported success. Neither knew about the other. The result was a library +that looked cleaned but wasn't. -``` -At Startup of Array: - array_start.sh ← single entry, launches everything - -Cron: - transcode_management.sh ← */3 * * * * - arrs_failed_stalled_recovery.sh ← 0 */6 * * * - rsync.sh ... emby-failover ← */30 * * * * - daily_sync_maintenance.sh ← 0 1 * * * - weekly_sync_maintenance.sh ← 30 2 * * 0 - weekly_health_digest.sh ← Saturday morning - -Manual only: - failover_test.sh, emby_database_repair.sh, repair tools -``` - -All job lists are configured in the `ORCHESTRATORS` section of `Master.conf`. No changes to orchestrator scripts needed when adding or removing jobs. +The fix: orchestrators enforce order. `daily_sync_maintenance.sh` runs permissions +first, then arr cleanup. The arr scripts see correct ownership every time because the +orchestrator guarantees it. No race, no silent failure, no coordinating cron entries. --- -## Scripts +### 🔴 Six Separate Notifications Instead of One -### `array_start.sh` +Before orchestrators, each script sent its own notification on completion. A single +daily run produced six separate notification pings — one for permissions, one for +each arr cleanup, one for the cleaner, one for docker restart. Six bells for one +maintenance window. Worse, if something failed in the middle, you'd get some +notifications and not others, and figuring out which step failed meant correlating +timestamps across multiple notification messages. -Single entry point for the User Scripts "At Startup of Array" schedule. Launches all array-start scripts in order — each as a background process. +The fix: orchestrators collect all results and send one notification at the end. +One summary. One bell. Clear pass/fail count. If something failed, the summary +tells you which job and what happened — no correlation needed. + +--- + +### 🔴 Transcode Manager Triggering Unnecessary SSD Flips + +`transcode_manager.sh` ran every 3 minutes on its own. It checked ramdisk usage — +saw 6.8GB used, threshold is 6.5GB, flipped sessions to SSD. One minute later +`transcode_cleanup.sh` ran and removed 4GB of stale segment files from ended sessions. +Actual usage was 2.8GB. Sessions were now on SSD for no reason. Users experiencing +slightly worse performance. The flip counter incremented for nothing. + +The fix: `transcode_management.sh` runs cleanup first, manager second, every cycle. +The manager always sees post-cleanup usage. Stale files can't trigger a flip because +they're gone before the manager looks. The correct order requires exactly one +orchestrator to enforce it. + +--- + +### 🔴 Failed Imports Sitting Stalled for Days + +A release downloads successfully but Lidarr can't import it — wrong format, incorrect +tags, file already exists. Lidarr marks it `importFailed` and stops. Nobody notices. +The download client has the file, Lidarr has given up, and nothing is going to happen +until someone manually opens Lidarr, identifies the problem, blocklists the release, +and triggers a new search. This takes minutes to do — but nobody does it at 3am +when it usually happens. + +The fix: `arrs_failed_stalled_recovery.sh` runs every 6 hours. It finds all +`importFailed`, `importPending`, `error`, and `stalled` items, blocklists them, removes +them from the queue, and triggers a new search — automatically. By morning the failed +import has already been replaced by a working one. No manual intervention required. + +--- + +### 🔴 Array Start Scripts Running in Wrong Order or Not at All + +Scripts configured in the User Scripts plugin as "At Startup of Array" run in an +unpredictable order. The ramdisk setup might run after Emby starts. The syslog +filter might run after containers have already created veth interfaces. PHP-FPM +tuning might run after the WebGUI has already served its first requests. Each +script competes for the same startup slot with no guaranteed order. + +The fix: `array_start.sh` is the only "At Startup of Array" entry. It launches +every startup script in a defined order, with one-second settle between each, and +reports which succeeded and which failed. Order is guaranteed. Nothing starts before +its dependency. Everything is visible in a single summary. + +--- + +## ━━━ THE ORCHESTRATOR MODEL ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +The User Scripts plugin contains exactly these entries: ```bash -# Scheduled as: At Startup of Array -/mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh +# At Startup of Array — single entry for all startup scripts: +array_start.sh + +# Cron — one entry per maintenance window: +*/3 * * * * transcode_management.sh +0 */6 * * * arrs_failed_stalled_recovery.sh +*/30 * * * * rsync.sh emby-failover ← not an orchestrator, direct call +0 1 * * * daily_sync_maintenance.sh +30 2 * * 0 weekly_sync_maintenance.sh + +# Manual only (not scheduled): +failover_test.sh, emby_database_repair.sh, repair tools ``` -One-shot scripts (ramdisk, syslog filter, php-fpm, network connect) run and exit naturally. Continuous scripts (system watchdog, docker watchdog, failover) run until the array stops. +Every job list is configured in the `ORCHESTRATORS` section of `master.conf`. +No orchestrator script ever changes when jobs are added or removed — only `master.conf` changes. -**Configuration:** +--- + +## ━━━ THE ORCHESTRATOR PATTERN ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +All orchestrators follow the same structure: + +``` +1. Setup → validate config, detect_hosts(), acquire_lock +2. Pre-flight → fail fast checks before any work begins +3. Job loop → run each job, track pass/fail, continue on failure +4. Summary → one clean report of all job results +5. Notification → one notify per run on failure (never per job) +``` + +Properties that apply to every orchestrator: + +``` +Consistent output → every orchestrator looks the same in logs +No silent failures → pass/fail tracked per job, all in summary +Resilient → one job failing does not stop the rest +Single notification → one bell per run, not one per job +--dry-run cascade → passes --dry-run through to every child script +--status support → show configured jobs and exit +``` + +--- + +## ━━━ SCRIPTS AT A GLANCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +| Script | What It Orchestrates | Schedule | +|--------|---------------------|----------| +| `array_start.sh` | All array startup scripts in order | At Startup of Array | +| `transcode_management.sh` | Cleanup then manager — order critical | Every 3 minutes | +| `arrs_failed_stalled_recovery.sh` | Failed import + stalled download recovery | Every 6 hours | +| `daily_sync_maintenance.sh` | git pull → sync → media maintenance → restarts | 1am daily | +| `weekly_sync_maintenance.sh` | Stop → update → clean sync → start → weekly restarts | 2:30am Sunday | +| `media_management.sh` | Permissions → cleaners → arr cleanup | Via daily_sync (or manual) | + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🚀 array_start.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Single "At Startup of Array" entry for the entire ecosystem. Launches every startup +script in order — each as a background process — and reports which succeeded and which +failed. You never need to add individual scripts to the User Scripts startup list. ```bash -# Master.conf — ORCHESTRATORS section +# Scheduled: At Startup of Array (User Scripts plugin) +# This is the ONLY "At Startup of Array" entry in User Scripts +``` + +--- + +### ── Execution Order ────────────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Order matters — each entry depends on the previous ones having run. +# See comments for why each position is correct. +# ARRAY_START_SCRIPTS=( - "unRAID_Essentials/ramdisk_setup.sh" # creates ramdisk before Emby starts - "unRAID_Essentials/docker_syslog_filter.sh" # suppress veth log noise - "unRAID_Essentials/php_fpm_max_children.sh" # WebGUI tuning - "Docker_Essentials/docker_network_connect.sh" # connect containers to extra networks - "unRAID_Essentials/system_watchdog.sh" # continuous system health monitor - "Docker_Essentials/docker_watchdog.sh" # continuous container health monitor - "Failover/failover.sh" # continuous mutual failover + # ── One-shot scripts — run and exit naturally ───────────────────────────── + "unRAID_Essentials/inotify_tuning.sh" # raise inotify BEFORE containers start + # containers inherit limits at startup — + # if Code-Server starts with low limits + # it keeps them until restart + "unRAID_Essentials/docker_syslog_filter.sh" # suppress veth noise BEFORE containers create + # veth interfaces — otherwise the first boot + # always has unfiltered veth spam + "unRAID_Essentials/php_fpm_max_children.sh" # WebGUI tuning — before any WebGUI requests + "unRAID_Essentials/ramdisk_setup.sh" # create tmpfs + symlink BEFORE Emby starts — + # Emby needs the transcode path to exist + "Docker_Essentials/docker_network_connect.sh" # ensure networks + connections BEFORE + # watchdogs check container states + + # ── Continuous scripts — run until array stops ───────────────────────────── + "unRAID_Essentials/system_watchdog.sh" # system health BEFORE docker watchdog — + # system watchdog writes state file that + # docker watchdog reads every cycle + "Docker_Essentials/docker_watchdog.sh" # container health BEFORE failover — + # containers must be healthy for failover + # to make reliable decisions + "Failover/failover.sh" # failover LAST — needs everything else stable ) ``` -Add or remove scripts from `ARRAY_START_SCRIPTS` — no changes to `array_start.sh` needed. Order matters — ramdisk before Emby, network before watchdogs, watchdogs before failover. - --- -### `transcode_management.sh` - -Runs `transcode_cleanup.sh` then `transcode_manager.sh` in the correct order every 3 minutes. Replaces two separate cron entries with one. +### ── One-Shot vs Continuous Detection ─────────────────────────────────────── ```bash -# Scheduled as: */3 * * * * -/mnt/user/appdata/unraid_scripts/Orchestrators/transcode_management.sh -``` - -**Why cleanup must run before manager:** - -If the manager runs first it sees inflated ramdisk usage from stale segment files left by ended sessions — and triggers an unnecessary flip to SSD. Cleanup runs first to clear those files, then the manager decides based on real active session usage. - -``` -Without correct order: - Manager checks usage → 6.8GB (includes stale files) → flips to SSD - Cleanup runs → removes stale files → actual usage 2.1GB - Unnecessary flip — sessions now on SSD - -With correct order: - Cleanup runs → removes stale files → actual usage 2.1GB - Manager checks usage → 2.1GB → stays on ramdisk ✅ -``` - -**Daily statistics tracking:** - -Every cycle `transcode_management.sh` records stats to `TRANSCODE_DAILY_LOG`: -- Peak ramdisk usage for the day -- Total flip count for the day -- Ramdisk vs SSD session counts -- Files cleaned - -`weekly_health_digest.sh` reads this log for the weekly transcode summary. The log is bounded to `TRANSCODE_LOG_RETENTION` days — auto-purges on every write. - ---- - -### `arrs_failed_stalled_recovery.sh` - -Automatically detects and recovers from failed imports and stalled downloads across Sonarr, Radarr, and Lidarr. Blocklists the bad release and triggers a new search — hands-free recovery while you sleep. - -```bash -# Scheduled as: 0 */6 * * * (every 6 hours) -/mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh -``` - -Targets four problem types: `importFailed`, `importPending`, `error` status, and `stalled` downloads. Items newer than `ARR_IMPORT_RECOVERY_AGE` (6 hours) are skipped — gives the arr time to retry on its own first. - -**API versions:** Sonarr v4 → `/api/v3/` — Radarr v6 → `/api/v3/` — Lidarr v3 → `/api/v1/` - -Lidarr runs on HOST1 only — exits cleanly on HOST2. - -**Configuration:** - -```bash -# Master.conf — MEDIA section -ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this - -HOST1_SONARR_RECOVERY=true -HOST1_RADARR_RECOVERY=true -HOST1_LIDARR_RECOVERY=true -HOST2_SONARR_RECOVERY=true -HOST2_RADARR_RECOVERY=true +# ───────────────────────────────────────────────────────────────────────────── +# Each script is launched with `bash script.sh &` — background process. +# After 1 second: +# PID still alive → continuous script (running in background) +# logged as: "system_watchdog.sh — running (PID 12345)" +# PID dead + exit 0 → one-shot completed successfully +# logged as: "inotify_tuning.sh — completed (one-shot)" +# PID dead + exit N → failure +# logged as: "ramdisk_setup.sh — exited with code 1" +# full path printed — debugging is immediate +# +# This means the orchestrator correctly identifies and reports all startup +# scripts without needing to know in advance which ones are continuous. ``` --- -### `daily_sync_maintenance.sh` - -Full daily maintenance window orchestrator — git pull, media share sync, media management, and docker daily restarts. All driven by `Master.conf` arrays. +### ── Auto-Fix Permissions ──────────────────────────────────────────────────── ```bash -# Scheduled as: 0 1 * * * (1am daily — on both servers) -/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh +# ───────────────────────────────────────────────────────────────────────────── +# Scripts that are not executable are chmod +x'd automatically before launch. +# A permissions problem on a startup script does not cause a silent skip — +# the orchestrator fixes it and proceeds, then logs that it did so. +# This prevents "why didn't X run on startup" questions. ``` -**Execution order:** +--- -``` -1. Pre-sync jobs (DAILY_MAINTENANCE_SCRIPTS — git pull first): - git_pull_execute.sh ← always runs first — pulls latest scripts - -2. Media share sync: - HOST*_DAILY_SYNC_SHARES ← each server pushes its own truth shares - HOST*_PERSONAL_SHARES ← personal encrypted shares appended after - -3. Post-sync jobs (DAILY_MAINTENANCE_SCRIPTS — remaining): - media_management.sh ← permissions + cleaners + arr cleanup - docker_daily_restart.sh ← daily container restarts -``` - -**Bidirectional — same script, correct direction automatically:** - -``` -HOST1 runs daily_sync_maintenance.sh: - git pull → sync HOST1_DAILY_SYNC_SHARES → TO HOST2 → media_management → docker restart - -HOST2 runs daily_sync_maintenance.sh: - git pull → sync HOST2_DAILY_SYNC_SHARES → TO HOST1 → media_management → docker restart -``` - -`detect_hosts()` determines which server is local at runtime and selects the correct share list. No script changes needed — only `Master.conf` changes required. - -**Why one pre-flight check upfront:** - -Connectivity and rootfs are checked once before the loop starts — not once per share. If the remote is unreachable or rootfs is nearly full, the whole run fails fast. Individual share checks still run per-share inside `rsync.sh`. - -**Configuration:** +### ── Usage ─────────────────────────────────────────────────────────────────── ```bash -# Master.conf — ORCHESTRATORS section +# ───────────────────────────────────────────────────────────────────────────── +# Normal — called by User Scripts at array start. Never run manually in production. +# array_start.sh runs once and exits — the continuous scripts it launched +# keep running as background processes. +# ───────────────────────────────────────────────────────────────────────────── +array_start.sh +# ───────────────────────────────────────────────────────────────────────────── +# Dry run — show what would be launched, in order, without launching anything. +# Use to verify the ARRAY_START_SCRIPTS list before an array restart. +# ───────────────────────────────────────────────────────────────────────────── +array_start.sh --dry-run + +# ───────────────────────────────────────────────────────────────────────────── +# Status — show each configured script with its current running state. +# RUNNING (PID XXXXX) — continuous script currently active +# not running — one-shot that has completed, or continuous not yet started +# FILE NOT FOUND — script path wrong or missing +# ───────────────────────────────────────────────────────────────────────────── +array_start.sh --status +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🎬 transcode_management.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Runs `transcode_cleanup.sh` then `transcode_manager.sh` in the correct order every +3 minutes. Exists because the order is not optional — the manager must always see +post-cleanup usage to make accurate flip decisions. + +```bash +# Scheduled: */3 * * * * (every 3 minutes) +``` + +--- + +### ── Why Order Is Non-Negotiable ───────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Emby writes segment files to the transcode directory as it buffers streams. +# When a stream ends, Emby deletes its own active files — but may leave behind +# stale segment files from sessions that ended uncleanly. These consume real +# ramdisk space. The manager has no way to know if they're active or stale. +# +# Without correct order: +# Manager runs → sees 6.8GB used (stale files inflating) → exceeds threshold +# → flips sessions to SSD → flip counter incremented +# Cleanup runs → removes 4GB of stale files → actual usage was 2.8GB +# → flip was unnecessary — sessions now on SSD for no reason +# +# With correct order (this orchestrator): +# Cleanup runs → removes stale files → actual usage 2.8GB +# Manager runs → sees 2.8GB → below threshold → stays on ramdisk ✅ +# → no flip, no wasted counter, correct decision every time +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── What Each Child Script Does ──────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# transcode_cleanup.sh: +# Identifies segment files not currently open by any process (via lsof) +# Removes them from the transcode directory +# If usage drops enough after cleanup → triggers flip-back to ramdisk +# (handles the recovery direction so manager doesn't have to) +# +# transcode_manager.sh: +# Reads current ramdisk usage after cleanup has run +# Compares against RAMDISK_WARN_GB threshold +# Flips the symlink if needed (ramdisk → SSD or SSD → ramdisk) +# Writes one entry to TRANSCODE_DAILY_LOG for the weekly coffee report +# Shows active Emby sessions with their play method +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Daily Log ──────────────────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each cycle. +# Format: DATE|RAMDISK_USED_GB|FLIP_COUNT|RAM_SESSIONS|SSD_SESSIONS +# +# The sunday_morning_coffee_report.sh reads this log for weekly stats: +# Peak ramdisk usage across the week +# Total flip count (unnecessary flips visible here) +# Session split: how often ramdisk vs SSD was used +# +# Log trimmed to TRANSCODE_LOG_RETENTION days on every write — bounded, never grows. +# + TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db" + TRANSCODE_LOG_RETENTION=90 # days + TRANSCODE_STATE_FILE="/tmp/transcode_state.db" # /tmp — resets on reboot +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +transcode_management.sh # normal run (every 3 min via cron) +transcode_management.sh --dry-run # passes --dry-run to both child scripts +transcode_management.sh --status # show config, current state, daily log stats +transcode_management.sh --log # verbose output from both child scripts +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔄 arrs_failed_stalled_recovery.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Automatic recovery from failed imports and stalled downloads across Sonarr, Radarr, +and Lidarr. Blocklists the bad release, removes it from the queue, and triggers a +new search — hands-free recovery while you sleep. + +```bash +# Scheduled: 0 */6 * * * (every 6 hours) +``` + +--- + +### ── Four Problem Types ─────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# The arr queue API reports these status values for problem items: +# +# importFailed — downloaded successfully but arr couldn't import the file +# Common causes: wrong format for quality profile, corrupted file, +# duplicate already in library, permission issue on import path +# Self-resolution: never — arr stops trying after first failure +# +# importPending — downloaded, stuck waiting for import to begin +# Common causes: import queue backed up, arr paused, API error +# Self-resolution: sometimes — but stuck for hours is always wrong +# +# error — serious failure state not covered by the above +# Common causes: indexer issues, download client unreachable, disk full +# +# stalled — download stuck with no connections or no progress +# Common causes: no seeders, VPN routing issue, tracker ban +# Self-resolution: never without a source change +# +# NOT touched — items with status "downloading" or "imported" are never touched. +# Safe to run at any time — only processes items that are already broken. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── What It Does Per Problem Item ────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# For each problem item found, in order: +# +# 1. Blocklist the release +# Prevents the arr from grabbing the exact same release again immediately. +# The bad indexer result goes into the blocklist — future searches skip it. +# +# 2. Remove from queue +# Tells the download client to stop and remove the failed download. +# Frees up the slot for the replacement. +# +# 3. Trigger new search +# Arr searches for a different release meeting the quality profile. +# If a suitable alternative exists, it starts downloading automatically. +# If not, the item is marked as "awaiting upgrade" — arr will retry when +# a new indexer result appears. +# +# The entire cycle from "failed import" to "replacement downloading" happens +# without any human involvement. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Age Threshold ──────────────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Items newer than ARR_IMPORT_RECOVERY_AGE hours are skipped. +# Arrs retry on their own schedule after initial failures — a 30-minute-old +# importFailed may still resolve itself. Waiting 6 hours before intervening +# gives the arr a full retry cycle before this script steps in. +# + ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this +``` + +--- + +### ── Host Awareness ─────────────────────────────────────────────────────────── + +```bash +# master.conf + master_host*.conf +# ───────────────────────────────────────────────────────────────────────────── +# Each arr is independently toggled per host. +# Lidarr only runs on HOST1 — exits cleanly on HOST2 with no action. +# HOST2 has its own Sonarr and Radarr for its anime shares. +# + HOST1_SONARR_RECOVERY=true # HOST1 Sonarr — Tv_Shows + HOST1_RADARR_RECOVERY=true # HOST1 Radarr — Movies + HOST1_LIDARR_RECOVERY=true # HOST1 Lidarr — Music (HOST1 only) + HOST2_SONARR_RECOVERY=true # HOST2 Sonarr — Anime_Shows + HOST2_RADARR_RECOVERY=true # HOST2 Radarr — Anime_Movies +# +# API versions (current — update if arr major version changes): +# Sonarr v4 → /api/v3/ +# Radarr v6 → /api/v3/ +# Lidarr v3 → /api/v1/ +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +arrs_failed_stalled_recovery.sh # normal run +arrs_failed_stalled_recovery.sh --dry-run # show what would be actioned, no changes +arrs_failed_stalled_recovery.sh --log # verbose — show each queue item evaluated +arrs_failed_stalled_recovery.sh --status # show arr config and API connectivity +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 📅 daily_sync_maintenance.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Full daily maintenance window orchestrator. The entire 1am window — git pull, media +share sync, media maintenance, and docker daily restarts — in one scheduled entry. +Runs on both servers; `detect_hosts()` determines which direction each sync goes. + +```bash +# Scheduled: 0 1 * * * (1am daily — on BOTH servers) +``` + +--- + +### ── Execution Order ────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# The split into pre-sync and post-sync is based on a simple rule: +# git pull runs before anything — maintenance uses the latest scripts +# rsync runs in the middle — media management uses the post-sync state +# docker restart runs last — after everything else has completed +# +# 1. Pre-sync: git_pull_execute.sh +# Pull latest scripts first. Everything that follows runs on current code. +# +# 2. Rsync window: HOST*_DAILY_SYNC_SHARES + HOST*_PERSONAL_SHARES +# Each server pushes its own truth shares to the other. +# HOST1 → pushes Movies, Tv_Shows, Music → HOST2 +# HOST2 → pushes Anime_Shows, Anime_Movies → HOST1 +# Personal encrypted shares appended after standard shares. +# Drive temperature exit codes respected — skip share or abort all on CRIT. +# +# 3. Post-sync: DAILY_MAINTENANCE_SCRIPTS (everything except git pull) +# media_management.sh → permissions + cleaners + arr cleanup +# docker_daily_restart.sh → nightly container restarts +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Bidirectional — Same Script, Correct Direction Automatic ──────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID at runtime and aliases HOST*_DAILY_SYNC_SHARES +# to DAILY_SYNC_SHARES. The script uses DAILY_SYNC_SHARES — always the right +# list for whichever server is running. +# +# No HOST1/HOST2 comparisons in the script. Configuration drives direction. +# +# HOST1 runs this script at 1am: +# → pushes HOST1_DAILY_SYNC_SHARES (Movies, Tv_Shows, Music) → HOST2 +# → media_management.sh on HOST1's shares +# → docker_daily_restart.sh on HOST1's containers +# +# HOST2 runs this script at 1am: +# → pushes HOST2_DAILY_SYNC_SHARES (Anime_Shows, Anime_Movies) → HOST1 +# → media_management.sh on HOST2's shares +# → docker_daily_restart.sh on HOST2's containers +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Configuration ──────────────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Script lists — git_pull is split out as pre-sync; everything else is post-sync. +# The orchestrator recognises git_pull_execute.sh by name and routes it correctly. +# DAILY_MAINTENANCE_SCRIPTS=( - "git_pull_execute.sh" # always first - "Docker_Essentials/docker_daily_restart.sh" # after sync and media jobs -) - -# Media jobs run between sync and docker restart -# Permissions first, cleaners second, arr cleanup last -MEDIA_MANAGEMENT_JOBS=( - "Media/media_shares_permissions.sh" # permissions — everything depends on this - "Media/media_cleaner.sh anime" # clean junk before arr scripts scan - "Media/media_cleaner.sh media" - "Media/lidarr_cleanup.sh" # arr cleanup last — depends on clean folders - "Media/sonarr_cleanup.sh" - "Media/radarr_cleanup.sh" + "git_pull_execute.sh" # PRE-SYNC — always first + "Media/media_shares_permissions.sh" # POST-SYNC — permissions before arr + "Media/media_cleaner.sh anime" # POST-SYNC — junk before orphan scan + "Media/media_cleaner.sh media" # POST-SYNC + "Media/lidarr_cleanup.sh" # POST-SYNC — orphan cleanup last + "Media/sonarr_cleanup.sh" # POST-SYNC + "Media/radarr_cleanup.sh" # POST-SYNC + "Docker_Essentials/docker_daily_restart.sh" # POST-SYNC — restarts after everything ) +# master_host1.conf HOST1_DAILY_SYNC_SHARES=( - /mnt/user/Movies - /mnt/user/Tv_Shows - /mnt/user/Music - # all HOST1-owned shares + "/mnt/user/Movies" # HOST1 source of truth — push to HOST2 + "/mnt/user/Tv_Shows" # HOST1 source of truth + "/mnt/user/Music" # HOST1 source of truth + "/mnt/user/Kids_Movies" + "/mnt/user/Kids_Tv_Shows" + "/mnt/user/Sports" + "/mnt/user/stand-up_comedy" ) +HOST1_PERSONAL_SHARES=( + "/mnt/user/Personal" # encrypted personal share — appended after standard +) + +# master_host2.conf HOST2_DAILY_SYNC_SHARES=( - /mnt/user/Anime_Shows - /mnt/user/Anime_Movies + "/mnt/user/Anime_Shows" # HOST2 source of truth — push to HOST1 + "/mnt/user/Anime_Movies" # HOST2 source of truth ) ``` -**Adding a media job:** +--- + +### ── Adding or Removing a Job ──────────────────────────────────────────────── ```bash -MEDIA_MANAGEMENT_JOBS=( +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Add a new media script — just insert it in the right position: +DAILY_MAINTENANCE_SCRIPTS=( + "git_pull_execute.sh" "Media/media_shares_permissions.sh" "Media/media_cleaner.sh anime" "Media/media_cleaner.sh media" - "Media/my_new_script.sh" # ← just add it here + "Media/my_new_script.sh" # ← add here in the correct order "Media/lidarr_cleanup.sh" "Media/sonarr_cleanup.sh" "Media/radarr_cleanup.sh" + "Docker_Essentials/docker_daily_restart.sh" ) -``` -**Disabling a media job temporarily:** - -```bash -MEDIA_MANAGEMENT_JOBS=( +# Disable a job temporarily — comment it out, do not delete: +DAILY_MAINTENANCE_SCRIPTS=( + "git_pull_execute.sh" "Media/media_shares_permissions.sh" -# "Media/media_cleaner.sh anime" # ← disabled, not deleted +# "Media/media_cleaner.sh anime" # ← temporarily disabled "Media/media_cleaner.sh media" "Media/lidarr_cleanup.sh" - "Media/sonarr_cleanup.sh" - "Media/radarr_cleanup.sh" + ... ) +# ───────────────────────────────────────────────────────────────────────────── +# No changes to daily_sync_maintenance.sh needed in either case. ``` -**Relationship to failover writeback:** - -The same share lists are used by `failover.sh` for Tier 4 writeback — but in the opposite direction. If HOST1 was down for 24hr+ and HOST2's arrs accumulated content, writeback pushes `HOST1_DAILY_SYNC_SHARES` FROM HOST2 BACK TO HOST1. No duplicate configuration needed. - --- -### `weekly_sync_maintenance.sh` - -Weekly maintenance window orchestrator — critical appdata clean sync, container updates, and weekly docker restarts. Runs Sunday 2:30am, fits before the 3am network reboot. +### ── Drive Temperature Exit Codes ─────────────────────────────────────────── ```bash -# Scheduled as: 30 2 * * 0 (Sunday 2:30am) -/mnt/user/appdata/unraid_scripts/Orchestrators/weekly_sync_maintenance.sh +# ───────────────────────────────────────────────────────────────────────────── +# rsync.sh returns specific exit codes for temperature issues. +# The orchestrator handles these correctly — one hot drive does not abort all others. +# +# exit 0 → success — continue to next share +# exit 1 → temp WARN — skip this share, continue to next share +# exit 2 → temp CRITICAL — abort ALL remaining shares in this window +# notify immediately with which share triggered the abort +# exit N → other failure — skip this share, continue to next share +# ───────────────────────────────────────────────────────────────────────────── ``` -**Execution order:** +--- -``` -1. Stop local containers — auth stack + Emby stopped locally -2. Stop remote containers — auth stack + Emby stopped remotely via SSH -3. Pull updates locally — if CRITICAL_SYNC_UPDATES=true -4. Pull updates remotely — if CRITICAL_SYNC_UPDATES_REMOTE=true -5. rsync WEEKLY_SYNC_JOBS — Emby + Critical-Data clean sync -6. Start remote containers — starts on new images, correct order -7. Start local containers — starts on new images, correct order - -Post-sync jobs (WEEKLY_MAINTENANCE_SCRIPTS): -8. docker_weekly_restart.sh -``` - -**Why two Emby syncs:** - -The emby-failover dirty sync runs every 30-60 minutes with Emby running — WAL files excluded, watch states and library pushed continuously. HOST2 stays current on what users are watching. But it is not a clean database snapshot. - -The weekly clean sync stops Emby on both sides, checkpoints the WAL, and pushes a full consistent mirror. HOST2 gets a gold-standard Emby state once per week. - -``` -emby-failover every 30-60min (Emby running): - users.db, library.db, authentication.db, config/ - WAL excluded — safe mid-write - HOST2 always within 30-60min of HOST1 on watch state - -weekly clean sync Sunday 2:30am (Emby stopped): - Full clean mirror — all databases flushed - metadata, plugins, config all included - ~30s downtime — both Emby instances down during sync only - Cache stays warm on HOST2 all week — only reset Sunday -``` - -**Why weekly instead of nightly:** - -Emby builds a warm image cache on HOST2 naturally throughout the week. Syncing nightly resets this cache — users experience slow image loads every morning. Weekly sync lets the cache stay warm for 6 days and only resets on Sunday night when most users are asleep. - -**What it syncs:** - -``` -WEEKLY_SYNC_JOBS (configurable in Master.conf): - /mnt/user/Media_Server/Emby ← emby profile — full clean mirror - /mnt/user/appdata-Failover/Critical-Data ← critical-data profile — auth stack - -Emby excludes: logs, transcodes, cache, crash* -Auth excludes: logs, *.tmp, nginx/temp, nginx/cache, notification.txt -``` - -**Container update window:** - -Containers are already stopped for the sync — container image updates pull at zero extra downtime. Both servers start on the same new image version after the sync. +### ── Relationship to Failover Writeback ────────────────────────────────────── ```bash -# Master.conf toggles -CRITICAL_SYNC_UPDATES=true # pull updates locally -CRITICAL_SYNC_UPDATES_REMOTE=true # pull updates on remote via SSH - -# Toggle false to skip updates without changing the schedule -CRITICAL_SYNC_UPDATES=false +# ───────────────────────────────────────────────────────────────────────────── +# The same HOST*_DAILY_SYNC_SHARES lists are used by failover.sh for Tier 4 +# writeback — but in the opposite direction. +# +# Normal (daily_sync_maintenance.sh): +# HOST1 → pushes Movies, Tv_Shows → HOST2 +# +# Tier 4 failover writeback (HOST1 returns after 24hr+ outage): +# HOST2 → pushes Movies, Tv_Shows → HOST1 +# (HOST2 was running HOST1's arrs and accumulated content) +# +# Same list, correct direction for the situation, zero duplication. +# No separate writeback list to maintain. +# ───────────────────────────────────────────────────────────────────────────── ``` -**Why auth stack matters:** +--- -The auth stack (Authelia, NPM, Mariadb, Redis, LLDAP) runs warm on both servers. During normal operation HOST2 serves its own domain independently. The weekly clean sync is the only time auth state propagates from HOST1 to HOST2. - -- New user added on HOST1 → propagates to HOST2 on Sunday automatically -- Proxy rule changes → propagated Sunday -- No manual intervention needed for routine auth changes - -**Sunday maintenance window:** - -``` -2:30am weekly_sync_maintenance.sh ← clean sync + updates (~3-5min) -2:50am CA Auto Update plugin ← plugin updates -2:55am CA container updates ← docker container updates -3:00am Network reboot ← router/switch restart - -Everything comes back clean: - Network fresh, Emby updated, auth stack updated - All in one maintenance window while users sleep -``` - -**Configuration:** +### ── Usage ─────────────────────────────────────────────────────────────────── ```bash -# Master.conf — ORCHESTRATORS section +daily_sync_maintenance.sh # normal run +daily_sync_maintenance.sh --dry-run # preview all jobs without syncing or changing +daily_sync_maintenance.sh --log # verbose per-share and per-job output +daily_sync_maintenance.sh --status # show configured shares and jobs for this host +``` -WEEKLY_SYNC_JOBS=( - "/mnt/user/Media_Server/Emby" - "/mnt/user/appdata-Failover/Critical-Data" +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 📅 weekly_sync_maintenance.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Weekly maintenance window orchestrator — clean sync of Emby and auth stack, container +image updates, and weekly docker restarts. Runs Sunday 2:30am; fits inside the Sunday +maintenance block before the 7am coffee report. + +```bash +# Scheduled: 30 2 * * 0 (Sunday 2:30am) +``` + +--- + +### ── Execution Order ────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Containers stop BEFORE sync — clean static source, full bandwidth. +# Containers start AFTER sync — on fresh data, in dependency order. +# DDNS and failover continue running throughout — only managed containers stop. +# +# 1. Pre-flight checks — connectivity, remote Docker daemon, remote rootfs +# 2. Stop local containers — Emby + auth stack stopped on this server +# 3. Stop remote containers — Emby + auth stack stopped on remote via SSH +# 4. Pull updates locally — if WEEKLY_SYNC_UPDATES=true (containers already stopped) +# 5. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true +# 6. rsync WEEKLY_SYNC_SHARES — full clean mirror at full bandwidth +# 7. Start remote containers — new image, correct dependency order +# 8. Start local containers — new image, correct dependency order +# Post-sync jobs: docker_weekly_restart.sh +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Why Weekly Not Nightly for Emby ───────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Two Emby syncs run in parallel — dirty and clean: +# +# emby-failover dirty sync (every 30 minutes, Emby running): +# watch states, library deltas, user activity — continuous coverage +# WAL files excluded — safe to copy while Emby writes +# HOST2 always within 30 minutes of HOST1 on playback state +# +# weekly clean sync (Sunday 2:30am, Emby stopped): +# Full clean mirror — all databases checkpointed and flushed +# Metadata, plugins, config all included +# ~30 seconds of Emby downtime — both instances stopped during rsync only +# +# Why not nightly: +# Emby builds a warm image thumbnail cache on HOST2 throughout the week. +# Nightly sync resets this cache — cold loads every morning for users. +# Weekly sync: cache stays warm for 6 days. Resets Sunday night while users sleep. +# One weekly reset at an acceptable time is better than six unnecessary resets. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Container Update Window ───────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Containers are already stopped for the sync — pulling updated images costs +# nothing extra in downtime. Both servers start on the same image version +# after the window completes. +# + WEEKLY_SYNC_UPDATES=true # pull updated images locally + WEEKLY_SYNC_UPDATES_REMOTE=true # pull updated images on remote via SSH +# +# Toggle false to skip updates without changing the schedule: +# WEEKLY_SYNC_UPDATES=false # skips pulls, containers still restart on current image +``` + +--- + +### ── Why Auth Stack Weekly Sync Matters ────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# NPM, Authelia, LLDAP, Mariadb run warm on both servers continuously. +# HOST1 is source of truth — changes propagate to HOST2 via weekly clean sync. +# +# What propagates automatically every Sunday: +# New users added in LLDAP on HOST1 → appear on HOST2 by Monday +# Proxy rules changed in NPM on HOST1 → live on HOST2 by Monday +# Authelia policies updated on HOST1 → enforced on HOST2 by Monday +# TLS certificates renewed on HOST1 → valid on HOST2 by Monday +# +# No manual sync needed for routine auth administration. +# Anything done on HOST1 is on HOST2 within a week. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Sunday Maintenance Window ─────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# This script is part of a coordinated Sunday maintenance block: +# +# 2:30am weekly_sync_maintenance.sh ← clean sync + image updates (~5-10min) +# 2:50am CA Auto Update plugin ← unRAID plugin updates +# 2:55am CA container updates ← additional container updates +# 3:00am Network reboot ← router/switch restart +# +# Everything comes back clean: +# Network fresh, Emby + auth updated, containers on latest images. +# All in one window while users sleep. +# Sunday morning coffee report at 7am shows the post-maintenance state. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Configuration ──────────────────────────────────────────────────────────── + +```bash +# master.conf + WEEKLY_SYNC_SHARES=( + "/mnt/user/Media_Server/Emby" # emby profile — full clean mirror + "/mnt/user/appdata-Failover/Critical-Data" # critical-data — auth stack clean state + ) + + WEEKLY_MAINTENANCE_SCRIPTS=( + "Docker_Essentials/docker_weekly_restart.sh" # weekly restart of less-critical services + ) + + WEEKLY_SYNC_UPDATES=true + WEEKLY_SYNC_UPDATES_REMOTE=true +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +weekly_sync_maintenance.sh # normal run +weekly_sync_maintenance.sh --dry-run # preview — no stops, no syncs, no starts +weekly_sync_maintenance.sh --log # verbose per-share per-job output +weekly_sync_maintenance.sh --status # show configured shares, jobs, update toggles +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🧹 media_management.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Runs all media maintenance scripts sequentially in the order defined by +`MEDIA_MAINTENANCE_JOBS` in `master.conf`. Called by `daily_sync_maintenance.sh` +as a post-sync job — not scheduled separately. Available for manual runs when +media maintenance is needed outside the normal window. + +```bash +# Called by: daily_sync_maintenance.sh (post-sync) +# Manual use: run directly for ad hoc media maintenance +``` + +--- + +### ── Execution Order ────────────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Order is critical — see README-Media.md for detailed explanation. +# Short version: permissions first, cleaners second, arr cleanup last. +# Each step depends on the previous having completed correctly. +# +MEDIA_MAINTENANCE_JOBS=( + "Media/media_shares_permissions.sh" # 1. permissions — arr cleanup depends on this + "Media/media_cleaner.sh anime" # 2. junk removal — orphan scan depends on this + "Media/media_cleaner.sh media" # 3. same for media shares + "Media/lidarr_cleanup.sh" # 4. orphan cleanup — last, after permissions + clean + "Media/sonarr_cleanup.sh" # 5. + "Media/radarr_cleanup.sh" # 6. ) - -WEEKLY_MAINTENANCE_SCRIPTS=( - "Docker_Essentials/docker_weekly_restart.sh" -) - -CRITICAL_SYNC_UPDATES=true -CRITICAL_SYNC_UPDATES_REMOTE=true +# ───────────────────────────────────────────────────────────────────────────── +# Add a script: insert in the correct position for your use case. +# Remove a script: comment it out with # +# No changes to media_management.sh needed in either case. ``` --- -### `media_management.sh` - -Runs all media maintenance scripts sequentially in the order defined in `Master.conf`. Absorbed into `daily_sync_maintenance.sh` via `MEDIA_MANAGEMENT_JOBS` — not scheduled separately. Available for manual runs. +### ── Usage ─────────────────────────────────────────────────────────────────── ```bash -# Manual use only — called automatically by daily_sync_maintenance.sh -bash /mnt/user/appdata/unraid_scripts/Orchestrators/media_management.sh --dry-run -bash /mnt/user/appdata/unraid_scripts/Orchestrators/media_management.sh +media_management.sh # normal run — all jobs in order +media_management.sh --dry-run # preview without any deletions or changes +media_management.sh --log # verbose output from all jobs +media_management.sh --status # show configured job list and exit ``` --- -## The Orchestrator Pattern - -All orchestrators follow the same pattern: - -``` -1. Setup — validate config, detect hosts if needed, acquire lock -2. Pre-flight — fail fast checks before doing any work -3. Job loop — run each job, track pass/fail, continue on failure -4. Summary — one clean report of all results -5. Notification — one notification per run, not one per job -``` - -This means: -- **Consistent output** — every orchestrator looks the same in logs -- **No silent failures** — pass/fail tracked per job, reported in summary -- **Single notification** — one bell ring per run -- **Resilient** — one job failing doesn't stop the rest - ---- - -## Scheduling +## ━━━ COMPLETE SCHEDULE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ```bash -# At Startup of Array -array_start.sh # single entry — launches all startup scripts +# ───────────────────────────────────────────────────────────────────────────── +# At Startup of Array — single entry: +# ───────────────────────────────────────────────────────────────────────────── +array_start.sh -# Every 3 minutes -*/3 * * * * transcode_management.sh +# ───────────────────────────────────────────────────────────────────────────── +# Frequent — every 3 minutes: +# ───────────────────────────────────────────────────────────────────────────── +*/3 * * * * transcode_management.sh -# Every 6 hours -0 */6 * * * arrs_failed_stalled_recovery.sh +# ───────────────────────────────────────────────────────────────────────────── +# Periodic — not an orchestrator, direct rsync call: +# ───────────────────────────────────────────────────────────────────────────── +*/30 * * * * rsync.sh /mnt/user/Media_Server/Emby --profile=emby-failover -# Every 30-60 minutes -*/30 * * * * rsync.sh /mnt/user/Media_Server/Emby --profile=emby-failover +# ───────────────────────────────────────────────────────────────────────────── +# Every 6 hours: +# ───────────────────────────────────────────────────────────────────────────── +0 */6 * * * arrs_failed_stalled_recovery.sh -# Daily 1am — full daily maintenance window: -# git pull → media sync → permissions → cleaners → arr cleanup → docker restart -0 1 * * * daily_sync_maintenance.sh +# ───────────────────────────────────────────────────────────────────────────── +# Daily — 1am: +# git pull → rsync all truth shares → permissions + cleaners + arr cleanup → docker restart +# ───────────────────────────────────────────────────────────────────────────── +0 1 * * * daily_sync_maintenance.sh -# Weekly — Sunday morning -30 2 * * 0 weekly_sync_maintenance.sh # clean sync + updates + docker weekly restart -50 2 * * 0 CA plugin update -55 2 * * 0 CA container updates +# ───────────────────────────────────────────────────────────────────────────── +# Weekly — Sunday maintenance block: +# ───────────────────────────────────────────────────────────────────────────── +30 2 * * 0 weekly_sync_maintenance.sh # clean sync + updates (~5-10min) +50 2 * * 0 CA Auto Update plugin # plugin updates +55 2 * * 0 CA container updates # container image updates +0 3 * * 0 Network reboot # router/switch restart ``` -`daily_sync_maintenance.sh` owns the entire daily window — git pull, media sync, permissions, cleaners, arr cleanup, and docker restarts in one scheduled run. Everything configured in `Master.conf` via `DAILY_MAINTENANCE_SCRIPTS` and `MEDIA_MANAGEMENT_JOBS`. - --- -## Adding a New Orchestrator +## ━━━ ADDING A NEW ORCHESTRATOR ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -If you find yourself running 3 or more related scripts on the same schedule, consider wrapping them in a new orchestrator. Model it directly on `media_management.sh` which handles dry-run passthrough, status display, pass/fail tracking and summary reporting. - -Minimal skeleton: +If you find yourself running 3+ related scripts on the same schedule, wrap them +in a new orchestrator. Model directly on `media_management.sh` which has the +complete pattern — dry-run passthrough, status display, pass/fail tracking, summary. ```bash -JOBS=( - "Folder/script1.sh" - "Folder/script2.sh arg" -) +# Minimal skeleton — the full pattern in its simplest form: +#!/bin/bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../load_config.sh" +parse_args "$@" +SCRIPTS_ROOT="$SCRIPT_DIR/.." PASS=() FAIL=() -for JOB in "${JOBS[@]}"; do - SCRIPT=$(echo "$JOB" | cut -d' ' -f1) - ARG=$(echo "$JOB" | cut -d' ' -f2-) +# Read job list from master.conf — never hardcode jobs in the orchestrator +for script_entry in "${MY_MAINTENANCE_JOBS[@]:-}"; do + [[ -z "$script_entry" ]] && continue - if bash "$ECOSYSTEM_ROOT/$SCRIPT" $ARG; then - PASS+=("$SCRIPT") + read -r -a parts <<< "$script_entry" + script_path="$SCRIPTS_ROOT/${parts[0]}" + script_name=$(basename "${parts[0]}") + extra_args=("${parts[@]:1}") + + [[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run") + + if bash "$script_path" "${extra_args[@]}"; then + PASS+=("$script_name") else - FAIL+=("$SCRIPT") + FAIL+=("$script_name") fi done + +# One summary — one notification +echo "Passed: ${#PASS[@]} Failed: ${#FAIL[@]}" +[[ ${#FAIL[@]} -gt 0 ]] && \ + notify "My maintenance failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \ + "My Orchestrator" "warning" ``` \ No newline at end of file diff --git a/Orchestrators/array_start.sh b/Orchestrators/array_start.sh index a2c3713..883340f 100644 --- a/Orchestrators/array_start.sh +++ b/Orchestrators/array_start.sh @@ -1,53 +1,136 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Array Start Orchestrator ----------------------------------- -# ----------------------------------------------------------------------------------------------- -# Single entry point for "At Startup of Array" in User Scripts plugin. -# Launches everything configured in ARRAY_START_SCRIPTS in Master.conf. +# ============================================================================================== +# ================================= Array Start Orchestrator =================================== +# ============================================================================================== +# Single entry point for "At Startup of Array" in the User Scripts plugin. +# Launches everything configured in ARRAY_START_SCRIPTS in master.conf. +# This script exits after launching all scripts — unRAID sees it complete normally. # -# What it launches (configured in Master.conf ARRAY_START_SCRIPTS): -# Transcodes/ramdisk_setup.sh — creates tmpfs + symlink before Emby starts (one-shot) -# unRAID_Essentials/docker_syslog_filter.sh — suppress veth log noise (one-shot) -# unRAID_Essentials/php_fpm_max_children.sh — WebGUI performance tuning (one-shot) -# Docker_Essentials/docker_network_connect.sh — ensure networks exist + connect containers (one-shot) -# unRAID_Essentials/system_watchdog.sh — system health monitor (continuous loop) -# Docker_Essentials/docker_watchdog.sh — container health monitor (continuous loop) -# Failover/failover.sh — mutual failover monitor (continuous loop) +# ── WHAT IT LAUNCHES ────────────────────────────────────────────────────────────────────────── +# Configured in master.conf ARRAY_START_SCRIPTS — no changes to this script ever needed. +# Current order (order matters — see below): # -# One-shot scripts run and exit naturally — array_start.sh confirms completion. -# Continuous scripts run until array stops or SIGTERM received. -# This orchestrator exits after launching all scripts — unRAID sees it complete normally. +# ONE-SHOT (run and exit naturally): +# unRAID_Essentials/inotify_tuning.sh — raise inotify limits before containers start +# unRAID_Essentials/docker_syslog_filter.sh — suppress veth log noise before logs fill +# unRAID_Essentials/php_fpm_max_children.sh — WebGUI performance tuning +# Transcodes/ramdisk_setup.sh — create tmpfs + symlink before Emby starts +# Docker_Essentials/docker_network_connect.sh — ensure networks + container connections # -# Add or remove scripts: edit ARRAY_START_SCRIPTS in Master.conf. -# Order matters — ramdisk first, network before watchdogs, watchdogs before failover. -# No changes to this script ever needed. -# ----------------------------------------------------------------------------------------------- +# CONTINUOUS (run until array stops): +# unRAID_Essentials/system_watchdog.sh — system health monitor (last line of defense) +# Docker_Essentials/docker_watchdog.sh — container health monitor +# Failover/failover.sh — mutual failover monitor +# +# ── WHY ORDER MATTERS ───────────────────────────────────────────────────────────────────────── +# inotify_tuning.sh — must run BEFORE Code-Server and other containers start +# containers that start with low inotify limits keep them ✅ +# docker_syslog_filter — must run BEFORE any container starts creating veth interfaces +# ramdisk_setup.sh — must run BEFORE Emby starts transcoding +# docker_network_connect — must run BEFORE watchdogs check container states +# system_watchdog.sh — before docker_watchdog (system > container priority) +# docker_watchdog.sh — before failover (containers must be healthy for failover) +# failover.sh — last — needs everything else stable to make decisions +# +# ── ONE-SHOT vs CONTINUOUS DETECTION ───────────────────────────────────────────────────────── +# Script is launched in background with bash script.sh & +# After 1 second: if PID still alive → continuous (running in background) +# if PID dead + exit 0 → one-shot completed successfully +# if PID dead + exit N → failure +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# Root check — all launched scripts require root +# acquire_lock — prevents duplicate array start launches +# detect_hosts() — MY_ID in notifications +# validate_unraid_cmd — notify validated before use +# chmod +x auto-fix — non-executable scripts fixed before launch +# Full path on failure — shows exact path for debugging +# notify on failures — alert if any script fails to launch +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# ARRAY_START_SCRIPTS — ordered list of scripts to launch at array start +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# array_start.sh — normal launch (called by User Scripts at array start) +# array_start.sh --dry-run — show what would be launched without launching +# array_start.sh --status — show configured scripts and their current state +# array_start.sh --log — verbose output per script +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -source "$ECOSYSTEM_ROOT/Master.conf" -source "$ECOSYSTEM_ROOT/common.sh" +source "$ECOSYSTEM_ROOT/load_config.sh" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Array Start — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +parse_args "$@" +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi -success "Running as root" +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" -SCRIPT_COUNT=${#ARRAY_START_SCRIPTS[@]} -info "Launching $SCRIPT_COUNT script(s)..." +acquire_lock + +detect_hosts + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — scripts will not be launched" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY ARRAY START STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_GEAR Scripts: ${#ARRAY_START_SCRIPTS[@]} configured" + echo "" + + for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do + [[ -z "$relative_path" ]] && continue + script_path="$ECOSYSTEM_ROOT/$relative_path" + script_name=$(basename "$script_path") + + if [[ ! -f "$script_path" ]]; then + echo " $ICON_ERROR $script_name — FILE NOT FOUND" + echo " $script_path" + continue + fi + + [[ ! -x "$script_path" ]] && flag=" (not executable — will auto-fix)" || flag="" + + # Check if currently running + if pgrep -f "$script_path" >/dev/null 2>&1; then + RUN_PID=$(pgrep -f "$script_path" | head -1) + echo " $ICON_RUNNING $script_name — RUNNING (PID $RUN_PID)${flag}" + else + echo " $ICON_NOT_RUNNING $script_name — not running${flag}" + fi + done + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Launch Scripts ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_GEAR Array Start — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +log "Ecosystem root: $ECOSYSTEM_ROOT" +log "Launching ${#ARRAY_START_SCRIPTS[@]} script(s)..." echo "" +START=$(date +%s) LAUNCHED=0 FAILED=0 +FAILED_SCRIPTS=() for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do [[ -z "$relative_path" ]] && continue @@ -55,57 +138,80 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do SCRIPT_PATH="$ECOSYSTEM_ROOT/$relative_path" SCRIPT_NAME=$(basename "$SCRIPT_PATH") + # File existence check if [[ ! -f "$SCRIPT_PATH" ]]; then - error "$SCRIPT_NAME — not found at $SCRIPT_PATH" - ((FAILED++)) + error "$SCRIPT_NAME — not found" + error " Expected: $SCRIPT_PATH" + (( FAILED++ )) + FAILED_SCRIPTS+=("$SCRIPT_NAME") continue fi + # Auto-fix permissions — chmod +x if needed if [[ ! -x "$SCRIPT_PATH" ]]; then - error "$SCRIPT_NAME — not executable" - ((FAILED++)) + warn "$SCRIPT_NAME — not executable, fixing..." + chmod +x "$SCRIPT_PATH" || { + error "$SCRIPT_NAME — chmod +x failed" + (( FAILED++ )) + FAILED_SCRIPTS+=("$SCRIPT_NAME") + continue + } + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would launch: $SCRIPT_NAME" + (( LAUNCHED++ )) continue fi - info "$ICON_START Launching $SCRIPT_NAME..." + log "$ICON_START Launching $SCRIPT_NAME..." bash "$SCRIPT_PATH" & PID=$! - # Brief pause to let script initialize and catch immediate failures + # Brief settle — 1s enough to detect immediate failures sleep 1 if kill -0 "$PID" 2>/dev/null; then - success "$SCRIPT_NAME — running (PID $PID)" - ((LAUNCHED++)) + # Still running → continuous script + warn "$SCRIPT_NAME — running (PID $PID) ✅" + (( LAUNCHED++ )) else - # Script exited — check if it was a one-shot (exit 0) or a failure + # Exited — check if one-shot success or failure wait "$PID" EXIT_CODE=$? if [[ "$EXIT_CODE" -eq 0 ]]; then - success "$SCRIPT_NAME — completed (one-shot)" - ((LAUNCHED++)) + log "$SCRIPT_NAME — completed (one-shot) ✅" + (( LAUNCHED++ )) else error "$SCRIPT_NAME — exited with code $EXIT_CODE" - ((FAILED++)) + error " Path: $SCRIPT_PATH" + (( FAILED++ )) + FAILED_SCRIPTS+=("$SCRIPT_NAME") fi fi done -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +END=$(date +%s) + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_SUCCESS Launched: $LAUNCHED" -echo "$ICON_ERROR Failed: $FAILED" -echo "$ICON_TIME Time: $(date '+%H:%M:%S')" +[[ "$FAILED" -gt 0 ]] && echo "$ICON_ERROR Failed: $FAILED — ${FAILED_SCRIPTS[*]}" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" -if [[ "$FAILED" -gt 0 ]]; then - echo "$ICON_WARN Status: $FAILED script(s) failed to launch — check logs" - notify "Array start on $(hostname) — $FAILED script(s) failed to launch" "Array Start" "warning" +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — no scripts launched" +elif [[ "$FAILED" -gt 0 ]]; then + warn "Status: $FAILED script(s) failed — ${FAILED_SCRIPTS[*]}" + notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \ + "Array Start" "warning" else - echo "$ICON_DONE Status: $ICON_SUCCESS All scripts launched" + log "$ICON_DONE Status: all $LAUNCHED script(s) launched ✅" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Orchestrators/critical_sync_maintenance.sh b/Orchestrators/critical_sync_maintenance.sh index 8ea6252..2cdef74 100644 --- a/Orchestrators/critical_sync_maintenance.sh +++ b/Orchestrators/critical_sync_maintenance.sh @@ -1,69 +1,136 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# ----------------------------- Critical Sync Maintenance -------------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ============================= Critical Sync Maintenance ====================================== +# ============================================================================================== # Orchestrator for time-sensitive syncs that run every 15 minutes. # Keeps the mirror current between the less frequent daily and weekly windows. -# Schedule: */15 * * * * (every 15 minutes) +# Schedule: */15 * * * * (every 15 minutes via User Scripts plugin) # -# Execution order: +# ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── # 1. Critical-Data rsync — auth stack, NPM config, certs (containers stopped both sides) # 2. emby-failover rsync — dirty Emby sync (watch states, library — Emby stays running) -# 3. partnership --check — read both state files, detect changes, act accordingly +# 3. CRITICAL_MAINTENANCE_SCRIPTS — any scripts configured for critical window +# 4. partnership --check — read both state files, detect changes, act accordingly # -# Why every 15 minutes: +# ── WHY EVERY 15 MINUTES ────────────────────────────────────────────────────────────────────── # Auth stack changes (new users, proxy rules, certs) propagate within 15min ✅ # Emby watch states stay in sync — mirror users see correct playback position ✅ # Partnership state changes detected and acted on quickly ✅ +# Lock prevents: daily rsync doing Critical-Data mid-critical window ✅ # -# Rsync gate: -# RSYNC_ENABLED=false → skips all syncs (global gate) -# CRITICAL_RSYNC_ENABLED=false → skips critical syncs (per-orchestrator) -# partnership --check still runs regardless of rsync gate -# (state check doesn't need rsync to work) +# ── RSYNC GATE ──────────────────────────────────────────────────────────────────────────────── +# RSYNC_ENABLED=false → skips all syncs (global gate) +# CRITICAL_RSYNC_ENABLED=false → skips critical syncs only (per-orchestrator gate) +# partnership --check always runs regardless — state check doesn't need rsync # -# Lock behavior: -# acquire_lock "strict" — if previous 15min run still going, skip this cycle +# ── LOCK BEHAVIOUR ──────────────────────────────────────────────────────────────────────────── +# acquire_lock "strict" — if previous 15min run still going, skip this cycle entirely # Critical-Data taking > 15min is a problem worth knowing about -# Lock prevents pile-up ✅ +# Strict mode prevents pile-up without waiting — log and move on ✅ # -# Configuration in Master.conf: -# CRITICAL_RSYNC_ENABLED — enable/disable rsync section -# PARTNERSHIP_ENABLED — enable/disable partnership check -# CRITICAL_SYNC_SHARES — shares synced every 15min -# ----------------------------------------------------------------------------------------------- +# ── SILENT WHEN HEALTHY ─────────────────────────────────────────────────────────────────────── +# Runs 96 times per day — clean runs must produce zero output ✅ +# Only failures and notable events produce visible output +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# CRITICAL_RSYNC_ENABLED — enable/disable rsync section +# CRITICAL_SYNC_SHARES — shares synced every 15min (HOST*_CRITICAL_SYNC_SHARES) +# CRITICAL_MAINTENANCE_SCRIPTS — scripts run in critical window (optional) +# PARTNERSHIP_ENABLED — enable/disable partnership check +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# critical_sync_maintenance.sh — normal run +# critical_sync_maintenance.sh --dry-run — preview syncs without transferring +# critical_sync_maintenance.sh --log — verbose per-share output +# critical_sync_maintenance.sh --status — show configuration and exit +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + acquire_lock "strict" detect_hosts resolve_remote_ip +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)" + echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}" + echo "$ICON_SYNC Critical enabled: ${CRITICAL_RSYNC_ENABLED:-false}" + echo "$ICON_SHIELD Partnership: ${PARTNERSHIP_ENABLED:-false}" + echo "" + echo "━━━ Critical Sync Shares ━━━" + if [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then + warn " No CRITICAL_SYNC_SHARES configured" + else + for share in "${CRITICAL_SYNC_SHARES[@]:-}"; do + [[ -z "$share" ]] && continue + SHARE_PATH="${share%%|*}" + SHARE_PROFILE="${share##*|}" + SHARE_NAME=$(basename "$SHARE_PATH") + [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]] && \ + echo " $ICON_SYNC $SHARE_NAME — no profile" || \ + echo " $ICON_SYNC $SHARE_NAME — profile: $SHARE_PROFILE" + done + fi + echo "" + echo "━━━ Critical Maintenance Scripts ━━━" + if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then + echo " None configured" + else + for entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]:-}"; do + [[ -z "$entry" || "$entry" == \#* ]] && continue + echo " $ICON_GEAR $(basename "${entry%% *}")" + done + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Critical Shares Sync ━━━ +# ============================================================================================== START=$(date +%s) RSYNC_OK=false PASS=() FAIL=() -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SYNC Critical Shares Sync ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_SYNC Critical Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━" - if ! check_rsync_enabled "CRITICAL"; then - warn "Critical rsync disabled — skipping sync, running partnership check only" + log "Critical rsync disabled — skipping sync, running partnership check only" +elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then + warn "CRITICAL_RSYNC_ENABLED=true but CRITICAL_SYNC_SHARES is empty for $MY_ID" + warn "Check HOST*_CRITICAL_SYNC_SHARES in master_host*.conf" else + log "Critical sync — $MY_ID → $REMOTE_ID — $(date '+%H:%M:%S')" + + # Build dry-run flag to pass through + RSYNC_DRY="" + [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run" + for share in "${CRITICAL_SYNC_SHARES[@]:-}"; do [[ -z "$share" ]] && continue @@ -72,101 +139,94 @@ else SHARE_PROFILE="${share##*|}" SHARE_NAME=$(basename "$SHARE_PATH") - echo "" - echo "━━━ $ICON_SYNC $SHARE_NAME ━━━" - SHARE_START=$(date +%s) if [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]]; then - # No profile specified - bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" + bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" $RSYNC_DRY else - bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" --profile="$SHARE_PROFILE" + bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" \ + --profile="$SHARE_PROFILE" $RSYNC_DRY fi RSYNC_EXIT=$? - SHARE_END=$(date +%s) - SHARE_DUR=$(format_duration $(( SHARE_END - SHARE_START ))) + SHARE_DUR=$(format_duration $(( $(date +%s) - SHARE_START ))) if [[ "$RSYNC_EXIT" -eq 0 ]]; then PASS+=("$SHARE_NAME") - success "$SHARE_NAME — done in $SHARE_DUR ✅" + log "$SHARE_NAME — done in $SHARE_DUR ✅" RSYNC_OK=true else FAIL+=("$SHARE_NAME") - error "$SHARE_NAME — failed after $SHARE_DUR" + error "$SHARE_NAME — failed after $SHARE_DUR (exit $RSYNC_EXIT)" fi done fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_CLEAN Critical Maintenance Scripts ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_CLEAN Critical Maintenance — $(date '+%Y-%m-%d %H:%M:%S') ━━━" - -if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then - log "No CRITICAL_MAINTENANCE_SCRIPTS defined — skipping" -else +# ============================================================================================== +# ━━━ Critical Maintenance Scripts ━━━ +# ============================================================================================== +if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do - [[ -z "$script_entry" ]] && continue - # Strip leading comment lines - [[ "$script_entry" == \#* ]] && continue + [[ -z "$script_entry" || "$script_entry" == \#* ]] && continue SCRIPT_PATH="$SCRIPT_DIR/../${script_entry%% *}" SCRIPT_ARGS="${script_entry#* }" [[ "$SCRIPT_ARGS" == "$script_entry" ]] && SCRIPT_ARGS="" + [[ "$DRY_RUN" == true ]] && SCRIPT_ARGS="$SCRIPT_ARGS --dry-run" SCRIPT_NAME=$(basename "$SCRIPT_PATH") - echo " → $SCRIPT_NAME" if [[ ! -f "$SCRIPT_PATH" ]]; then warn "$SCRIPT_NAME not found at $SCRIPT_PATH — skipping" continue fi + log "Running: $SCRIPT_NAME" bash "$SCRIPT_PATH" $SCRIPT_ARGS EXIT_CODE=$? - if [[ "$EXIT_CODE" -ne 0 ]]; then + [[ "$EXIT_CODE" -ne 0 ]] && \ warn "$SCRIPT_NAME exited with code $EXIT_CODE" - fi done fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SHIELD Partnership Check ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_SHIELD Partnership Check ━━━" +# ============================================================================================== +# ━━━ Partnership Check ━━━ +# ============================================================================================== +if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then + PARTNER_DRY="" + [[ "$DRY_RUN" == true ]] && PARTNER_DRY="--dry-run" -if [[ "${PARTNERSHIP_ENABLED:-false}" == false ]]; then - log "Partnership disabled — skipping check" -else - # Pass rsync outcome to --check so it can update last_seen_remote if [[ "$RSYNC_OK" == true ]]; then - bash "$SCRIPT_DIR/partnership_manage.sh" --check --remote-seen + bash "$SCRIPT_DIR/partnership_manage.sh" \ + --check --remote-seen $PARTNER_DRY else - bash "$SCRIPT_DIR/partnership_manage.sh" --check --remote-unseen + bash "$SCRIPT_DIR/partnership_manage.sh" \ + --check --remote-unseen $PARTNER_DRY fi +else + log "Partnership disabled — skipping check" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== END=$(date +%s) +DURATION=$(format_duration $(( END - START ))) -echo "" -echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━" -echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" - -if [[ ${#PASS[@]} -gt 0 ]]; then - echo "$ICON_SUCCESS Synced: ${PASS[*]}" -fi +# Silent when healthy — only show summary if there were failures or notable events if [[ ${#FAIL[@]} -gt 0 ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_TIME Duration: $DURATION" + [[ ${#PASS[@]} -gt 0 ]] && log "Synced: ${PASS[*]}" echo "$ICON_ERROR Failed: ${FAIL[*]}" - notify "Critical sync failed on $(hostname) — ${FAIL[*]}" "Critical Sync" "warning" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \ + "Critical Sync" "warning" + exit 1 +else + log "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s)" fi -if [[ ${#PASS[@]} -eq 0 ]] && [[ ${#FAIL[@]} -eq 0 ]]; then - echo "$ICON_SKIP Rsync: disabled" -fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file + +exit 0 \ No newline at end of file diff --git a/Orchestrators/daily_sync_maintenance.sh b/Orchestrators/daily_sync_maintenance.sh index 7de06d9..f4be56d 100644 --- a/Orchestrators/daily_sync_maintenance.sh +++ b/Orchestrators/daily_sync_maintenance.sh @@ -1,81 +1,139 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Daily Sync Maintenance ------------------------------------ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ============================= Daily Sync Maintenance ========================================= +# ============================================================================================== # Daily orchestrator — runs the full daily maintenance window in the correct order. -# Schedule: 0 1 * * * (1am daily) +# Schedule: 0 1 * * * (1am daily via User Scripts plugin) # -# Execution order: -# 1. git_pull_execute.sh — pull latest scripts first, always -# 2. Media share sync — HOST*_DAILY_SYNC_SHARES pushed to remote -# 3. HOST*_PERSONAL_SHARES — personal encrypted shares after media -# 4. media_shares_permissions.sh — fix ownership before arr cleanup -# 5. media_cleaner.sh anime — remove junk from anime shares -# 6. media_cleaner.sh media — remove junk from media shares -# 7. lidarr_cleanup.sh — remove orphaned music files -# 8. sonarr_cleanup.sh — remove orphaned TV files -# 9. radarr_cleanup.sh — remove orphaned movie files -# 10. docker_daily_restart.sh — restart containers that need daily restart +# ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── +# Pre-sync: +# git_pull_execute.sh — pull latest scripts first, always # -# What triggers weekly_health_digest.sh: -# NOT this script — weekly_health_digest.sh runs on its own schedule (Saturday) -# This script writes no stats — it just syncs and maintains +# Rsync window (DAILY_SYNC_SHARES per host): +# HOST*_DAILY_SYNC_SHARES — media shares pushed to mirror +# HOST*_PERSONAL_SHARES — encrypted personal shares # -# Configuration in Master.conf: -# DAILY_MAINTENANCE_SCRIPTS — pre/post-sync scripts (git pull, docker restart) -# DAILY_MAINTENANCE_SCRIPTS — media maintenance jobs run after sync -# HOST1_DAILY_SYNC_SHARES — shares HOST1 pushes to HOST2 -# HOST2_DAILY_SYNC_SHARES — shares HOST2 pushes to HOST1 -# HOST1/2_PERSONAL_SHARES — encrypted personal shares +# Post-sync maintenance (DAILY_MAINTENANCE_SCRIPTS): +# media_shares_permissions.sh — fix ownership before arr cleanup +# media_cleaner.sh anime — remove junk from anime shares +# media_cleaner.sh media — remove junk from media shares +# lidarr_cleanup.sh — remove orphaned music files +# sonarr_cleanup.sh — remove orphaned TV files +# radarr_cleanup.sh — remove orphaned movie files +# docker_daily_restart.sh — restart containers needing daily restart # +# ── WHY ORDER MATTERS ───────────────────────────────────────────────────────────────────────── +# git pull first — maintenance runs on latest code, not yesterday's +# Rsync before cleanup — cleanup sees the post-sync state +# Permissions before arr cleanup — arrs need correct ownership to delete/rename +# Arr cleanup after permissions — clean ownership = successful orphan deletion +# Docker restart last — containers already processed by cleanup +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── # Bidirectional — same script runs on both servers, correct direction automatic. -# Per-share rsync handled by rsync.sh — this script tracks pass/fail and total time. -# ----------------------------------------------------------------------------------------------- +# detect_hosts() aliases DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars. +# No manual HOST1/HOST2 comparisons — MY_ID routes correctly on any server. +# +# ── DRIVE TEMP HANDLING ─────────────────────────────────────────────────────────────────────── +# rsync.sh returns exit codes for temperature issues: +# exit 1 = temp WARN — skip this share, continue to next +# exit 2 = temp CRITICAL — abort ALL remaining syncs in this window +# All other failures — skip share, continue to next +# +# ── SILENT WHEN HEALTHY ─────────────────────────────────────────────────────────────────────── +# Runs daily at 1am — clean run should produce minimal output. +# Each job reports log() on success (silent), warn()/error() on failure (visible). +# Summary always shown — gives window timing and share/job counts. +# Notify only on failure — successful daily maintenance doesn't need notification. +# +# ── CONFIGURATION (master.conf + master_host*.conf) ─────────────────────────────────────────── +# HOST*_DAILY_SYNC_SHARES — shares pushed to mirror each day +# HOST*_PERSONAL_SHARES — encrypted personal shares +# DAILY_MAINTENANCE_SCRIPTS — maintenance jobs (permissions, cleanup, restart) +# DAILY_RSYNC_ENABLED — enable/disable rsync section +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# daily_sync_maintenance.sh — normal run +# daily_sync_maintenance.sh --dry-run — preview without syncing or changing +# daily_sync_maintenance.sh --log — verbose per-share/per-job output +# daily_sync_maintenance.sh --status — show configured shares and jobs +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh" SCRIPTS_ROOT="$SCRIPT_DIR/.." -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" +parse_args "$@" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" detect_hosts resolve_remote_ip acquire_lock -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SHIELD Pre-flight Checks ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" -check_connectivity -check_remote_rootfs +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY DAILY SYNC STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)" + echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}" + echo "$ICON_SYNC Daily enabled: ${DAILY_RSYNC_ENABLED:-false}" + echo "" + echo "━━━ Daily Sync Shares ━━━" + for share in "${DAILY_SYNC_SHARES[@]:-}"; do + [[ -n "$share" ]] && echo " $ICON_SYNC $share" + done + for share in "${PERSONAL_SHARES[@]:-}"; do + [[ -n "$share" ]] && echo " $ICON_SYNC $share (personal)" + done + echo "" + echo "━━━ Daily Maintenance Scripts ━━━" + for entry in "${DAILY_MAINTENANCE_SCRIPTS[@]:-}"; do + [[ -z "$entry" ]] && continue + echo " $ICON_GEAR $(basename "${entry%% *}") ${entry#* }" + done + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi -WINDOW_START=$(date +%s) -JOB_PASS=() -JOB_FAIL=() - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GIT Pre-sync Jobs ━━━ -# git_pull_execute.sh runs first — pulls latest scripts before anything else runs -# Identified by script name — all other DAILY_MAINTENANCE_SCRIPTS run after sync -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_SYNC Pre-sync Jobs ━━━" +# ============================================================================================== +# ── BUILD SHARE LIST via detect_hosts aliases ───────────────────────────────────────────────── +# detect_hosts() sets DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars +# No manual HOST1/HOST2 comparison needed — aliased automatically per server +# ============================================================================================== +ALL_SHARES=() +for share in "${DAILY_SYNC_SHARES[@]:-}"; do + [[ -n "$share" ]] && ALL_SHARES+=("$share") +done +for share in "${PERSONAL_SHARES[@]:-}"; do + [[ -n "$share" ]] && ALL_SHARES+=("$share") +done +SHARE_COUNT=${#ALL_SHARES[@]} +# ── Split maintenance scripts: git pull runs pre-sync, rest run post-sync ───────────────────── PRE_SYNC_SCRIPTS=() POST_SYNC_SCRIPTS=() - -for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do +for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]:-}"; do [[ -z "$script_entry" ]] && continue script_name=$(basename "${script_entry%% *}") if [[ "$script_name" == "git_pull_execute.sh" ]]; then @@ -85,236 +143,181 @@ for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do fi done -for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do - [[ -z "$script_entry" ]] && continue - script_args=($script_entry) - script_path="$SCRIPTS_ROOT/${script_args[0]}" - script_name=$(basename "${script_args[0]}") - extra_args=("${script_args[@]:1}") +# Helper — run a maintenance script, track pass/fail +run_job() { + local script_entry="$1" + local extra_dry="" + [[ "$DRY_RUN" == true ]] && extra_dry="--dry-run" - echo "" - info "$ICON_START Running: $script_name" + read -r -a script_args <<< "$script_entry" + local script_path="$SCRIPTS_ROOT/${script_args[0]}" + local script_name + script_name=$(basename "${script_args[0]}") + local extra_args=("${script_args[@]:1}") if [[ ! -f "$script_path" ]]; then error "$script_name — not found at $script_path" JOB_FAIL+=("$script_name") - continue + return 1 fi - if bash "$script_path" "${extra_args[@]}"; then - success "$script_name — done" - JOB_PASS+=("$script_name") + log "Running: $script_name ${extra_args[*]}" + # shellcheck disable=SC2086 + if bash "$script_path" "${extra_args[@]}" $extra_dry; then + log "$script_name — done ✅" + JOB_PASS+=("$script_name ${extra_args[*]}") else - error "$script_name — failed" - JOB_FAIL+=("$script_name") + error "$script_name — failed (exit $?)" + JOB_FAIL+=("$script_name ${extra_args[*]}") fi -done +} -# ----------------------------------------------------------------------------------------------- -# Build share list — host-specific truth shares + personal shares -# ----------------------------------------------------------------------------------------------- +WINDOW_START=$(date +%s) +JOB_PASS=() +JOB_FAIL=() PASS=() FAIL=() SHARE_TIMES=() -TOTAL_START=$(date +%s) -ALL_SHARES=() +echo "" +echo "━━━ $ICON_GEAR Daily Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - for share in "${HOST1_DAILY_SYNC_SHARES[@]}"; do - [[ -n "$share" ]] && ALL_SHARES+=("$share") - done - for share in "${HOST1_PERSONAL_SHARES[@]}"; do - [[ -n "$share" ]] && ALL_SHARES+=("$share") - done -elif [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]]; then - for share in "${HOST2_DAILY_SYNC_SHARES[@]}"; do - [[ -n "$share" ]] && ALL_SHARES+=("$share") - done - for share in "${HOST2_PERSONAL_SHARES[@]}"; do - [[ -n "$share" ]] && ALL_SHARES+=("$share") +# ============================================================================================== +# ━━━ Pre-sync — git pull ━━━ +# ============================================================================================== +if [[ ${#PRE_SYNC_SCRIPTS[@]} -gt 0 ]]; then + echo "" + echo "━━━ $ICON_GIT Pre-sync ━━━" + for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do + run_job "$script_entry" done fi -SHARE_COUNT=${#ALL_SHARES[@]} - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SYNC Media Share Sync ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_SYNC Media Share Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -echo "$ICON_SUMMARY Shares: $SHARE_COUNT" +# ============================================================================================== +# ━━━ Media Share Sync ━━━ +# ============================================================================================== echo "" +echo "━━━ $ICON_SYNC Media Share Sync — $SHARE_COUNT share(s) ━━━" +TOTAL_START=$(date +%s) SHARE_INDEX=0 ABORT_ALL_SYNCS=false -# Tier 1 + Tier 2 rsync gate check if ! check_rsync_enabled "DAILY"; then - warn "Rsync disabled — skipping all $SHARE_COUNT share syncs" - warn "Proceeding to media management jobs..." + warn "Daily rsync disabled — skipping all $SHARE_COUNT share syncs" + warn "Proceeding to maintenance jobs..." +elif [[ "$SHARE_COUNT" -eq 0 ]]; then + warn "No shares configured for $MY_ID — check HOST*_DAILY_SYNC_SHARES in master_host*.conf" else + # Pre-flight — connectivity then remote rootfs + check_connectivity + check_remote_rootfs + + RSYNC_DRY="" + [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run" + for SHARE in "${ALL_SHARES[@]}"; do - SHARE_INDEX=$((SHARE_INDEX + 1)) - SHARE_NAME=$(basename "$SHARE") - SHARE_START=$(date +%s) - - echo "━━━ $ICON_SYNC Share $SHARE_INDEX of $SHARE_COUNT: $SHARE_NAME ━━━" - - if [[ "$ABORT_ALL_SYNCS" == true ]]; then - warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)" - FAIL+=("$SHARE_NAME:temp-critical") - echo "" - continue - fi - - bash "$RSYNC_SCRIPT" "$SHARE" - RSYNC_EXIT=$? - - SHARE_END=$(date +%s) - SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))") - - if [[ "$RSYNC_EXIT" -eq 0 ]]; then - PASS+=("$SHARE_NAME") - echo "$ICON_DONE $SHARE_NAME complete" - elif [[ "$RSYNC_EXIT" -eq 1 ]]; then - # Temp warning — skip this profile, continue to next - FAIL+=("$SHARE_NAME:temp-warn") - warn "$SHARE_NAME skipped — drive temps too high" - elif [[ "$RSYNC_EXIT" -eq 2 ]]; then - # Temp critical — abort all remaining syncs - FAIL+=("$SHARE_NAME:temp-critical") - ABORT_ALL_SYNCS=true - error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs" - notify "Daily sync aborted on $(hostname) — drive temps CRITICAL during $SHARE_NAME sync" "Daily Sync" "warning" - else - FAIL+=("$SHARE_NAME") - error "$SHARE_NAME failed — continuing to next share" - fi - - echo "" -done -fi # end check_rsync_enabled "DAILY" - -TOTAL_END=$(date +%s) -TOTAL_DURATION=$((TOTAL_END - TOTAL_START)) - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_CLEAN Post-sync Media Jobs ━━━ -# Reads DAILY_MAINTENANCE_SCRIPTS from Master.conf — permissions, cleaners, arr cleanup -# Runs after sync completes — correct ownership available, clean folders guaranteed -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_CLEAN Post-sync Media Jobs ━━━" - -if [[ ${#DAILY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then - for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do - [[ -z "$script_entry" ]] && continue - script_args=($script_entry) - script_path="$SCRIPTS_ROOT/${script_args[0]}" - script_name=$(basename "${script_args[0]}") - extra_args=("${script_args[@]:1}") + (( SHARE_INDEX++ )) + SHARE_NAME=$(basename "$SHARE") + SHARE_START=$(date +%s) echo "" - info "$ICON_START Running: $script_name ${extra_args[*]}" + echo "━━━ $ICON_SYNC Share $SHARE_INDEX/$SHARE_COUNT: $SHARE_NAME ━━━" - if [[ ! -f "$script_path" ]]; then - error "$script_name — not found at $script_path" - JOB_FAIL+=("$script_name ${extra_args[*]}") + if [[ "$ABORT_ALL_SYNCS" == true ]]; then + warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)" + FAIL+=("$SHARE_NAME:temp-critical") continue fi - if [[ "$DRY_RUN" == true ]]; then - if bash "$script_path" "${extra_args[@]}" --dry-run; then - success "$script_name — done (dry run)" - JOB_PASS+=("$script_name ${extra_args[*]}") - else - error "$script_name — failed" - JOB_FAIL+=("$script_name ${extra_args[*]}") - fi - else - if bash "$script_path" "${extra_args[@]}"; then - success "$script_name — done" - JOB_PASS+=("$script_name ${extra_args[*]}") - else - error "$script_name — failed" - JOB_FAIL+=("$script_name ${extra_args[*]}") - fi - fi + bash "$RSYNC_SCRIPT" "$SHARE" $RSYNC_DRY + RSYNC_EXIT=$? + + SHARE_TIMES+=("$SHARE_NAME:$(( $(date +%s) - SHARE_START ))") + + case "$RSYNC_EXIT" in + 0) + PASS+=("$SHARE_NAME") + log "$SHARE_NAME — done ✅" + ;; + 1) + FAIL+=("$SHARE_NAME:temp-warn") + warn "$SHARE_NAME skipped — drive temps too high" + ;; + 2) + FAIL+=("$SHARE_NAME:temp-critical") + ABORT_ALL_SYNCS=true + error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs" + notify "Daily sync aborted on $(hostname) ($MY_ID) — drive temps CRITICAL during $SHARE_NAME" \ + "Daily Sync" "warning" + ;; + *) + FAIL+=("$SHARE_NAME") + error "$SHARE_NAME failed (exit $RSYNC_EXIT) — continuing to next share" + ;; + esac + done fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Post-sync System Jobs ━━━ -# Reads remaining DAILY_MAINTENANCE_SCRIPTS — docker restart etc. -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Post-sync Jobs ━━━" - -for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do - [[ -z "$script_entry" ]] && continue - script_args=($script_entry) - script_path="$SCRIPTS_ROOT/${script_args[0]}" - script_name=$(basename "${script_args[0]}") - extra_args=("${script_args[@]:1}") +TOTAL_END=$(date +%s) +# ============================================================================================== +# ━━━ Post-sync Maintenance Jobs ━━━ +# ============================================================================================== +if [[ ${#POST_SYNC_SCRIPTS[@]} -gt 0 ]]; then echo "" - info "$ICON_START Running: $script_name" - - if [[ ! -f "$script_path" ]]; then - error "$script_name — not found at $script_path" - JOB_FAIL+=("$script_name") - continue - fi - - if bash "$script_path" "${extra_args[@]}"; then - success "$script_name — done" - JOB_PASS+=("$script_name") - else - error "$script_name — failed" - JOB_FAIL+=("$script_name") - fi -done + echo "━━━ $ICON_CLEAN Post-sync Maintenance ━━━" + for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do + echo "" + run_job "$script_entry" + done +fi WINDOW_END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" -echo "━━━━━ $ICON_SUMMARY DAILY SYNC MAINTENANCE SUMMARY ━━━━━" -echo "$ICON_TIME Window: $(date -d @$WINDOW_START '+%Y-%m-%d %H:%M:%S') → $(date -d @$WINDOW_END '+%H:%M:%S')" -echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))" +echo "━━━━━ $ICON_SUMMARY DAILY MAINTENANCE SUMMARY ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')" +echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))" echo "" -echo "$ICON_SYNC Media shares:" +echo "$ICON_SYNC Shares ($SHARE_COUNT):" for entry in "${SHARE_TIMES[@]}"; do - SHARE_NAME="${entry%%:*}" - DURATION="${entry##*:}" - if printf '%s\n' "${FAIL[@]}" | grep -qx "$SHARE_NAME"; then - echo " $ICON_ERROR $SHARE_NAME — $(format_duration $DURATION)" + sname="${entry%%:*}" + sdur="${entry##*:}" + if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then + echo " $ICON_ERROR $sname — $(format_duration "$sdur")" else - echo " $ICON_DONE $SHARE_NAME — $(format_duration $DURATION)" + echo " $ICON_DONE $sname — $(format_duration "$sdur")" fi done -echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT" +[[ "$SHARE_COUNT" -eq 0 || "${DAILY_RSYNC_ENABLED:-false}" == "false" ]] && \ + echo " (rsync disabled)" +echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT" echo "" if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then - echo "$ICON_GEAR Jobs (media + system):" - for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done - for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done + echo "$ICON_GEAR Jobs:" + for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done + for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done echo "" fi TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} )) if [[ "$TOTAL_FAIL" -gt 0 ]]; then - echo "$ICON_WARN Status: $TOTAL_FAIL failure(s) — check logs" - notify "Daily sync maintenance completed with failures on $(hostname) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" "Daily Maintenance" "warning" + warn "Status: $TOTAL_FAIL failure(s)" + notify "Daily maintenance completed with failures on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \ + "Daily Maintenance" "warning" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" exit 1 else - echo "$ICON_DONE Status: $ICON_SUCCESS ALL COMPLETE" - notify "Daily sync maintenance complete on $(hostname) — ${#PASS[@]} shares synced, ${#JOB_PASS[@]} jobs run in $(format_duration $(( WINDOW_END - WINDOW_START )))" "Daily Maintenance" "normal" + log "$ICON_DONE Status: all complete — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi \ No newline at end of file diff --git a/Orchestrators/sunday_morning_coffee_report.sh b/Orchestrators/sunday_morning_coffee_report.sh index a259cf6..a9ff89b 100644 --- a/Orchestrators/sunday_morning_coffee_report.sh +++ b/Orchestrators/sunday_morning_coffee_report.sh @@ -1,84 +1,158 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# ----------------------------- Sunday Morning Coffee Report ----------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ============================= Sunday Morning Coffee Report =================================== +# ============================================================================================== # Weekly system overview — everything that happened this week in one clean read. # Designed to be read over coffee Sunday morning while the system is fully caught up # from the 2:30am maintenance window. # Schedule: 0 7 * * 0 (7am Sunday — after weekly_sync_maintenance.sh finishes at ~3am) # Maintenance window completes → 4 hours of fresh data → report ready ☕ # -# Sections: -# 🖥️ System — uptime, memory, boot drive, cache drive, reboots -# 📀 Array — disk count, parity status, ZFS health, drive temps -# 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split -# 🎵 Media Activity — arr cleanup stats, arr recovery stats, library health -# 🌐 Rsync — weekly transfer totals, per-share breakdown -# 🛡️ Watchdog — system watchdog, docker watchdog, failover (all continuous loops) -# strikes, skip list, restarts, system snapshot, container overview -# 🔐 Security — SSL cert expiry per domain -# 📊 Emby — weekly stream count, top users, top content -# ⚙️ Health — SMART summary, docker container count, Gitea sync status -# ⚠️ Issues — anything requiring attention collected above +# ── SECTIONS ────────────────────────────────────────────────────────────────────────────────── +# 🖥️ System — uptime, memory, boot drive, cache drive, reboots this week +# 📀 Array — disk count, parity status, ZFS health, drive temps +# 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split +# 🎵 Media Activity — arr cleanup stats, arr recovery stats, queue depth +# 🌐 Rsync — weekly transfer totals, per-share breakdown, failures +# 🛡️ Watchdog — system watchdog, docker watchdog, failover state +# 🔐 Security — SSL cert expiry per domain +# 📊 Emby — weekly stream count, active now, top users +# ⚙️ System Health — SMART summary, inotify, php-fpm, Docker, Gitea sync +# ⚠️ Issues — anything requiring attention collected from above sections # -# Data sources (reads only — no writes except the notification): -# DATA_DIR stats files — arr cleanup, recovery, transcode, bandwidth history -# /boot/config — failover state, watchdog reboot log -# /tmp — watchdog strike state files -# /proc, /sys — system memory, uptime -# /var/local/emhttp/ — unRAID array info -# Emby API — session history -# Arr APIs — current queue depth -# tailscale — network status -# openssl — live SSL cert check -# smartctl — drive health +# ── DATA SOURCES (reads only) ───────────────────────────────────────────────────────────────── +# DATA_DIR stats files — arr cleanup, recovery, transcode, bandwidth history +# /boot/config — failover state, watchdog reboot log +# /tmp — watchdog strike state files +# /proc, /sys — system memory, uptime, inotify +# /var/local/emhttp/ — unRAID array info +# Emby API — session history, active streams +# Arr APIs — current queue depth (SONARR_URL etc. from detect_hosts()) +# tailscale — remote server reachability +# openssl — live SSL cert check per domain +# smartctl — drive SMART health # -# All configuration in Master.conf. -# Supports --dry-run to preview report without sending notification. -# ----------------------------------------------------------------------------------------------- +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID and aliases all HOST*_ vars. +# Report header and footer show MY_ID — clear which server's weekly report this is. +# No manual HOST1/HOST2 comparisons — all via MY_ID/REMOTE_ID. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# Root check — smartctl, docker, openssl need root +# acquire_lock — prevents duplicate reports +# detect_hosts() — correct vars per server +# DOCKER_TIMEOUT — all docker calls protected +# validate_unraid_cmd — notify validated before use +# openssl check — security section skipped gracefully if not available +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# NOTIFY_UNRAID — send via unRAID notification system +# DISCORD_WEBHOOK — send to Discord channel (MY_ID_ prefixed per host) +# CERT_MONITOR_DOMAINS / CERT_WARN_DAYS / CERT_CRIT_DAYS +# ZFS_REPORT_IGNORE_POOLS / SMART_IGNORE_DRIVES +# All threshold vars read from master.conf at runtime +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# sunday_morning_coffee_report.sh — generate and send report +# sunday_morning_coffee_report.sh --dry-run — generate without sending notification +# sunday_morning_coffee_report.sh --status — show data file availability +# sunday_morning_coffee_report.sh --log — verbose section output +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- +DOCKER_TIMEOUT=15 + +# ============================================================================================== # ━━━ Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root — smartctl and docker require root" + exit 1 +fi + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — unRAID notifications disabled" + acquire_lock detect_hosts REPORT_DATE=$(date '+%A, %B %-d, %Y') WEEK_START=$(date -d "7 days ago" '+%Y-%m-%d') +WEEK_EPOCH=$(date -d "$WEEK_START" +%s) TODAY=$(date '+%Y-%m-%d') NOW=$(date +%s) -REPORT=() # all report lines -ISSUES=() # items needing attention -FINDINGS=() # notable but not critical +REPORT=() +ISSUES=() +FINDINGS=() +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY COFFEE REPORT STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "" + echo "━━━ Data Files ━━━" + for f in \ + "$ARR_CLEANUP_STATS:arr cleanup stats" \ + "$ARR_RECOVERY_STATS:arr recovery stats" \ + "$TRANSCODE_DAILY_LOG:transcode daily log" \ + "$BANDWIDTH_LOG:bandwidth log" \ + "$TUNING_MONITOR_LOG:tuning monitor log" \ + "$SYS_WATCHDOG_STATE_FILE:system watchdog state" \ + "$SYS_WATCHDOG_REBOOT_LOG:watchdog reboot log" \ + "$WATCHDOG_STATE_FILE:docker watchdog state" \ + "$WATCHDOG_CONTAINER_RESTART_LOG:container restart log" \ + "$FAILOVER_STATE_FILE:failover state"; do + path="${f%%:*}" + label="${f##*:}" + if [[ -f "$path" ]] && [[ -s "$path" ]]; then + COUNT=$(wc -l < "$path" 2>/dev/null || echo "?") + echo " $ICON_SUCCESS $label ($COUNT lines)" + elif [[ -f "$path" ]]; then + echo " $ICON_WARN $label (exists but empty)" + else + echo " $ICON_SKIP $label (not found)" + fi + done + echo "" + echo "━━━ Runtime Dependencies ━━━" + command -v smartctl >/dev/null 2>&1 && echo " $ICON_SUCCESS smartctl" || \ + echo " $ICON_SKIP smartctl (not installed)" + command -v openssl >/dev/null 2>&1 && echo " $ICON_SUCCESS openssl" || \ + echo " $ICON_SKIP openssl (security section disabled)" + command -v docker >/dev/null 2>&1 && echo " $ICON_SUCCESS docker" || \ + echo " $ICON_SKIP docker" + command -v zpool >/dev/null 2>&1 && echo " $ICON_SUCCESS zpool" || \ + echo " $ICON_SKIP zpool (ZFS section disabled)" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — report generated, notification not sent" + +# ============================================================================================== +# ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== section() { REPORT+=("") REPORT+=("$1") REPORT+=("$(printf '%.0s─' {1..50})") } -line() { - REPORT+=(" $1") -} - -issue() { - ISSUES+=("$1") - REPORT+=(" ⚠️ $1") -} - -finding() { - FINDINGS+=("$1") - REPORT+=(" ℹ️ $1") -} +line() { REPORT+=(" $1"); } +issue() { ISSUES+=("$1"); REPORT+=(" ⚠️ $1"); } +finding() { FINDINGS+=("$1"); REPORT+=(" ℹ️ $1"); } format_bytes() { local bytes=$1 @@ -91,76 +165,83 @@ format_bytes() { fi } -# ----------------------------------------------------------------------------------------------- -# ━━━ 🖥️ System ━━━ -# ----------------------------------------------------------------------------------------------- -section "🖥️ SYSTEM" +# Lock/uptime helpers — local copies needed in report context +_get_lock_pid() { + local f="$LOCK_DIR/${1}.lock" + [[ -f "$f" ]] && { local c; c=$(cat "$f" 2>/dev/null); echo "${c%%:*}"; } +} +_get_lock_name() { + local f="$LOCK_DIR/${1}.lock" + [[ -f "$f" ]] && { local c; c=$(cat "$f" 2>/dev/null); echo "${c##*:}"; } +} +_is_running() { + local pid name + pid=$(_get_lock_pid "$1"); name=$(_get_lock_name "$1") + [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$name" == "$1" ]] +} +_lock_age() { + local f="$LOCK_DIR/${1}.lock" + [[ -f "$f" ]] && echo $(( NOW - $(stat -c %Y "$f" 2>/dev/null || echo "$NOW") )) || echo 0 +} +_fmt_uptime() { + local s=$1 d=$(($1/86400)) h=$((($1%86400)/3600)) m=$((($1%3600)/60)) + (( d > 0 )) && echo "${d}d ${h}h ${m}m" || \ + (( h > 0 )) && echo "${h}h ${m}m" || echo "${m}m" +} + +# ============================================================================================== +# ━━━ 🖥️ SYSTEM ━━━ +# ============================================================================================== +section "🖥️ SYSTEM — $MY_ID ($LOCAL_SERVER_NAME)" -# Uptime UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime) UPTIME_DAYS=$(( UPTIME_SECONDS / 86400 )) UPTIME_HOURS=$(( (UPTIME_SECONDS % 86400) / 3600 )) BOOT_TIME=$(date -d "@$(( NOW - UPTIME_SECONDS ))" '+%A %-d %b at %-I:%M%p') line "Uptime: ${UPTIME_DAYS}d ${UPTIME_HOURS}hr (up since $BOOT_TIME)" -# Reboot history this week REBOOT_COUNT=0 if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then - WEEK_EPOCH=$(date -d "$WEEK_START" +%s) REBOOT_COUNT=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \ "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l) fi -if [[ "$REBOOT_COUNT" -gt 0 ]]; then - issue "Reboots this week: $REBOOT_COUNT (system watchdog triggered)" -else +[[ "${REBOOT_COUNT:-0}" -gt 0 ]] && \ + issue "Reboots this week: $REBOOT_COUNT (system watchdog triggered)" || \ line "Reboots this week: 0 ✅" -fi -# Memory -MEM_TOTAL_KB=$(awk '/MemTotal/ {print $2}' /proc/meminfo) -MEM_AVAIL_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) -MEM_USED_KB=$(( MEM_TOTAL_KB - MEM_AVAIL_KB )) +MEM_TOTAL_KB=$(awk '/MemTotal/ {print $2}' /proc/meminfo) +MEM_AVAIL_KB=$(awk '/MemAvailable/{print $2}' /proc/meminfo) +MEM_USED_GB=$(awk "BEGIN {printf \"%.1f\", ($MEM_TOTAL_KB - $MEM_AVAIL_KB) / 1048576}") MEM_TOTAL_GB=$(awk "BEGIN {printf \"%.0f\", $MEM_TOTAL_KB / 1048576}") -MEM_USED_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_USED_KB / 1048576}") -MEM_FREE_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_AVAIL_KB / 1048576}") - -ARC_SIZE=0 +ARC_SIZE="n/a" if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then - ARC_BYTES=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0) - ARC_SIZE=$(awk "BEGIN {printf \"%.1f\", $ARC_BYTES / 1073741824}") + ARC_B=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0) + ARC_SIZE=$(awk "BEGIN {printf \"%.1f\", $ARC_B / 1073741824}") fi line "Memory: ${MEM_USED_GB}GB used / ${MEM_TOTAL_GB}GB total (ARC: ${ARC_SIZE}GB)" -# Boot drive -BOOT_PCT=$(df /boot --output=pcent 2>/dev/null | tail -1 | tr -d ' %') -BOOT_USED=$(df /boot -h --output=used 2>/dev/null | tail -1 | tr -d ' ') -BOOT_SIZE=$(df /boot -h --output=size 2>/dev/null | tail -1 | tr -d ' ') -if [[ "${BOOT_PCT:-0}" -ge 80 ]]; then - issue "Boot drive: ${BOOT_PCT}% used (${BOOT_USED}/${BOOT_SIZE}) — getting full" -else - line "Boot drive: ${BOOT_PCT}% used (${BOOT_USED}/${BOOT_SIZE})" -fi +BOOT_PCT=$(df /boot --output=pcent 2>/dev/null | tail -1 | tr -d ' %') +BOOT_USED=$(df /boot -h --output=used 2>/dev/null | tail -1 | tr -d ' ') +BOOT_SIZE=$(df /boot -h --output=size 2>/dev/null | tail -1 | tr -d ' ') +[[ "${BOOT_PCT:-0}" -ge 80 ]] && \ + issue "Boot drive: ${BOOT_PCT}% used (${BOOT_USED}/${BOOT_SIZE}) — getting full" || \ + line "Boot drive: ${BOOT_PCT}% used (${BOOT_USED}/${BOOT_SIZE})" -# Cache drive -CACHE_PCT=$(df /mnt/cache --output=pcent 2>/dev/null | tail -1 | tr -d ' %') +CACHE_PCT=$(df /mnt/cache --output=pcent 2>/dev/null | tail -1 | tr -d ' %') CACHE_AVAIL=$(df /mnt/cache -h --output=avail 2>/dev/null | tail -1 | tr -d ' ') -CACHE_SIZE=$(df /mnt/cache -h --output=size 2>/dev/null | tail -1 | tr -d ' ') -if [[ "${CACHE_PCT:-0}" -ge 85 ]]; then - issue "Cache drive: ${CACHE_PCT}% used (${CACHE_AVAIL} free of ${CACHE_SIZE})" -else - line "Cache drive: ${CACHE_PCT}% used (${CACHE_AVAIL} free of ${CACHE_SIZE})" -fi +CACHE_SIZE=$(df /mnt/cache -h --output=size 2>/dev/null | tail -1 | tr -d ' ') +[[ "${CACHE_PCT:-0}" -ge 85 ]] && \ + issue "Cache drive: ${CACHE_PCT}% used (${CACHE_AVAIL} free of ${CACHE_SIZE})" || \ + line "Cache drive: ${CACHE_PCT}% used (${CACHE_AVAIL} free of ${CACHE_SIZE})" -# ----------------------------------------------------------------------------------------------- -# ━━━ 📀 Array ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ 📀 ARRAY ━━━ +# ============================================================================================== section "📀 ARRAY" -# unRAID array info — parse disks.ini correctly -# Format uses ["diskN"] ["parity"] ["parity2"] etc. if [[ -f /var/local/emhttp/disks.ini ]]; then DISK_COUNT=$(grep -c '^\["disk[0-9]' /var/local/emhttp/disks.ini 2>/dev/null || echo "?") - PARITY_COUNT=$(grep -c '^\["parity' /var/local/emhttp/disks.ini 2>/dev/null || echo "?") + PARITY_COUNT=$(grep -c '^\["parity' /var/local/emhttp/disks.ini 2>/dev/null || echo "?") DISK_COUNT="${DISK_COUNT//[^0-9]/}"; DISK_COUNT="${DISK_COUNT:-?}" PARITY_COUNT="${PARITY_COUNT//[^0-9]/}"; PARITY_COUNT="${PARITY_COUNT:-?}" line "Array: ${DISK_COUNT} data disks + ${PARITY_COUNT} parity" @@ -168,7 +249,6 @@ else line "Array: disks.ini not found" fi -# Parity status from parity-checks.log PARITY_LOG="/boot/config/parity-checks.log" if [[ -f "$PARITY_LOG" ]]; then LAST_CHECK=$(tail -1 "$PARITY_LOG" 2>/dev/null) @@ -179,28 +259,21 @@ if [[ -f "$PARITY_LOG" ]]; then PARITY_DURATION=$(echo "$LAST_CHECK" | cut -d'|' -f2) PARITY_DUR_HR=$(( PARITY_DURATION / 3600 )) PARITY_DUR_MIN=$(( (PARITY_DURATION % 3600) / 60 )) - - # Negative = sync corrections (not data errors), positive = real errors - if [[ "$PARITY_ERRORS" -gt 0 ]]; then + if [[ "${PARITY_ERRORS:-0}" -gt 0 ]]; then issue "Parity: $PARITY_ERRORS errors on last check ($PARITY_DATE)" - elif [[ "$PARITY_ERRORS" -lt 0 ]]; then + elif [[ "${PARITY_ERRORS:-0}" -lt 0 ]]; then CORRECTIONS=$(( PARITY_ERRORS * -1 )) line "Parity: OK — last: $PARITY_DATE ($PARITY_ACTION, ${PARITY_DUR_HR}h${PARITY_DUR_MIN}m, $CORRECTIONS correction(s)) ✅" else line "Parity: OK — last: $PARITY_DATE ($PARITY_ACTION, ${PARITY_DUR_HR}h${PARITY_DUR_MIN}m, 0 errors) ✅" fi - - # Check if currently running RESYNC=$(grep "^mdResync=" /var/local/emhttp/var.ini 2>/dev/null | cut -d= -f2 | tr -d '"') - if [[ "$RESYNC" != "0" ]]; then - finding "Parity check currently in progress" - fi + [[ "$RESYNC" != "0" ]] && finding "Parity check currently in progress" fi else line "Parity: no check log found" fi -# ZFS pool health — filter ignored pools if command -v zpool >/dev/null 2>&1; then while IFS=$'\t' read -r pool health; do [[ -z "$pool" ]] && continue @@ -209,19 +282,16 @@ if command -v zpool >/dev/null 2>&1; then [[ "$pool" == "$ignore" ]] && SKIP=true && break done [[ "$SKIP" == true ]] && continue - if [[ "$health" == "ONLINE" ]]; then - line "ZFS $pool: ONLINE ✅" - else + [[ "$health" == "ONLINE" ]] && \ + line "ZFS $pool: ONLINE ✅" || \ issue "ZFS $pool: $health — check immediately" - fi done < <(zpool list -H -o name,health 2>/dev/null) fi -# Drive temperatures from SMART — build lookup table first then display per disk if command -v smartctl >/dev/null 2>&1; then declare -A DISK_TEMPS - TEMP_WARN=false ALL_NORMAL=true + get_unraid_temp_thresholds for disk in /dev/sd? /dev/nvme?; do [[ ! -e "$disk" ]] && continue @@ -234,36 +304,33 @@ if command -v smartctl >/dev/null 2>&1; then TEMP=$(smartctl -A "$disk" 2>/dev/null | awk ' /^190 / || /^194 / { print $10; exit } - /Temperature_Celsius/ { print $10; exit } - /Airflow_Temperature/ { print $10; exit } + /Temperature_Celsius/ { print $10; exit } ') - if [[ -z "$TEMP" ]]; then - TEMP=$(smartctl -A "$disk" 2>/dev/null | \ - awk '/^Temperature:/ { print $2; exit }') - fi + [[ -z "$TEMP" ]] && TEMP=$(smartctl -A "$disk" 2>/dev/null | \ + awk '/^Temperature:/ { print $2; exit }') TEMP="${TEMP//[^0-9]/}" [[ -z "$TEMP" ]] && continue DISK_TEMPS["$DISK_NAME"]="$TEMP" - if [[ "$TEMP" -ge "${SMART_TEMP_CRIT:-55}" ]]; then + if is_ssd "$disk"; then + WARN_T="${UNRAID_SSD_HOT:-50}"; CRIT_T="${UNRAID_SSD_MAX:-60}" + else + WARN_T="${UNRAID_DISK_HOT:-45}"; CRIT_T="${UNRAID_DISK_MAX:-55}" + fi + + if [[ "$TEMP" -ge "$CRIT_T" ]]; then issue "Drive $DISK_NAME: ${TEMP}°C — CRITICAL" - TEMP_WARN=true ALL_NORMAL=false - elif [[ "$TEMP" -ge "${SMART_TEMP_WARN:-45}" ]]; then + elif [[ "$TEMP" -ge "$WARN_T" ]]; then finding "Drive $DISK_NAME: ${TEMP}°C — warm" - TEMP_WARN=true ALL_NORMAL=false fi done - if [[ "$ALL_NORMAL" == true ]]; then - line "Drive temps: all normal ✅" - fi + [[ "$ALL_NORMAL" == true ]] && line "Drive temps: all normal ✅" - # Show per-disk summary with device name from disks.ini + temp if [[ -f /var/local/emhttp/disks.ini ]] && [[ ${#DISK_TEMPS[@]} -gt 0 ]]; then - # Parse disk → device mapping from disks.ini DISK_SUMMARY="" CURRENT_DISK="" while IFS= read -r ini_line; do @@ -282,63 +349,50 @@ if command -v smartctl >/dev/null 2>&1; then fi fi -# ----------------------------------------------------------------------------------------------- -# ━━━ 🎬 Transcodes ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ 🎬 TRANSCODES ━━━ +# ============================================================================================== section "🎬 TRANSCODES" if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then - RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ') - RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}") - RAMDISK_AVAIL_KB=$(df "$RAMDISK_PATH" --output=avail | tail -1 | tr -d ' ') + RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ') + RAMDISK_AVAIL_KB=$(df "$RAMDISK_PATH" --output=avail 2>/dev/null | tail -1 | tr -d ' ') + RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}") RAMDISK_AVAIL_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_AVAIL_KB / 1048576}") SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null | xargs basename 2>/dev/null || echo "unknown") line "Ramdisk now: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB free → $SYMLINK_TARGET" else - issue "Ramdisk not mounted" + issue "Ramdisk not mounted at $RAMDISK_PATH" fi -if [[ -f "$TRANSCODE_DAILY_LOG" ]]; then - WEEK_PEAK=$(awk -F'|' -v cutoff="$WEEK_START" \ - '$1 >= cutoff {if ($2 > max) max=$2} END {printf "%.2f", max+0}' \ - "$TRANSCODE_DAILY_LOG") - WEEK_FLIPS=$(awk -F'|' -v cutoff="$WEEK_START" \ - '$1 >= cutoff {sum+=$3} END {print sum+0}' "$TRANSCODE_DAILY_LOG") - WEEK_RAM=$(awk -F'|' -v cutoff="$WEEK_START" \ - '$1 >= cutoff {sum+=$4} END {print sum+0}' "$TRANSCODE_DAILY_LOG") - WEEK_SSD=$(awk -F'|' -v cutoff="$WEEK_START" \ - '$1 >= cutoff {sum+=$5} END {print sum+0}' "$TRANSCODE_DAILY_LOG") - WEEK_FILES=$(awk -F'|' -v cutoff="$WEEK_START" \ - '$1 >= cutoff {sum+=$6} END {print sum+0}' "$TRANSCODE_DAILY_LOG") - - line "Week peak: ${WEEK_PEAK}GB | flips: ${WEEK_FLIPS} | files cleaned: ${WEEK_FILES}" +if [[ -f "${TRANSCODE_DAILY_LOG:-}" ]] && [[ -s "$TRANSCODE_DAILY_LOG" ]]; then + WEEK_PEAK=$(awk -F'|' -v c="$WEEK_START" '$1>=c {if($2>max)max=$2} END{printf "%.2f",max+0}' "$TRANSCODE_DAILY_LOG") + WEEK_FLIPS=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$3} END{print sum+0}' "$TRANSCODE_DAILY_LOG") + WEEK_RAM=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$4} END{print sum+0}' "$TRANSCODE_DAILY_LOG") + WEEK_SSD=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$5} END{print sum+0}' "$TRANSCODE_DAILY_LOG") + line "Week peak: ${WEEK_PEAK}GB | flips: ${WEEK_FLIPS}" line "Sessions: ${WEEK_RAM} ramdisk / ${WEEK_SSD} SSD" - PEAK_INT=$(printf "%.0f" "$WEEK_PEAK" 2>/dev/null || echo 0) - WARN_INT=$(printf "%.0f" "$RAMDISK_WARN_GB" 2>/dev/null || echo 0) - if [[ "$PEAK_INT" -ge "$WARN_INT" ]]; then - finding "Transcode peak ${WEEK_PEAK}GB reached warn threshold — consider increasing RAMDISK_SIZE" - fi - if [[ "${WEEK_FLIPS:-0}" -ge "${TRANSCODE_FLIP_WARN:-3}" ]]; then + WARN_INT=$(printf "%.0f" "${RAMDISK_WARN_GB:-6.8}" 2>/dev/null || echo 0) + [[ "$PEAK_INT" -ge "$WARN_INT" ]] && \ + finding "Transcode peak ${WEEK_PEAK}GB near threshold — consider increasing HOST*_RAMDISK_SIZE" + [[ "${WEEK_FLIPS:-0}" -ge "${TRANSCODE_FLIP_WARN:-3}" ]] && \ finding "Transcode flips this week: $WEEK_FLIPS — monitor ramdisk headroom" - fi fi -# ----------------------------------------------------------------------------------------------- -# ━━━ 🎵 Media Activity ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ 🎵 MEDIA ACTIVITY ━━━ +# ============================================================================================== section "🎵 MEDIA ACTIVITY" -# Arr cleanup stats -if [[ -f "$ARR_CLEANUP_STATS" ]]; then +if [[ -f "${ARR_CLEANUP_STATS:-}" ]] && [[ -s "$ARR_CLEANUP_STATS" ]]; then for arr in lidarr sonarr radarr; do - WEEK_ORPHANS=$(awk -F'|' -v cutoff="$WEEK_START" -v a="$arr" \ - '$1 >= cutoff && $2 == a {sum+=$3} END {print sum+0}' "$ARR_CLEANUP_STATS") - WEEK_BYTES=$(awk -F'|' -v cutoff="$WEEK_START" -v a="$arr" \ - '$1 >= cutoff && $2 == a {sum+=$4} END {print sum+0}' "$ARR_CLEANUP_STATS") - WEEK_TRACKED=$(awk -F'|' -v cutoff="$WEEK_START" -v a="$arr" \ - 'BEGIN{max=0} $1 >= cutoff && $2 == a && $8 > max {max=$8} END {print max+0}' \ - "$ARR_CLEANUP_STATS") + WEEK_ORPHANS=$(awk -F'|' -v c="$WEEK_START" -v a="$arr" \ + '$1>=c && $2==a {sum+=$3} END{print sum+0}' "$ARR_CLEANUP_STATS") + WEEK_BYTES=$(awk -F'|' -v c="$WEEK_START" -v a="$arr" \ + '$1>=c && $2==a {sum+=$4} END{print sum+0}' "$ARR_CLEANUP_STATS") + WEEK_TRACKED=$(awk -F'|' -v c="$WEEK_START" -v a="$arr" \ + 'BEGIN{max=0} $1>=c && $2==a && $8+0>max {max=$8} END{print max+0}' "$ARR_CLEANUP_STATS") if [[ "${WEEK_ORPHANS:-0}" -gt 0 ]]; then FREED=$(format_bytes "${WEEK_BYTES:-0}") line "${arr^} cleanup: $WEEK_ORPHANS orphans removed ($FREED freed) | tracked: $WEEK_TRACKED files" @@ -347,15 +401,12 @@ if [[ -f "$ARR_CLEANUP_STATS" ]]; then fi done else - line "Arr cleanup stats: no data yet (runs after first cleanup)" + line "Arr cleanup stats: no data yet (runs after first weekly cleanup)" fi -# Arr recovery stats -if [[ -f "$ARR_RECOVERY_STATS" ]]; then - WEEK_ACTIONED=$(awk -F'|' -v cutoff="$WEEK_START" \ - '$1 >= cutoff {sum+=$3} END {print sum+0}' "$ARR_RECOVERY_STATS") - WEEK_RUNS=$(awk -F'|' -v cutoff="$WEEK_START" \ - '$1 >= cutoff {count++} END {print count+0}' "$ARR_RECOVERY_STATS") +if [[ -f "${ARR_RECOVERY_STATS:-}" ]] && [[ -s "$ARR_RECOVERY_STATS" ]]; then + WEEK_ACTIONED=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$3} END{print sum+0}' "$ARR_RECOVERY_STATS") + WEEK_RUNS=$(awk -F'|' -v c="$WEEK_START" '$1>=c {count++} END{print count+0}' "$ARR_RECOVERY_STATS") if [[ "${WEEK_ACTIONED:-0}" -gt 0 ]]; then line "Arr recovery: $WEEK_ACTIONED items auto-recovered across $WEEK_RUNS runs" else @@ -363,16 +414,16 @@ if [[ -f "$ARR_RECOVERY_STATS" ]]; then fi fi -# Arr queue depth — current snapshot -for arr_name in "Sonarr|${HOST1_SONARR_URL}|${HOST1_SONARR_API_KEY}|v3" \ - "Radarr|${HOST1_RADARR_URL}|${HOST1_RADARR_API_KEY}|v3" \ - "Lidarr|${HOST1_LIDARR_URL}|${HOST1_LIDARR_API_KEY}|v1"; do - IFS='|' read -r name url key ver <<< "$arr_name" - if [[ "$url" == *"your-"* ]] || [[ -z "$key" ]]; then continue; fi +# Arr queue depth — uses detect_hosts() aliased SONARR_URL/RADARR_URL/LIDARR_URL +for arr_entry in "Sonarr|${SONARR_URL:-}|${SONARR_API_KEY:-}|v3" \ + "Radarr|${RADARR_URL:-}|${RADARR_API_KEY:-}|v3" \ + "Lidarr|${LIDARR_URL:-}|${LIDARR_API_KEY:-}|v1"; do + IFS='|' read -r name url key ver <<< "$arr_entry" + [[ -z "$url" || -z "$key" ]] && continue QUEUE=$(curl -sf --max-time 5 -H "X-Api-Key: $key" \ "${url}/api/${ver}/queue?pageSize=1" 2>/dev/null | \ grep -o '"totalRecords":[0-9]*' | grep -o '[0-9]*' || echo "?") - if [[ "$QUEUE" == "0" ]] || [[ -z "$QUEUE" ]]; then + if [[ "$QUEUE" == "0" || -z "$QUEUE" ]]; then line "$name queue: empty ✅" elif [[ "$QUEUE" == "?" ]]; then finding "$name queue: API unavailable" @@ -381,345 +432,247 @@ for arr_name in "Sonarr|${HOST1_SONARR_URL}|${HOST1_SONARR_API_KEY}|v3" \ fi done -# ----------------------------------------------------------------------------------------------- -# ━━━ 🌐 Rsync ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ 🌐 RSYNC ━━━ +# ============================================================================================== section "🌐 RSYNC" -if [[ -f "$BANDWIDTH_LOG" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then - WEEK_TOTAL_BYTES=$(awk -F'|' -v cutoff="$WEEK_START" \ - '$1 >= cutoff {sum+=$3} END {print sum+0}' "$BANDWIDTH_LOG") - WEEK_SYNCS=$(awk -F'|' -v cutoff="$WEEK_START" \ - '$1 >= cutoff' "$BANDWIDTH_LOG" | wc -l) - WEEK_FAILED=$(awk -F'|' -v cutoff="$WEEK_START" \ - '$1 >= cutoff && $5 != "success"' "$BANDWIDTH_LOG" | wc -l) +# New bandwidth log format: date|time|profile|duration|status|bytes|warn_flag +if [[ -f "${BANDWIDTH_LOG:-}" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then + WEEK_TOTAL_BYTES=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$6} END{print sum+0}' "$BANDWIDTH_LOG") + WEEK_SYNCS=$(awk -F'|' -v c="$WEEK_START" '$1>=c' "$BANDWIDTH_LOG" | wc -l) + WEEK_FAILED=$(awk -F'|' -v c="$WEEK_START" '$1>=c && $5!="success"' "$BANDWIDTH_LOG" | wc -l) + WEEK_LARGE=$(awk -F'|' -v c="$WEEK_START" '$1>=c && $7=="LARGE"' "$BANDWIDTH_LOG" | wc -l) WEEK_GB=$(awk "BEGIN {printf \"%.1f\", $WEEK_TOTAL_BYTES / 1073741824}") + line "Total: ${WEEK_GB}GB across $WEEK_SYNCS syncs" + [[ "${WEEK_FAILED:-0}" -gt 0 ]] && issue "Failed syncs this week: $WEEK_FAILED" || line "Sync failures: none ✅" + [[ "${WEEK_LARGE:-0}" -gt 0 ]] && finding "Large transfers (>${BANDWIDTH_WARN_GB}GB): $WEEK_LARGE" - line "Total transferred: ${WEEK_GB}GB across $WEEK_SYNCS syncs" - if [[ "${WEEK_FAILED:-0}" -gt 0 ]]; then - issue "Failed syncs this week: $WEEK_FAILED" - else - line "Sync failures: none ✅" - fi - - # Top 3 shares by transfer this week - TOP_SHARES=$(awk -F'|' -v cutoff="$WEEK_START" \ - '$1 >= cutoff {bytes[$4]+=$3} END {for(s in bytes) print bytes[s], s}' \ + # Top 3 profiles by transfer — profile is field $3 + TOP_SHARES=$(awk -F'|' -v c="$WEEK_START" \ + '$1>=c {bytes[$3]+=$6} END {for(s in bytes) print bytes[s], s}' \ "$BANDWIDTH_LOG" | sort -rn | head -3) if [[ -n "$TOP_SHARES" ]]; then - while IFS=' ' read -r bytes share; do - [[ -z "$share" ]] && continue - SIZE=$(format_bytes "$bytes") - line " → $share: $SIZE" + while IFS=' ' read -r bytes profile; do + [[ -z "$profile" ]] && continue + line " → $profile: $(format_bytes "$bytes")" done <<< "$TOP_SHARES" fi else line "No bandwidth data yet" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ 🛡️ Watchdog ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ 🛡️ WATCHDOG ━━━ +# ============================================================================================== section "🛡️ WATCHDOG" -get_lock_pid_cr() { - local script_name="$1" - local lockfile="$LOCK_DIR/${script_name}.lock" - if [[ -f "$lockfile" ]]; then - local content - content=$(cat "$lockfile" 2>/dev/null) - echo "${content%%:*}" - fi -} - -get_lock_name_cr() { - local script_name="$1" - local lockfile="$LOCK_DIR/${script_name}.lock" - if [[ -f "$lockfile" ]]; then - local content - content=$(cat "$lockfile" 2>/dev/null) - echo "${content##*:}" - fi -} - -is_loop_running() { - local script_name="$1" - local pid name - pid=$(get_lock_pid_cr "$script_name") - name=$(get_lock_name_cr "$script_name") - [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$name" == "$script_name" ]] -} - -get_lock_age_cr() { - local script_name="$1" - local lockfile="$LOCK_DIR/${script_name}.lock" - if [[ -f "$lockfile" ]]; then - local mtime - mtime=$(stat -c %Y "$lockfile" 2>/dev/null || echo 0) - echo $(( $(date +%s) - mtime )) - else - echo 0 - fi -} - -format_uptime_cr() { - local seconds=$1 - local days=$(( seconds / 86400 )) - local hours=$(( (seconds % 86400) / 3600 )) - local mins=$(( (seconds % 3600) / 60 )) - if (( days > 0 )); then echo "${days}d ${hours}h ${mins}m" - elif (( hours > 0 )); then echo "${hours}h ${mins}m" - else echo "${mins}m"; fi -} - -# ── System Watchdog ── +# ── System Watchdog ─────────────────────────────────────────────────────────────────────────── line "⚙️ System Watchdog" - -SYS_PID_CR=$(get_lock_pid_cr "system_watchdog") -if is_loop_running "system_watchdog"; then - SYS_AGE_CR=$(get_lock_age_cr "system_watchdog") - SYS_UP_CR=$(format_uptime_cr "$SYS_AGE_CR") - SYS_CYCLE_CR=$(( SYS_AGE_CR / SYSTEM_WATCHDOG_INTERVAL )) - line " ✅ Running │ PID: $SYS_PID_CR │ Uptime: $SYS_UP_CR │ ~Cycle: $SYS_CYCLE_CR" +SYS_PID=$(_get_lock_pid "system_watchdog") +if _is_running "system_watchdog"; then + SYS_AGE=$(_lock_age "system_watchdog") + line " ✅ Running │ PID: $SYS_PID │ Uptime: $(_fmt_uptime "$SYS_AGE") │ ~Cycle: $(( SYS_AGE / SYSTEM_WATCHDOG_INTERVAL ))" else issue "system_watchdog NOT RUNNING" fi -# System strikes if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then - SYS_ACTIVE=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$") + SYS_ACTIVE=$(grep -v ":0$\|^watchdog_cycle=\|^mem_shutdown" \ + "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$") if [[ -n "$SYS_ACTIVE" ]]; then while IFS=: read -r key count; do [[ -z "$key" ]] && continue - issue " ⚠️ Strike: $key — $count/$SYS_WATCHDOG_STRIKE_LIMIT" + issue " Strike: $key — $count/$SYS_WATCHDOG_STRIKE_LIMIT" done <<< "$SYS_ACTIVE" else line " ✅ Strikes: none" fi fi -# Reboots this week if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then - WEEK_EPOCH=$(date -d "$WEEK_START" +%s) - REBOOT_COUNT=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \ - "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l) - if [[ "${REBOOT_COUNT:-0}" -gt 0 ]]; then - issue " 🔄 Reboots this week: $REBOOT_COUNT (watchdog triggered)" - else - line " ✅ Reboots this week: 0" - fi + WD_REBOOTS=$(awk -v c="$WEEK_EPOCH" '$1>=c' "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l) + [[ "${WD_REBOOTS:-0}" -gt 0 ]] && \ + issue " Watchdog reboots this week: $WD_REBOOTS" || \ + line " ✅ Watchdog reboots this week: 0" fi -# Skip list if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE") SKIP_LIST=$(tr '\n' ' ' < "$SYS_WATCHDOG_FAILED_FILE") - issue " ⛔ Skip list ($SKIP_COUNT): $SKIP_LIST" + issue " Skip list ($SKIP_COUNT): $SKIP_LIST" else line " ✅ Skip list: empty" fi -# System snapshot -ROOTFS_PCT_CR=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %') -MEM_AVAIL_CR=$(awk '/MemAvailable/ {printf "%.1f", $2/1048576}' /proc/meminfo) -MEM_TOTAL_CR=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo) -LOAD_CR=$(awk '{print $1}' /proc/loadavg) -ZOMBIE_CR=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null) -ZOMBIE_CR="${ZOMBIE_CR//[^0-9]/}" -ZOMBIE_CR="${ZOMBIE_CR:-0}" -ARC_GB_CR="n/a" -if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then - ARC_B=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) - ARC_GB_CR=$(awk "BEGIN {printf \"%.1f\", $ARC_B / 1073741824}") -fi -line " 📊 rootfs: ${ROOTFS_PCT_CR}% │ RAM: ${MEM_AVAIL_CR}GB free/${MEM_TOTAL_CR}GB │ ARC: ${ARC_GB_CR}GB │ load: ${LOAD_CR} │ zombies: ${ZOMBIE_CR}" +ROOTFS_PCT=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %') +MEM_AVAIL_GB=$(awk '/MemAvailable/{printf "%.1f",$2/1048576}' /proc/meminfo) +MEM_TOTAL_GB_SYS=$(awk '/MemTotal/{printf "%.0f",$2/1048576}' /proc/meminfo) +LOAD_NOW=$(awk '{print $1}' /proc/loadavg) +ZOMBIE_NOW=$(ps aux 2>/dev/null | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0) +ZOMBIE_NOW="${ZOMBIE_NOW//[^0-9]/}"; ZOMBIE_NOW="${ZOMBIE_NOW:-0}" +ARC_NOW="n/a" +[[ -f /proc/spl/kstat/zfs/arcstats ]] && \ + ARC_NOW=$(awk '/^size /{printf "%.1f",$3/1073741824}' /proc/spl/kstat/zfs/arcstats) +line " 📊 rootfs:${ROOTFS_PCT}% │ RAM:${MEM_AVAIL_GB}GB free/${MEM_TOTAL_GB_SYS}GB │ ARC:${ARC_NOW}GB │ load:${LOAD_NOW} │ zombies:${ZOMBIE_NOW}" REPORT+=("") -# ── Docker Watchdog ── +# ── Docker Watchdog ─────────────────────────────────────────────────────────────────────────── line "🐳 Docker Watchdog" - -DOCKER_PID_CR=$(get_lock_pid_cr "docker_watchdog") -if is_loop_running "docker_watchdog"; then - DOCKER_AGE_CR=$(get_lock_age_cr "docker_watchdog") - DOCKER_UP_CR=$(format_uptime_cr "$DOCKER_AGE_CR") - DOCKER_CYCLE_CR=$(( DOCKER_AGE_CR / DOCKER_WATCHDOG_INTERVAL )) - line " ✅ Running │ PID: $DOCKER_PID_CR │ Uptime: $DOCKER_UP_CR │ ~Cycle: $DOCKER_CYCLE_CR" +DOCKER_PID=$(_get_lock_pid "docker_watchdog") +if _is_running "docker_watchdog"; then + DOCKER_AGE=$(_lock_age "docker_watchdog") + line " ✅ Running │ PID: $DOCKER_PID │ Uptime: $(_fmt_uptime "$DOCKER_AGE") │ ~Cycle: $(( DOCKER_AGE / DOCKER_WATCHDOG_INTERVAL ))" else issue "docker_watchdog NOT RUNNING" fi -# Container strikes if [[ -f "$WATCHDOG_STATE_FILE" ]]; then DOCK_ACTIVE=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$") if [[ -n "$DOCK_ACTIVE" ]]; then while IFS=: read -r key count; do [[ -z "$key" ]] && continue - issue " ⚠️ Strike: $key — $count" + issue " Strike: $key — $count" done <<< "$DOCK_ACTIVE" else line " ✅ Container strikes: none" fi fi -# Container restarts this week if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then - WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_START" \ - '$2 >= cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l) - if [[ "${WEEK_RESTARTS:-0}" -gt 0 ]]; then - RESTARTED=$(awk -F'|' -v cutoff="$WEEK_START" \ - '$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \ + WK_RESTARTS=$(awk -F'|' -v c="$WEEK_START" '$2>=c' \ + "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l) + if [[ "${WK_RESTARTS:-0}" -gt 0 ]]; then + RESTARTED=$(awk -F'|' -v c="$WEEK_START" '$2>=c{print $1}' \ + "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \ sort | uniq -c | sort -rn | head -5 | \ awk '{print $2"("$1")"}' | tr '\n' ' ') - finding " 🔄 Restarts this week: $WEEK_RESTARTS — $RESTARTED" + finding " Container restarts this week: $WK_RESTARTS — $RESTARTED" else line " ✅ Container restarts this week: none" fi fi -# Docker overview — filter WATCHDOG_SCAN_IGNORE from stopped list if command -v docker >/dev/null 2>&1; then - RUNNING_CR=$(docker ps -q 2>/dev/null | wc -l) - TOTAL_CR=$(docker ps -aq 2>/dev/null | wc -l) - UNHEALTHY_CR=$(docker ps --filter health=unhealthy -q 2>/dev/null | wc -l) - - STOPPED_FILTERED_CR=() + RUNNING_NOW=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l) + TOTAL_NOW=$( timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l) + UNHEALTHY_NOW=$(timeout "$DOCKER_TIMEOUT" docker ps \ + --filter health=unhealthy -q 2>/dev/null | wc -l) + STOPPED_NAMES=() while IFS= read -r name; do [[ -z "$name" ]] && continue SKIP=false for ignore in "${WATCHDOG_SCAN_IGNORE[@]:-}"; do [[ "$name" == "$ignore" ]] && SKIP=true && break done - [[ "$SKIP" == false ]] && STOPPED_FILTERED_CR+=("$name") - done < <(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null) - - STOPPED_COUNT_CR="${#STOPPED_FILTERED_CR[@]}" - line " 📦 $RUNNING_CR/$TOTAL_CR running │ unhealthy: $UNHEALTHY_CR" - if [[ "$STOPPED_COUNT_CR" -gt 0 ]]; then - issue " ⚠️ Stopped (unexpected): ${STOPPED_FILTERED_CR[*]}" - else - line " ✅ All containers running" - fi + [[ "$SKIP" == false ]] && STOPPED_NAMES+=("$name") + done < <(timeout "$DOCKER_TIMEOUT" docker ps -af "status=exited" \ + --format "{{.Names}}" 2>/dev/null) + line " 📦 $RUNNING_NOW/$TOTAL_NOW running │ unhealthy: $UNHEALTHY_NOW" + [[ ${#STOPPED_NAMES[@]} -gt 0 ]] && \ + issue " Stopped (unexpected): ${STOPPED_NAMES[*]}" || \ + line " ✅ All containers running" fi REPORT+=("") -# ── Failover ── +# ── Failover ───────────────────────────────────────────────────────────────────────────────── line "🔀 Failover" - -FAILOVER_PID_CR=$(get_lock_pid_cr "failover") -if is_loop_running "failover"; then - FAILOVER_AGE_CR=$(get_lock_age_cr "failover") - FAILOVER_UP_CR=$(format_uptime_cr "$FAILOVER_AGE_CR") - line " ✅ Running │ PID: $FAILOVER_PID_CR │ Uptime: $FAILOVER_UP_CR" +FO_PID=$(_get_lock_pid "failover") +if _is_running "failover"; then + FO_AGE=$(_lock_age "failover") + line " ✅ Running │ PID: $FO_PID │ Uptime: $(_fmt_uptime "$FO_AGE")" +elif [[ "${FAILOVER_ENABLED:-true}" == false ]]; then + line " ⏸️ Not running — FAILOVER_ENABLED=false" else - if [[ "${FAILOVER_ENABLED:-true}" == false ]]; then - line " ⏸️ Not running — disabled in Master.conf (FAILOVER_ENABLED=false)" - else - issue "failover NOT RUNNING" - fi + issue "failover NOT RUNNING" fi -FAILOVER_STATE_CR="UNKNOWN" -FAILOVER_STATE_SECS_CR=0 +FO_STATE="UNKNOWN" +FO_STATE_SECS=0 if [[ -f "$FAILOVER_STATE_FILE" ]]; then - FAILOVER_STATE_CR=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) - FAILOVER_LAST_EPOCH_CR=$(grep "^last_change_epoch=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) - [[ -n "$FAILOVER_LAST_EPOCH_CR" ]] && \ - FAILOVER_STATE_SECS_CR=$(( $(date +%s) - FAILOVER_LAST_EPOCH_CR )) + FO_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) + FO_EPOCH=$(grep "^last_change_epoch=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) + [[ -n "$FO_EPOCH" ]] && FO_STATE_SECS=$(( NOW - FO_EPOCH )) fi -STATE_DUR_CR=$(format_uptime_cr "${FAILOVER_STATE_SECS_CR:-0}") +FO_DUR=$(_fmt_uptime "${FO_STATE_SECS:-0}") -case "$FAILOVER_STATE_CR" in +case "$FO_STATE" in NORMAL) - line " ✅ State: NORMAL │ Duration: $STATE_DUR_CR" - ;; + line " ✅ State: NORMAL │ Duration: $FO_DUR" ;; FAILOVER) - issue " ⚠️ State: FAILOVER — remote down for $STATE_DUR_CR" - FAILOVER_MINS_CR=$(( FAILOVER_STATE_SECS_CR / 60 )) - if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - T2=$HOST2_TIER2_DELAY; T3=$HOST2_TIER3_DELAY; T4=$HOST2_TIER4_DELAY - else - T2=$HOST1_TIER2_DELAY; T3=$HOST1_TIER3_DELAY; T4=$HOST1_TIER4_DELAY - fi - (( FAILOVER_MINS_CR >= T2 )) && line " 🔄 Tier 2: ✅ active" || \ - line " 🔄 Tier 2: ⏳ in $(( T2 - FAILOVER_MINS_CR ))min" - (( FAILOVER_MINS_CR >= T3 )) && line " 🔄 Tier 3: ✅ active" || \ - line " 🔄 Tier 3: ⏳ in $(( T3 - FAILOVER_MINS_CR ))min" - (( FAILOVER_MINS_CR >= T4 )) && line " 🔄 Tier 4: ✅ active" || \ - line " 🔄 Tier 4: ⏳ in $(( T4 - FAILOVER_MINS_CR ))min" - ;; - NO_INTERNET) - issue " ❌ State: NO_INTERNET — DDNS stopped │ Duration: $STATE_DUR_CR" - ;; - DARK) - issue " ❌ State: DARK — remote down AND no internet │ Duration: $STATE_DUR_CR" - ;; - *) - finding " ❓ State: ${FAILOVER_STATE_CR:-unknown}" + issue " State: FAILOVER — $REMOTE_SERVER_NAME down for $FO_DUR" + FO_MINS=$(( FO_STATE_SECS / 60 )) + # Use REMOTE_ID-based tier delay vars — no HOST1/HOST2 hardcoding + T2_VAR="${REMOTE_ID}_TIER2_DELAY"; T3_VAR="${REMOTE_ID}_TIER3_DELAY"; T4_VAR="${REMOTE_ID}_TIER4_DELAY" + T2="${!T2_VAR:-240}"; T3="${!T3_VAR:-720}"; T4="${!T4_VAR:-1440}" + (( FO_MINS >= T2 )) && line " Tier 2: ✅ active" || line " Tier 2: ⏳ in $(( T2 - FO_MINS ))min" + (( FO_MINS >= T3 )) && line " Tier 3: ✅ active" || line " Tier 3: ⏳ in $(( T3 - FO_MINS ))min" + (( FO_MINS >= T4 )) && line " Tier 4: ✅ active" || line " Tier 4: ⏳ in $(( T4 - FO_MINS ))min" ;; + NO_INTERNET) issue " State: NO_INTERNET — DDNS stopped │ Duration: $FO_DUR" ;; + DARK) issue " State: DARK — remote down AND no internet │ Duration: $FO_DUR" ;; + *) finding " State: ${FO_STATE:-unknown}" ;; esac -# Tailscale if command -v tailscale >/dev/null 2>&1; then - REMOTE_IP_CR=$(tailscale ip -4 "$HOST2" 2>/dev/null) - if [[ -n "$REMOTE_IP_CR" ]]; then - if ping -c 1 -W 2 "$REMOTE_IP_CR" >/dev/null 2>&1; then - line " 🌐 $HOST2: $REMOTE_IP_CR — reachable ✅" - else - issue " 🌐 $HOST2: $REMOTE_IP_CR — not responding" - fi + REMOTE_TS_IP=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) + if [[ -n "$REMOTE_TS_IP" ]]; then + ping -c 1 -W 2 "$REMOTE_TS_IP" >/dev/null 2>&1 && \ + line " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_TS_IP — reachable ✅" || \ + issue " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_TS_IP — not responding" else - issue " 🌐 $HOST2 not visible on Tailscale" + issue " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME) not visible on Tailscale" fi fi -# ----------------------------------------------------------------------------------------------- -# ━━━ 🔐 Security ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ 🔐 SECURITY ━━━ +# ============================================================================================== section "🔐 SECURITY" -for domain in "${CERT_MONITOR_DOMAINS[@]:-}"; do - [[ -z "$domain" ]] && continue - EXPIRY=$(echo | timeout "${CERT_TIMEOUT:-10}" openssl s_client \ - -connect "${domain}:443" -servername "$domain" 2>/dev/null | \ - openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) - if [[ -z "$EXPIRY" ]]; then - issue "$domain: could not check certificate" - continue - fi - EXPIRY_EPOCH=$(date -d "$EXPIRY" +%s 2>/dev/null || echo 0) - DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW) / 86400 )) - if [[ "$DAYS_LEFT" -le "${CERT_CRIT_DAYS:-7}" ]]; then - issue "$domain: ${DAYS_LEFT} days remaining — CRITICAL, renew now" - elif [[ "$DAYS_LEFT" -le "${CERT_WARN_DAYS:-30}" ]]; then - finding "$domain: ${DAYS_LEFT} days remaining — renew soon" - else - line "$domain: ${DAYS_LEFT} days remaining ✅" - fi -done - -# ----------------------------------------------------------------------------------------------- -# ━━━ 📊 Emby ━━━ -# ----------------------------------------------------------------------------------------------- -section "📊 EMBY" - -# Select correct Emby URL and key for this host -if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - EMBY_URL="$HOST1_EMBY_URL" - EMBY_KEY="$HOST1_EMBY_API_KEY" +if ! command -v openssl >/dev/null 2>&1; then + line "SSL certs: openssl not available — skipping" +elif [[ ${#CERT_MONITOR_DOMAINS[@]} -eq 0 ]]; then + line "SSL certs: no domains configured (CERT_MONITOR_DOMAINS empty)" else - EMBY_URL="$HOST2_EMBY_URL" - EMBY_KEY="$HOST2_EMBY_API_KEY" + for domain in "${CERT_MONITOR_DOMAINS[@]:-}"; do + [[ -z "$domain" ]] && continue + EXPIRY=$(echo | timeout "${CERT_TIMEOUT:-10}" openssl s_client \ + -connect "${domain}:443" -servername "$domain" 2>/dev/null | \ + openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) + if [[ -z "$EXPIRY" ]]; then + issue "$domain: could not check certificate" + continue + fi + EXPIRY_EPOCH=$(date -d "$EXPIRY" +%s 2>/dev/null || echo 0) + DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW) / 86400 )) + if [[ "$DAYS_LEFT" -le "${CERT_CRIT_DAYS:-7}" ]]; then + issue "$domain: ${DAYS_LEFT} days remaining — CRITICAL, renew now" + elif [[ "$DAYS_LEFT" -le "${CERT_WARN_DAYS:-30}" ]]; then + finding "$domain: ${DAYS_LEFT} days remaining — renew soon" + else + line "$domain: ${DAYS_LEFT} days remaining ✅" + fi + done fi -if [[ "$EMBY_KEY" != *"your-"* ]] && [[ -n "$EMBY_KEY" ]]; then +# ============================================================================================== +# ━━━ 📊 EMBY ━━━ +# ============================================================================================== +section "📊 EMBY" - # Check if Emby is reachable +# detect_hosts() already aliased EMBY_URL and EMBY_API_KEY +if [[ -z "${EMBY_API_KEY:-}" ]]; then + line "Emby: API key not configured" +elif [[ "${EMBY_API_KEY:-}" == *"your-"* ]]; then + line "Emby: placeholder API key — configure HOST*_EMBY_API_KEY" +else EMBY_SYSTEM=$(curl -sf --max-time 5 \ - -H "X-Emby-Token: $EMBY_KEY" \ - "${EMBY_URL}/System/Info?api_key=$EMBY_KEY" 2>/dev/null) + -H "X-Emby-Token: $EMBY_API_KEY" \ + "${EMBY_URL}/System/Info" 2>/dev/null) if [[ -z "$EMBY_SYSTEM" ]]; then line "Emby: API unavailable — check if Emby is running" @@ -727,28 +680,22 @@ if [[ "$EMBY_KEY" != *"your-"* ]] && [[ -n "$EMBY_KEY" ]]; then EMBY_VERSION=$(echo "$EMBY_SYSTEM" | grep -o '"Version":"[^"]*"' | cut -d'"' -f4) line "Emby: v${EMBY_VERSION:-unknown} — reachable ✅" - # Active sessions right now SESSIONS=$(curl -sf --max-time 5 \ - -H "X-Emby-Token: $EMBY_KEY" \ - "${EMBY_URL}/Sessions?api_key=$EMBY_KEY" 2>/dev/null) + -H "X-Emby-Token: $EMBY_API_KEY" \ + "${EMBY_URL}/Sessions" 2>/dev/null) ACTIVE_COUNT=$(echo "$SESSIONS" | grep -o '"NowPlayingItem"' | wc -l) ACTIVE_COUNT="${ACTIVE_COUNT//[^0-9]/}"; ACTIVE_COUNT="${ACTIVE_COUNT:-0}" line "Active streams now: $ACTIVE_COUNT" - # Weekly stats via user_usage_stats plugin if available ACTIVITY_LOG=$(curl -sf --max-time 10 \ - -H "X-Emby-Token: $EMBY_KEY" \ - "${EMBY_URL}/user_usage_stats/user_activity?days=7&api_key=$EMBY_KEY" \ - 2>/dev/null) + -H "X-Emby-Token: $EMBY_API_KEY" \ + "${EMBY_URL}/user_usage_stats/user_activity?days=7" 2>/dev/null) if [[ -n "$ACTIVITY_LOG" ]] && echo "$ACTIVITY_LOG" | grep -q "user_name"; then - # Parse total plays — sum all total_plays fields TOTAL_PLAYS=$(echo "$ACTIVITY_LOG" | \ grep -o '"total_plays":[0-9]*' | \ awk -F: '{sum+=$2} END {print sum+0}') line "Streams this week: $TOTAL_PLAYS total plays" - - # Top 3 users by play count echo "$ACTIVITY_LOG" | \ grep -o '"user_name":"[^"]*","total_plays":[0-9]*' | \ awk -F'"' '{name=$4; plays=$NF; gsub(/.*:/,"",plays); print plays, name}' | \ @@ -760,69 +707,50 @@ if [[ "$EMBY_KEY" != *"your-"* ]] && [[ -n "$EMBY_KEY" ]]; then line "Weekly stats: user_usage_stats plugin not available" fi fi -else - line "Emby: API key not configured" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ ⚙️ System Health ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ ⚙️ SYSTEM HEALTH ━━━ +# ============================================================================================== section "⚙️ SYSTEM HEALTH" -# inotify + php-fpm weekly stats if [[ -f "${TUNING_MONITOR_LOG:-}" ]] && [[ -s "$TUNING_MONITOR_LOG" ]]; then - # inotify weekly stats - INOTIFY_PEAK=$(awk -F'|' -v cutoff="$WEEK_START" \ - '$1 >= cutoff {if ($3+0 > max) max=$3+0} END {print max+0}' "$TUNING_MONITOR_LOG") - INOTIFY_AVG=$(awk -F'|' -v cutoff="$WEEK_START" \ - '$1 >= cutoff {sum+=$3; count++} END {if(count>0) printf "%.0f", sum/count; else print 0}' \ - "$TUNING_MONITOR_LOG") - INOTIFY_LOW=$(awk -F'|' -v cutoff="$WEEK_START" \ - 'BEGIN{min=99999} $1 >= cutoff {if($3+0 < min) min=$3+0} END {print min+0}' \ - "$TUNING_MONITOR_LOG") - INOTIFY_LIMIT_NOW=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 1024) - INOTIFY_NOW=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l) - INOTIFY_NOW="${INOTIFY_NOW//[^0-9]/}"; INOTIFY_NOW="${INOTIFY_NOW:-0}" - INOTIFY_NOW_PCT=$(( INOTIFY_NOW * 100 / INOTIFY_LIMIT_NOW )) - INOTIFY_WARN_COUNT=$(awk -F'|' -v cutoff="$WEEK_START" \ - '$1 >= cutoff && $6 == 1 {count++} END {print count+0}' "$TUNING_MONITOR_LOG") + INOTIFY_PEAK=$(awk -F'|' -v c="$WEEK_START" \ + '$1>=c {if($3+0>max) max=$3+0} END{print max+0}' "$TUNING_MONITOR_LOG") + INOTIFY_AVG=$(awk -F'|' -v c="$WEEK_START" \ + '$1>=c {sum+=$3;cnt++} END{if(cnt>0)printf "%.0f",sum/cnt;else print 0}' "$TUNING_MONITOR_LOG") + INOTIFY_WARNS=$(awk -F'|' -v c="$WEEK_START" \ + '$1>=c && $6==1 {cnt++} END{print cnt+0}' "$TUNING_MONITOR_LOG") + INOW=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l) + INOW="${INOW//[^0-9]/}"; INOW="${INOW:-0}" + ILIM=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 1024) + IPCT=$(( INOW * 100 / ILIM )) + [[ "${INOTIFY_WARNS:-0}" -gt 0 ]] && \ + issue "inotify: ${INOW}/${ILIM} now (${IPCT}%) | week peak:$INOTIFY_PEAK avg:$INOTIFY_AVG | ⚠️ $INOTIFY_WARNS warning(s)" || \ + line "inotify: ${INOW}/${ILIM} now (${IPCT}%) | week peak:$INOTIFY_PEAK avg:$INOTIFY_AVG ✅" - if [[ "${INOTIFY_WARN_COUNT:-0}" -gt 0 ]]; then - issue "inotify: ${INOTIFY_NOW}/${INOTIFY_LIMIT_NOW} now (${INOTIFY_NOW_PCT}%) | week peak: $INOTIFY_PEAK avg: $INOTIFY_AVG low: $INOTIFY_LOW | ⚠️ warnings: $INOTIFY_WARN_COUNT" - else - line "inotify: ${INOTIFY_NOW}/${INOTIFY_LIMIT_NOW} now (${INOTIFY_NOW_PCT}%) | week peak: $INOTIFY_PEAK avg: $INOTIFY_AVG low: $INOTIFY_LOW ✅" - fi - - # php-fpm weekly stats - PHPFPM_PEAK=$(awk -F'|' -v cutoff="$WEEK_START" \ - '$1 >= cutoff {if ($7+0 > max) max=$7+0} END {print max+0}' "$TUNING_MONITOR_LOG") - PHPFPM_AVG=$(awk -F'|' -v cutoff="$WEEK_START" \ - '$1 >= cutoff {sum+=$7; count++} END {if(count>0) printf "%.0f", sum/count; else print 0}' \ - "$TUNING_MONITOR_LOG") - PHPFPM_MAX_NOW="${PHP_MAX_CHILDREN:-250}" - PHPFPM_NOW=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0) - PHPFPM_NOW="${PHPFPM_NOW//[^0-9]/}"; PHPFPM_NOW="${PHPFPM_NOW:-0}" - PHPFPM_WARN_COUNT=$(awk -F'|' -v cutoff="$WEEK_START" \ - '$1 >= cutoff && $10 == 1 {count++} END {print count+0}' "$TUNING_MONITOR_LOG") - - if [[ "${PHPFPM_WARN_COUNT:-0}" -gt 0 ]]; then - issue "php-fpm: ${PHPFPM_NOW}/${PHPFPM_MAX_NOW} workers now | week peak: $PHPFPM_PEAK avg: $PHPFPM_AVG | ⚠️ warnings: $PHPFPM_WARN_COUNT" - else - line "php-fpm: ${PHPFPM_NOW}/${PHPFPM_MAX_NOW} workers now | week peak: $PHPFPM_PEAK avg: $PHPFPM_AVG ✅" - fi + PHPFPM_PEAK=$(awk -F'|' -v c="$WEEK_START" \ + '$1>=c {if($7+0>max) max=$7+0} END{print max+0}' "$TUNING_MONITOR_LOG") + PHPFPM_AVG=$(awk -F'|' -v c="$WEEK_START" \ + '$1>=c {sum+=$7;cnt++} END{if(cnt>0)printf "%.0f",sum/cnt;else print 0}' "$TUNING_MONITOR_LOG") + PHPFPM_WARNS=$(awk -F'|' -v c="$WEEK_START" \ + '$1>=c && $10==1 {cnt++} END{print cnt+0}' "$TUNING_MONITOR_LOG") + PNOW=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0) + PNOW="${PNOW//[^0-9]/}"; PNOW="${PNOW:-0}" + PMAX="${PHP_MAX_CHILDREN:-250}" + [[ "${PHPFPM_WARNS:-0}" -gt 0 ]] && \ + issue "php-fpm: ${PNOW}/${PMAX} workers now | week peak:$PHPFPM_PEAK avg:$PHPFPM_AVG | ⚠️ $PHPFPM_WARNS warning(s)" || \ + line "php-fpm: ${PNOW}/${PMAX} workers now | week peak:$PHPFPM_PEAK avg:$PHPFPM_AVG ✅" else - # No log yet — just show live values - INOTIFY_LIMIT_NOW=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 1024) - INOTIFY_NOW=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l) - INOTIFY_NOW="${INOTIFY_NOW//[^0-9]/}"; INOTIFY_NOW="${INOTIFY_NOW:-0}" - INOTIFY_NOW_PCT=$(( INOTIFY_NOW * 100 / INOTIFY_LIMIT_NOW )) - PHPFPM_NOW=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0) - PHPFPM_NOW="${PHPFPM_NOW//[^0-9]/}"; PHPFPM_NOW="${PHPFPM_NOW:-0}" - line "inotify: ${INOTIFY_NOW}/${INOTIFY_LIMIT_NOW} (${INOTIFY_NOW_PCT}%) — no weekly data yet" - line "php-fpm: ${PHPFPM_NOW}/${PHP_MAX_CHILDREN:-250} workers — no weekly data yet" + INOW=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l) + INOW="${INOW//[^0-9]/}"; INOW="${INOW:-0}" + ILIM=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 1024) + PNOW=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0) + PNOW="${PNOW//[^0-9]/}"; PNOW="${PNOW:-0}" + line "inotify: ${INOW}/${ILIM} — no weekly data yet" + line "php-fpm: ${PNOW}/${PHP_MAX_CHILDREN:-250} workers — no weekly data yet" fi -# SMART summary if command -v smartctl >/dev/null 2>&1; then SMART_ISSUES=0 for disk in /dev/sd? /dev/nvme?; do @@ -833,28 +761,24 @@ if command -v smartctl >/dev/null 2>&1; then [[ "$DISK_NAME" == "$ignore" ]] && SKIP=true && break done [[ "$SKIP" == true ]] && continue - HEALTH=$(smartctl -H "$disk" 2>/dev/null | grep "SMART overall-health" | \ - awk '{print $NF}') + HEALTH=$(smartctl -H "$disk" 2>/dev/null | grep "SMART overall-health" | awk '{print $NF}') if [[ "$HEALTH" != "PASSED" ]] && [[ -n "$HEALTH" ]]; then issue "SMART $DISK_NAME: $HEALTH — check immediately" - ((SMART_ISSUES++)) + (( SMART_ISSUES++ )) fi done - if [[ "$SMART_ISSUES" -eq 0 ]]; then - line "SMART: all drives PASSED ✅" - fi + [[ "$SMART_ISSUES" -eq 0 ]] && line "SMART: all drives PASSED ✅" fi -# Gitea sync status -if command -v git >/dev/null 2>&1 && [[ -d "$TARGET_DIR/.git" ]]; then +if command -v git >/dev/null 2>&1 && [[ -d "${TARGET_DIR:-}/.git" ]]; then CURRENT_COMMIT=$(git -C "$TARGET_DIR" rev-parse --short HEAD 2>/dev/null || echo "unknown") - LAST_PULL=$(git -C "$TARGET_DIR" log -1 --format="%ar" 2>/dev/null || echo "unknown") + LAST_PULL=$(git -C "$TARGET_DIR" log -1 --format="%ar" 2>/dev/null || echo "unknown") line "Gitea: commit $CURRENT_COMMIT (pulled $LAST_PULL)" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ ⚠️ Issues Requiring Attention ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ ⚠️ ISSUES REQUIRING ATTENTION ━━━ +# ============================================================================================== if [[ ${#ISSUES[@]} -gt 0 ]]; then REPORT+=("") REPORT+=("⚠️ ISSUES REQUIRING ATTENTION") @@ -873,21 +797,21 @@ if [[ ${#FINDINGS[@]} -gt 0 ]]; then done fi -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== # ━━━ Footer ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== REPORT+=("") if [[ ${#ISSUES[@]} -eq 0 ]]; then - STATUS="✅ All systems healthy — enjoy your Sunday" + REPORT+=("✅ All systems healthy — enjoy your Sunday ☕") else - STATUS="⚠️ ${#ISSUES[@]} issue(s) need attention" + REPORT+=("⚠️ ${#ISSUES[@]} issue(s) need attention") fi -REPORT+=("$STATUS") -REPORT+=("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━") +REPORT+=("$MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S')") +REPORT+=("$(printf '%.0s━' {1..50})") -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== # ━━━ Output and Send ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== HEADER="☕ SUNDAY MORNING COFFEE REPORT — $REPORT_DATE" DIVIDER="$(printf '%.0s━' {1..50})" BODY=$(printf '%s\n' "$HEADER" "${REPORT[@]}") @@ -896,32 +820,31 @@ echo "" echo "$DIVIDER" echo "$HEADER" echo "$DIVIDER" - for report_line in "${REPORT[@]}"; do echo "$report_line" done - echo "" -# Send notification if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — notification not sent" else - # Notify with full body — suppress body from terminal log to avoid duplication if [[ "${NOTIFY_UNRAID:-false}" == true ]]; then NOTIFY_SCRIPT="/usr/local/emhttp/plugins/dynamix/scripts/notify" if [[ -x "$NOTIFY_SCRIPT" ]]; then - "$NOTIFY_SCRIPT" -s "☕ Weekly Report" -d "$BODY" -i "normal" 2>/dev/null - log "$ICON_NOTIFY unRAID notification sent" + "$NOTIFY_SCRIPT" -s "☕ Weekly Report — $MY_ID" -d "$BODY" -i "normal" 2>/dev/null + log "unRAID notification sent" fi fi if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then - PAYLOAD=$(printf '{"content": "%s — **%s**\\n%s"}' \ - "$ICON_NOTIFY" "☕ Weekly Report" "$BODY") - curl -s -H "Content-Type: application/json" \ + # Escape body for JSON + ESCAPED_BODY=$(echo "$BODY" | python3 -c \ + 'import sys,json; print(json.dumps(sys.stdin.read()))' 2>/dev/null || \ + echo "\"$BODY\"") + PAYLOAD="{\"content\": ${ESCAPED_BODY}}" + curl -sf -H "Content-Type: application/json" \ -d "$PAYLOAD" "$DISCORD_WEBHOOK" >/dev/null 2>&1 && \ - log "$ICON_NOTIFY Discord notification sent" || \ + log "Discord notification sent" || \ warn "Discord notification failed" fi - success "Report sent" + log "Report generated — $MY_ID — ${#ISSUES[@]} issue(s) ${#FINDINGS[@]} finding(s)" fi \ No newline at end of file diff --git a/Orchestrators/transcode_management.sh b/Orchestrators/transcode_management.sh index fd6badd..36f2d12 100644 --- a/Orchestrators/transcode_management.sh +++ b/Orchestrators/transcode_management.sh @@ -1,132 +1,127 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Transcode Management --------------------------------------- -# ----------------------------------------------------------------------------------------------- -# Runs transcode cleanup then transcode manager in the correct order every cycle. -# Cleanup runs first — clears stale files so manager sees accurate usage. -# Manager runs after — threshold decisions based on real current usage post-cleanup. -# -# Running cleanup before manager prevents unnecessary SSD flips caused by stale -# segment files from ended sessions inflating the ramdisk usage reading. -# -# Also tracks daily transcode statistics to a bounded log for weekly_health_digest.sh: -# Peak ramdisk usage per day -# Total flip count per day -# Ramdisk vs SSD session ratio -# Files cleaned per day -# -# Scheduled as: */3 * * * * (every 3 minutes) +# ============================================================================================== +# ============================= Transcode Management =========================================== +# ============================================================================================== +# Orchestrator — runs transcode_cleanup.sh then transcode_manager.sh in the correct order. # Replace individual transcode_manager and transcode_cleanup cron entries with this. +# Schedule: */3 * * * * (every 3 minutes via User Scripts plugin) # -# All configuration in Master.conf under Transcode Manager section. -# Supports --dry-run — passes through to both child scripts. -# ----------------------------------------------------------------------------------------------- +# ── WHY CLEANUP BEFORE MANAGER ──────────────────────────────────────────────────────────────── +# Cleanup runs first — removes stale segment files from ended sessions. +# Manager runs after — threshold decisions based on real current usage post-cleanup. +# Without this order, stale files inflate the ramdisk usage reading and trigger +# unnecessary SSD flips even when active sessions would fit on the ramdisk. +# +# ── WHAT EACH SCRIPT DOES ───────────────────────────────────────────────────────────────────── +# transcode_cleanup.sh — removes aged segment files not open by any process +# uses lsof for O(1) per-file active check (never per-file lsof) +# also triggers flip-back to ramdisk after cleanup if recovered ✅ +# +# transcode_manager.sh — checks ramdisk usage against thresholds +# flips symlink between ramdisk and SSD as needed +# writes one entry to TRANSCODE_DAILY_LOG after each run +# shows active Emby sessions with play method +# +# ── DAILY LOG ───────────────────────────────────────────────────────────────────────────────── +# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run: +# Format: DATE|RAMDISK_USED_GB|FLIP_COUNT|RAM_SESSIONS|SSD_SESSIONS +# This orchestrator does NOT write its own log — manager handles it ✅ +# Log trimmed to TRANSCODE_LOG_RETENTION days by manager on each write. +# Read by sunday_morning_coffee_report.sh and weekly_health_digest.sh. +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID and aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_WARN_GB etc. +# Each server manages its own transcode location independently. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# Root check — mount and docker operations require root +# acquire_lock — prevents concurrent 3-minute cycles overlapping +# detect_hosts() — correct paths per host +# --dry-run — passed through to both child scripts +# Exit code — worst exit code of both scripts returned +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# TRANSCODE_DAILY_LOG — daily stats log (written by transcode_manager.sh) +# TRANSCODE_LOG_RETENTION — days to keep (trimmed by manager) +# TRANSCODE_STATE_FILE — current state (ramdisk_used, flip_count etc.) +# All TRANSCODE_* threshold vars — see master.conf Transcode Manager section +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# transcode_management.sh — normal run (every 3 minutes via cron) +# transcode_management.sh --dry-run — preview without changes (passed to children) +# transcode_management.sh --status — show configuration and current state +# transcode_management.sh --log — verbose output from both child scripts +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# State and log files -# ----------------------------------------------------------------------------------------------- -TRANSCODE_DAILY_LOG="/boot/config/transcode_daily.db" -TRANSCODE_STATE_FILE="/tmp/transcode_state.db" +CLEANUP_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_cleanup.sh" +MANAGER_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_manager.sh" -# Bounded log — keeps last 90 days -TRANSCODE_LOG_RETENTION=90 - -touch "$TRANSCODE_DAILY_LOG" 2>/dev/null - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing to child scripts" +acquire_lock "wait" -acquire_lock +# detect_hosts() sets MY_ID and aliases all HOST*_TRANSCODE_* vars +detect_hosts -# ----------------------------------------------------------------------------------------------- -# HELPERS -# ----------------------------------------------------------------------------------------------- +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing through to child scripts" -# Read a value from transcode state file -state_get() { - grep "^${1}=" "$TRANSCODE_STATE_FILE" 2>/dev/null | cut -d= -f2 -} - -# Get today's date key -today() { - date '+%Y-%m-%d' -} - -# Update daily log entry for today -# Format: YYYY-MM-DD|peak_gb|flip_count|ram_sessions|ssd_sessions|files_cleaned -update_daily_log() { - local peak_gb="$1" - local flips="$2" - local ram_sessions="$3" - local ssd_sessions="$4" - local files_cleaned="$5" - local today_key - today_key=$(today) - - local existing - existing=$(grep "^${today_key}|" "$TRANSCODE_DAILY_LOG" 2>/dev/null) - - if [[ -z "$existing" ]]; then - # New entry for today - echo "${today_key}|${peak_gb}|${flips}|${ram_sessions}|${ssd_sessions}|${files_cleaned}" \ - >> "$TRANSCODE_DAILY_LOG" +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY TRANSCODE MANAGEMENT STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_RAM Ramdisk: $RAMDISK_PATH ($RAMDISK_SIZE)" + echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD" + echo "$ICON_LINK Symlink: $TRANSCODE_LINK" + echo "$ICON_TIME Schedule: every 3 minutes" + echo "" + echo "━━━ Child Scripts ━━━" + [[ -f "$CLEANUP_SCRIPT" ]] && \ + echo " $ICON_SUCCESS transcode_cleanup.sh — found" || \ + echo " $ICON_ERROR transcode_cleanup.sh — NOT FOUND at $CLEANUP_SCRIPT" + [[ -f "$MANAGER_SCRIPT" ]] && \ + echo " $ICON_SUCCESS transcode_manager.sh — found" || \ + echo " $ICON_ERROR transcode_manager.sh — NOT FOUND at $MANAGER_SCRIPT" + echo "" + echo "━━━ Daily Log ━━━" + if [[ -f "${TRANSCODE_DAILY_LOG:-}" ]] && [[ -s "$TRANSCODE_DAILY_LOG" ]]; then + ENTRY_COUNT=$(wc -l < "$TRANSCODE_DAILY_LOG") + OLDEST=$(awk -F'|' 'NR==1{print $1}' "$TRANSCODE_DAILY_LOG") + NEWEST=$(awk -F'|' 'END{print $1}' "$TRANSCODE_DAILY_LOG") + echo " $ICON_SUCCESS $TRANSCODE_DAILY_LOG ($ENTRY_COUNT entries, $OLDEST → $NEWEST)" else - # Update existing — keep highest peak, accumulate flips, sessions, files - local old_peak old_flips old_ram old_ssd old_files - old_peak=$(echo "$existing" | cut -d'|' -f2) - old_flips=$(echo "$existing" | cut -d'|' -f3) - old_ram=$(echo "$existing" | cut -d'|' -f4) - old_ssd=$(echo "$existing" | cut -d'|' -f5) - old_files=$(echo "$existing" | cut -d'|' -f6) - - # Peak — keep highest - local new_peak - new_peak=$(awk "BEGIN {print ($peak_gb > $old_peak) ? $peak_gb : $old_peak}") - - # Accumulate - local new_flips=$(( old_flips + flips )) - local new_ram=$(( old_ram + ram_sessions )) - local new_ssd=$(( old_ssd + ssd_sessions )) - local new_files=$(( old_files + files_cleaned )) - - # Replace line - sed -i "s|^${today_key}|.*|${today_key}|${new_peak}|${new_flips}|${new_ram}|${new_ssd}|${new_files}|" \ - "$TRANSCODE_DAILY_LOG" 2>/dev/null || { - # sed replacement failed — remove and re-add - sed -i "/^${today_key}|/d" "$TRANSCODE_DAILY_LOG" - echo "${today_key}|${new_peak}|${new_flips}|${new_ram}|${new_ssd}|${new_files}" \ - >> "$TRANSCODE_DAILY_LOG" - } + echo " $ICON_SKIP $TRANSCODE_DAILY_LOG — no data yet" fi + echo "" + echo "━━━ Current State ━━━" + if [[ -f "${TRANSCODE_STATE_FILE:-/tmp/transcode_state.db}" ]]; then + while IFS='=' read -r key val; do + [[ -n "$key" ]] && echo " $key = $val" + done < "${TRANSCODE_STATE_FILE:-/tmp/transcode_state.db}" + else + echo " State file not found (ramdisk_setup.sh creates it at array start)" + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi - # Purge old entries beyond retention - local cutoff - cutoff=$(date -d "${TRANSCODE_LOG_RETENTION} days ago" '+%Y-%m-%d') - awk -F'|' -v cutoff="$cutoff" '$1 >= cutoff' \ - "$TRANSCODE_DAILY_LOG" > "${TRANSCODE_DAILY_LOG}.tmp" && \ - mv "${TRANSCODE_DAILY_LOG}.tmp" "$TRANSCODE_DAILY_LOG" -} - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SYNC Run Cleanup ━━━ -# ----------------------------------------------------------------------------------------------- -CLEANUP_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_cleanup.sh" -MANAGER_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_manager.sh" - +# ============================================================================================== +# ━━━ Validate Child Scripts ━━━ +# ============================================================================================== if [[ ! -f "$CLEANUP_SCRIPT" ]]; then error "transcode_cleanup.sh not found: $CLEANUP_SCRIPT" exit 1 @@ -137,50 +132,26 @@ if [[ ! -f "$MANAGER_SCRIPT" ]]; then exit 1 fi -# Capture cleanup output for file count -CLEANUP_OUTPUT=$(bash "$CLEANUP_SCRIPT" $([[ "$DRY_RUN" == true ]] && echo "--dry-run") 2>&1) +# ============================================================================================== +# ━━━ Run Cleanup ━━━ +# ============================================================================================== +DRY_FLAG="" +[[ "$DRY_RUN" == true ]] && DRY_FLAG="--dry-run" + +bash "$CLEANUP_SCRIPT" $DRY_FLAG CLEANUP_EXIT=$? -echo "$CLEANUP_OUTPUT" -# Extract files cleaned from cleanup output -FILES_CLEANED=$(echo "$CLEANUP_OUTPUT" | grep -oE "Removed [0-9]+ file" | grep -oE "[0-9]+" | head -1) -FILES_CLEANED="${FILES_CLEANED:-0}" - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_WATCHDOG Run Manager ━━━ -# ----------------------------------------------------------------------------------------------- -MANAGER_OUTPUT=$(bash "$MANAGER_SCRIPT" $([[ "$DRY_RUN" == true ]] && echo "--dry-run") 2>&1) +# ============================================================================================== +# ━━━ Run Manager ━━━ +# ============================================================================================== +# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run +# No --no-log flag here — manager owns the log write for this cycle ✅ +bash "$MANAGER_SCRIPT" $DRY_FLAG MANAGER_EXIT=$? -echo "$MANAGER_OUTPUT" -# ----------------------------------------------------------------------------------------------- -# Collect stats for daily log -# ----------------------------------------------------------------------------------------------- -# Ramdisk usage from state file -RAMDISK_USED_GB=$(state_get "ramdisk_used_gb" 2>/dev/null || echo "0") -[[ -z "$RAMDISK_USED_GB" || "$RAMDISK_USED_GB" == "0" ]] && \ - RAMDISK_USED_GB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | \ - awk '{printf "%.2f", $1/1048576}' || echo "0") - -# Flip count from state file -FLIP_COUNT=$(state_get "flip_count_hour" 2>/dev/null || echo "0") -FLIP_COUNT="${FLIP_COUNT:-0}" - -# Session counts from manager output -RAM_SESSIONS=$(echo "$MANAGER_OUTPUT" | grep -oE "ramdisk \([0-9]+\)" | grep -oE "[0-9]+" | head -1) -SSD_SESSIONS=$(echo "$MANAGER_OUTPUT" | grep -oE "SSD \([0-9]+\)" | grep -oE "[0-9]+" | head -1) -RAM_SESSIONS="${RAM_SESSIONS:-0}" -SSD_SESSIONS="${SSD_SESSIONS:-0}" - -# Update daily log -if [[ "$DRY_RUN" == false ]]; then - update_daily_log "$RAMDISK_USED_GB" "$FLIP_COUNT" "$RAM_SESSIONS" "$SSD_SESSIONS" "$FILES_CLEANED" -fi - -# ----------------------------------------------------------------------------------------------- -# Exit with worst exit code -# ----------------------------------------------------------------------------------------------- -if [[ "$CLEANUP_EXIT" -ne 0 || "$MANAGER_EXIT" -ne 0 ]]; then - exit 1 -fi +# ============================================================================================== +# ━━━ Exit ━━━ +# ============================================================================================== +# Return worst exit code — caller knows if either script failed +[[ "$CLEANUP_EXIT" -ne 0 || "$MANAGER_EXIT" -ne 0 ]] && exit 1 exit 0 \ No newline at end of file diff --git a/Orchestrators/weekly_sync_maintenance.sh b/Orchestrators/weekly_sync_maintenance.sh index f2cffe7..e4d3fe6 100644 --- a/Orchestrators/weekly_sync_maintenance.sh +++ b/Orchestrators/weekly_sync_maintenance.sh @@ -1,99 +1,193 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# ----------------------------- Weekly Sync Maintenance ---------------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ============================= Weekly Sync Maintenance ======================================== +# ============================================================================================== # Weekly maintenance window orchestrator — clean sync, container updates, weekly restarts. -# Schedule: 30 2 * * 0 (Sunday 2:30am — fits before 3am network reboot) +# Schedule: 30 2 * * 0 (Sunday 2:30am — before Sunday 7am coffee report) # -# Execution order: -# 1. Stop local containers — Emby + auth stack stopped locally -# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH -# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true -# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true -# 5. rsync Emby — full clean mirror, both instances stopped -# 6. rsync Critical-Data — auth stack clean sync, databases flushed -# 7. Start remote containers — correct order, delayed start respected -# 8. Start local containers — correct order, delayed start respected -# 9. docker_weekly_restart.sh — weekly container restarts +# ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── +# 1. Stop local containers — Emby + auth stack stopped locally +# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH +# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true (zero extra downtime) +# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true +# 5. rsync WEEKLY_SYNC_SHARES — full clean mirror, containers stopped both sides +# 6. Start remote containers — correct order, delayed start respected +# 7. Start local containers — correct order, delayed start respected +# 8. WEEKLY_MAINTENANCE_SCRIPTS — weekly restarts etc. (docker_weekly_restart.sh) # -# Synced shares (WEEKLY_SYNC_SHARES in Master.conf): -# /mnt/user/Media_Server/Emby — emby profile — full mirror, cache resets weekly -# /mnt/user/appdata-Failover/Critical-Data — critical-data — auth stack clean state +# ── WHY WEEKLY NOT NIGHTLY FOR EMBY ────────────────────────────────────────────────────────── +# Emby builds a warm image cache on HOST2 throughout the week. +# Syncing nightly resets cache — cold loads every morning for users. +# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep. +# emby-failover dirty sync covers watch states + library every 15min between weekly syncs. # -# Why weekly instead of nightly for Emby: -# Emby builds a warm image cache on HOST2 throughout the week -# Syncing nightly resets cache — cold loads every morning for users -# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep -# emby-failover dirty sync covers watch states + library every 30-60min between syncs +# ── CONTAINER UPDATES ───────────────────────────────────────────────────────────────────────── +# Containers already stopped for sync — updates pull at zero extra downtime. +# Both servers start on identical image versions after the window completes. +# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in master.conf # -# Container updates during the window: -# Containers already stopped for sync — updates pull at zero extra downtime -# Both servers start on identical image versions after the window completes -# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in Master.conf +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID — used in banner, summary, and notifications. +# WEEKLY_SYNC_SHARES and WEEKLY_MAINTENANCE_SCRIPTS configured in master.conf. +# Same script runs correctly on both servers. # -# What triggers weekly_health_digest.sh: -# NOT this script — weekly_health_digest.sh runs on its own Saturday schedule +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# Root check — stop/start containers, rsync require root +# acquire_lock — prevents concurrent weekly windows +# check_connectivity — verifies remote before any remote operations +# check_remote_rootfs — aborts if remote rootfs nearly full +# DOCKER_TIMEOUT — all docker calls protected +# SSH_TIMEOUT — all SSH calls protected +# validate_unraid_cmd — notify validated before use +# Silent on success — runs weekly, only failures warrant notification # -# Configuration in Master.conf: -# WEEKLY_SYNC_SHARES — shares synced during the maintenance window -# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync (docker_weekly_restart) -# WEEKLY_SYNC_UPDATES — toggle container updates on/off -# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates on/off -# ----------------------------------------------------------------------------------------------- -# All configuration in Master.conf. -# Supports --dry-run to walk through without stopping containers, syncing, or updating. -# ----------------------------------------------------------------------------------------------- +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# WEEKLY_SYNC_SHARES — shares synced during window +# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync +# WEEKLY_SYNC_UPDATES — toggle local container updates +# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates +# WEEKLY_RSYNC_ENABLED — enable/disable rsync section +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# weekly_sync_maintenance.sh — normal run +# weekly_sync_maintenance.sh --dry-run — preview without stopping containers or syncing +# weekly_sync_maintenance.sh --log — verbose per-share/per-job output +# weekly_sync_maintenance.sh --status — show configuration and exit +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh" +SCRIPTS_ROOT="$SCRIPT_DIR/.." parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" +DOCKER_TIMEOUT=30 # container stop/start needs longer than normal +SSH_TIMEOUT=30 # remote pulls can be slow +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi -success "Running as root" +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" acquire_lock detect_hosts resolve_remote_ip -# Load container list from emby profile — used for update pulls -read -r -a MAINTENANCE_CONTAINERS <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}" +WINDOW_START=$(date +%s) +PASS=() +FAIL=() +JOB_PASS=() +JOB_FAIL=() -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SHIELD Pre-flight Checks ━━━ -# ----------------------------------------------------------------------------------------------- +# Load container lists from profile config +read -r -a MAINTENANCE_CONTAINERS <<< \ + "${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}" + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates" + +# ── Helper — run a post-sync maintenance script ──────────────────────────────────────────────── +run_job() { + local script_entry="$1" + local extra_dry="" + [[ "$DRY_RUN" == true ]] && extra_dry="--dry-run" + + read -r -a script_args <<< "$script_entry" + local script_path="$SCRIPTS_ROOT/${script_args[0]}" + local script_name + script_name=$(basename "${script_args[0]}") + local extra_args=("${script_args[@]:1}") + + if [[ ! -f "$script_path" ]]; then + error "$script_name — not found at $script_path" + JOB_FAIL+=("$script_name") + return 1 + fi + + log "Running: $script_name ${extra_args[*]}" + # shellcheck disable=SC2086 + if bash "$script_path" "${extra_args[@]}" $extra_dry; then + log "$script_name — done ✅" + JOB_PASS+=("$script_name ${extra_args[*]}") + else + error "$script_name — failed (exit $?)" + JOB_FAIL+=("$script_name ${extra_args[*]}") + fi +} + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)" + echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}" + echo "$ICON_SYNC Weekly enabled: ${WEEKLY_RSYNC_ENABLED:-false}" + echo "$ICON_GEAR Local updates: ${WEEKLY_SYNC_UPDATES:-false}" + echo "$ICON_GEAR Remote updates: ${WEEKLY_SYNC_UPDATES_REMOTE:-false}" + echo "" + echo "━━━ Weekly Sync Shares ━━━" + if [[ ${#WEEKLY_SYNC_SHARES[@]} -eq 0 ]]; then + warn " No WEEKLY_SYNC_SHARES configured" + else + for share in "${WEEKLY_SYNC_SHARES[@]:-}"; do + [[ -n "$share" ]] && echo " $ICON_SYNC $(basename "$share") ($share)" + done + fi + echo "" + echo "━━━ Weekly Maintenance Scripts ━━━" + if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then + echo " None configured" + else + for entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]:-}"; do + [[ -n "$entry" ]] && echo " $ICON_GEAR $(basename "${entry%% *}") ${entry#* }" + done + fi + echo "" + echo "━━━ Containers (from profile config) ━━━" + for c in "${MAINTENANCE_CONTAINERS[@]:-}"; do + [[ -n "$c" ]] && echo " $ICON_CONTAINERS $c" + done + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Pre-flight Checks ━━━ +# ============================================================================================== echo "" -echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━" +echo "━━━ $ICON_GEAR Weekly Sync Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "" +echo "━━━ $ICON_SHIELD Pre-flight ━━━" check_connectivity check_remote_rootfs -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Stop Containers ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — containers will not be stopped" else - # Load critical-data + emby container list for stops - read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}" + # Load container lists for stop functions + read -r -a CRITICAL_CONTAINER_NAMES <<< \ + "${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}" read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}" CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}" @@ -101,123 +195,118 @@ else stop_containers fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Container Updates ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Container Updates ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_GEAR Container Updates ━━━" if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would pull updates for local containers" - for c in "${MAINTENANCE_CONTAINERS[@]}"; do - [[ -z "$c" ]] && continue - warn "DRY RUN — would docker pull: $c" + for c in "${MAINTENANCE_CONTAINERS[@]:-}"; do + [[ -n "$c" ]] && warn "DRY RUN — would pull: $c" done else - info "Pulling local container updates..." - for c in "${MAINTENANCE_CONTAINERS[@]}"; do + log "Pulling local container updates..." + for c in "${MAINTENANCE_CONTAINERS[@]:-}"; do [[ -z "$c" ]] && continue - # Get image name from running or stopped container - IMAGE=$(docker inspect "$c" --format '{{.Config.Image}}' 2>/dev/null) + IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \ + "$c" --format '{{.Config.Image}}' 2>/dev/null) if [[ -z "$IMAGE" ]]; then log "$c — not found locally, skipping update" continue fi - info "Pulling $IMAGE for $c..." + log "Pulling $IMAGE for $c..." if docker pull "$IMAGE" >/dev/null 2>&1; then - success "$c — image updated" + log "$c — image updated ✅" else warn "$c — pull failed, will start on existing image" fi done fi else - info "WEEKLY_SYNC_UPDATES=false — skipping local updates" + log "WEEKLY_SYNC_UPDATES=false — skipping local updates" fi if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME" else - info "Pulling remote container updates on $REMOTE_SERVER_NAME..." - for c in "${MAINTENANCE_CONTAINERS[@]}"; do + log "Pulling remote container updates on $REMOTE_SERVER_NAME..." + for c in "${MAINTENANCE_CONTAINERS[@]:-}"; do [[ -z "$c" ]] && continue - IMAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + IMAGE=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" \ + root@"$REMOTE_SERVER" \ "docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null) if [[ -z "$IMAGE" ]]; then log "$c — not found on remote, skipping update" continue fi - info "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..." - if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker pull $IMAGE" >/dev/null 2>&1; then - success "$c — remote image updated" + log "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..." + if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" \ + root@"$REMOTE_SERVER" \ + "docker pull $IMAGE" >/dev/null 2>&1; then + log "$c — remote image updated ✅" else warn "$c — remote pull failed, will start on existing image" fi done fi else - info "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates" + log "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SYNC Critical Shares Sync ━━━ -# ----------------------------------------------------------------------------------------------- -PASS=() -FAIL=() -TOTAL_START=$(date +%s) - -SYNC_JOBS=("${WEEKLY_SYNC_SHARES[@]}") -SHARE_COUNT=${#SYNC_JOBS[@]} +# ============================================================================================== +# ━━━ Critical Shares Sync ━━━ +# ============================================================================================== +SHARE_COUNT=${#WEEKLY_SYNC_SHARES[@]} echo "" -echo "━━━ $ICON_SYNC Critical Shares Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -echo "$ICON_SUMMARY Jobs: $SHARE_COUNT" -echo "" +echo "━━━ $ICON_SYNC Critical Shares Sync — $SHARE_COUNT share(s) ━━━" +SYNC_START=$(date +%s) JOB_NUM=0 -# Tier 1 + Tier 2 rsync gate check if ! check_rsync_enabled "WEEKLY"; then - warn "Rsync disabled — skipping all $SHARE_COUNT weekly sync jobs" - warn "Proceeding to container updates and maintenance scripts..." + warn "Weekly rsync disabled — skipping all $SHARE_COUNT sync job(s)" + warn "Proceeding to container start and maintenance scripts..." +elif [[ "$SHARE_COUNT" -eq 0 ]]; then + warn "No WEEKLY_SYNC_SHARES configured — skipping sync" + warn "Check WEEKLY_SYNC_SHARES in master.conf" else - for JOB in "${SYNC_JOBS[@]}"; do - ((JOB_NUM++)) + RSYNC_DRY="" + [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run" + + for JOB in "${WEEKLY_SYNC_SHARES[@]}"; do + (( JOB_NUM++ )) JOB_NAME=$(basename "$JOB") + + echo "" echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━" - JOB_START=$(date +%s) + JOB_START=$(date +%s) + bash "$RSYNC_SCRIPT" "$JOB" $RSYNC_DRY + EXIT_CODE=$? + JOB_DUR=$(format_duration $(( $(date +%s) - JOB_START ))) - # Containers already stopped — rsync profile won't try to stop them again - # Pass --no-container-stop flag would be ideal but profiles handle this naturally - # since containers are already stopped, stop_containers finds nothing running - if [[ "$DRY_RUN" == true ]]; then - bash "$RSYNC_SCRIPT" "$JOB" --dry-run - else - bash "$RSYNC_SCRIPT" "$JOB" - fi + if [[ "$EXIT_CODE" -eq 0 ]]; then + PASS+=("$JOB_NAME") + log "$JOB_NAME — done in $JOB_DUR ✅" + else + FAIL+=("$JOB_NAME") + error "$JOB_NAME — failed after $JOB_DUR (exit $EXIT_CODE)" + fi + echo "" + done +fi - EXIT_CODE=$? - JOB_END=$(date +%s) - JOB_DURATION=$(format_duration $(( JOB_END - JOB_START ))) +SYNC_END=$(date +%s) - if [[ "$EXIT_CODE" -eq 0 ]]; then - PASS+=("$JOB_NAME") - success "$JOB_NAME — $ICON_SUCCESS done in $JOB_DURATION" - else - FAIL+=("$JOB_NAME") - error "$JOB_NAME — $ICON_ERROR failed after $JOB_DURATION" - fi - - echo "" -done -fi # end check_rsync_enabled "WEEKLY" - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Start Containers ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━" @@ -228,88 +317,57 @@ else start_local_containers fi -TOTAL_END=$(date +%s) -TOTAL_DURATION=$(format_duration $(( TOTAL_END - TOTAL_START ))) - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Post-sync Jobs ━━━ -# docker_weekly_restart.sh and any other WEEKLY_MAINTENANCE_SCRIPTS run after sync -# ----------------------------------------------------------------------------------------------- -JOB_PASS=() -JOB_FAIL=() -SCRIPTS_ROOT="$SCRIPT_DIR/.." - +# ============================================================================================== +# ━━━ Post-sync Jobs ━━━ +# ============================================================================================== if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then echo "" echo "━━━ $ICON_GEAR Post-sync Jobs ━━━" - - for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do + for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]:-}"; do [[ -z "$script_entry" ]] && continue - script_args=($script_entry) - script_path="$SCRIPTS_ROOT/${script_args[0]}" - script_name=$(basename "${script_args[0]}") - extra_args=("${script_args[@]:1}") - echo "" - info "$ICON_START Running: $script_name" - - if [[ ! -f "$script_path" ]]; then - error "$script_name — not found at $script_path" - JOB_FAIL+=("$script_name") - continue - fi - - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would run: $script_name" - JOB_PASS+=("$script_name (dry run)") - elif bash "$script_path" "${extra_args[@]}"; then - success "$script_name — done" - JOB_PASS+=("$script_name") - else - error "$script_name — failed" - JOB_FAIL+=("$script_name") - fi + run_job "$script_entry" done fi WINDOW_END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━" -echo "$ICON_TIME Duration: $TOTAL_DURATION" -echo "$ICON_GEAR Updates: local=$WEEKLY_SYNC_UPDATES remote=$WEEKLY_SYNC_UPDATES_REMOTE" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')" +echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))" +echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}" echo "" -echo "$ICON_SYNC Sync jobs:" -if [[ ${#PASS[@]} -gt 0 ]]; then - for job in "${PASS[@]}"; do echo " $ICON_SUCCESS $job"; done -fi -if [[ ${#FAIL[@]} -gt 0 ]]; then - for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done -fi -echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}" +echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):" +for job in "${PASS[@]:-}"; do echo " $ICON_DONE $job"; done +for job in "${FAIL[@]:-}"; do echo " $ICON_ERROR $job"; done +echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}" if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then echo "" echo "$ICON_GEAR Post-sync jobs:" - for job in "${JOB_PASS[@]}"; do echo " $ICON_SUCCESS $job"; done - for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done + for job in "${JOB_PASS[@]:-}"; do echo " $ICON_DONE $job"; done + for job in "${JOB_FAIL[@]:-}"; do echo " $ICON_ERROR $job"; done fi -echo "" - TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} )) +echo "" if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no changes made" + warn "DRY RUN — no changes made" elif [[ "$TOTAL_FAIL" -eq 0 ]]; then - echo "$ICON_DONE Status: $ICON_SUCCESS ALL COMPLETE" - notify "Weekly sync maintenance complete on $(hostname) — synced + updated (local=$WEEKLY_SYNC_UPDATES remote=$WEEKLY_SYNC_UPDATES_REMOTE)" "Weekly Maintenance" "normal" + log "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run" else - echo "$ICON_ERROR Status: $TOTAL_FAIL failure(s) — check logs" - notify "Weekly sync maintenance failed on $(hostname) — sync: ${#FAIL[@]} failed, jobs: ${#JOB_FAIL[@]} failed" "Weekly Maintenance" "warning" + warn "Status: $TOTAL_FAIL failure(s)" + notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \ + "Weekly Maintenance" "warning" fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/Partnership/README-Partnership.md b/Partnership/README-Partnership.md index 5cadc9b..6b9d4fd 100644 --- a/Partnership/README-Partnership.md +++ b/Partnership/README-Partnership.md @@ -1,438 +1,945 @@ -# Partnership +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# 🤝 PARTNERSHIP +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Manages the relationship lifecycle between two unRAID servers. Handles onboarding a new mirror, clean separation when someone wants to leave, and transferring ownership when the arrangement changes. +**Manages the full lifecycle of a two-server partnership — setup, daily operation, +clean separation, and ownership transfer.** One server owns the shared services. +The other mirrors them and benefits from them. The relationship has a beginning, +an ongoing operation, and an end — and every phase deserves the same engineering +discipline as the rest of the ecosystem. -> This script exists because the partnership between two servers has a lifecycle — and that lifecycle deserves the same engineering discipline as everything else in the ecosystem. A clean exit should be as easy as a clean setup. +> **This script exists because a clean exit should be as easy as a clean setup.** +> The partnership is not a permanent commitment. `--offboard` works from either server +> at any time. Everything HOST2 needs to run independently is already on HOST2. +> The only thing that stops working immediately is the sync — and that's intentional. --- -## The Relationship Model - -Two servers. One owns the shared services. One mirrors them. - -``` -HOST1 (owner) — source of truth - Auth stack ← all changes made here - NPM proxy rules ← created here, mirrored to mirror - Certs ← renewed here, mirrored to mirror - Emby ← runs here, dirty-synced every 15min - Master.conf ← source of truth, git push propagates - -HOST2 (mirror) — warm copy - Auth containers ← running, serving his domain - NPM ← running owner's config - Certs ← current, mirrored every 15min - WebUIs ← click any auth container → opens owner's WebUI via Tailscale - Never makes changes directly ← overwritten on next sync -``` - -**HOST1 is always the owner. HOST2 is always the mirror.** -`PARTNERSHIP_OWNER_HOST="HOST1"` in Master.conf — flips to `"HOST2"` only after `--transfer`. -Everything derives from HOST1/HOST2 — no duplicate IP or key configuration needed. - -**The mirror never needs to think about auth management.** His Docker UI opens, he clicks an auth container, lands on the owner's WebUI automatically via Tailscale. Changes made there sync to his server in 15 minutes. He benefits from every improvement the owner makes without doing anything. - -**The owner never needs to think about the mirror's server.** Everything propagates automatically. Critical-Data syncs every 15 minutes via `critical_sync_maintenance.sh`. Certs stay current. Config stays consistent. +## ━━━ THE PROBLEM THAT BUILT THIS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ --- -## What Makes It Work +### 🔴 Two Servers, One Auth Stack, No Clean Way to Share It -Three things make the mirror transparent to end users: +The auth stack — NginxProxyManager, LLDAP, Authelia, MariaDB, Redis — runs on HOST1. +HOST2 serves his own domain to his own household. He needs his own auth. But maintaining +two independent auth stacks means double the work: two places to add users, two places to +update proxy rules, two places to renew certs, two configurations that inevitably drift +apart. One change made on HOST1 has to be manually replicated to HOST2 — or it isn't, +and the configurations diverge silently. -**1. Auth stack warm on both servers** +The real cost isn't the initial setup. It's the maintenance burden that accumulates over +months — every new user, every proxy rule change, every cert renewal, every configuration +update applied in one place and forgotten in the other. -NPM, LLDAP, Authelia, certs — all running on both servers simultaneously. When traffic hits the mirror's domain, auth is already ready. No cold start, no broken auth window. - -**2. WebUIs pointing to owner** - -Every auth container on the mirror has its WebUI URL configured to point at the owner's Tailscale IP. From the mirror operator's perspective: click container, get owner's UI. From the owner's perspective: one place to manage everything. - -**3. Tailscale-only communication** - -All traffic between servers goes through Tailscale. Encrypted, no open ports, no VPN configuration. The WebUI redirect works because Tailscale keeps both servers permanently connected. HOST1 and HOST2 hostnames must match their exact Tailscale device names — already enforced by the ecosystem. +The fix: one auth stack with a managed mirror. HOST1 owns the configuration. HOST2 runs +a warm copy that stays current via 15-minute sync. HOST2's operator makes zero auth +management decisions — he clicks an auth container, gets HOST1's WebUI via Tailscale, +changes happen there, propagate to his server in 15 minutes. One place to manage +everything for both households. --- -## State Files +### 🔴 No Structure Around the Relationship Itself -Partnership state is tracked in two files on `/boot/config` — survives reboots, available before the array starts, minimal flash wear. +Setting up the mirror was a manual process. SSH in, reconfigure container WebUI URLs +one by one, copy auth config, verify connectivity, update state tracking. No defined +sequence. No dry-run capability. No verification that each step worked. If something +went wrong midway, the mirror was in an inconsistent state with no clear way to +understand what had and hadn't been done. -``` -/boot/config/partnership_HOST1.db ← HOST1 writes only -/boot/config/partnership_HOST2.db ← HOST2 writes only -``` +The same problem on offboard — worse, actually. Because offboard involves stopping a sync +that's been running for months, making a final copy of data, reconfiguring WebUIs back +to local addresses, removing Tailscale access, and notifying both servers. A manual +process with this many steps, taken under pressure at the moment of separation, +is the kind of process that leaves one or both parties in a bad state. -Each server writes **only its own file**. State propagates via SSH — no rsync needed. `critical_sync_maintenance.sh` calls `partnership_manage.sh --check` every 15 minutes, which SSHes to read the remote file and act on any changes. - -**This is how deferred offboard works:** HOST2 offboards while HOST1 is temporarily unreachable → HOST2 writes its state file → next time HOST1 can reach HOST2 → reads HOST2's state → finalises from its side automatically. - -A per-server offline counter (`/boot/config/partnership_offline_days.db`) tracks consecutive missed 15-minute cycles. After `PARTNERSHIP_OFFLINE_THRESHOLD` days either server is unreachable, both independently auto-offboard. +The fix: `partnership_manage.sh` with explicit modes for each lifecycle phase. Each mode +is a defined sequence. Every step is verified. Dry-run shows exactly what will happen +before anything changes. State files make the current relationship status unambiguous +from either server. --- -## The Lifecycle +### 🔴 No Safe Way to Check If the Other Server Has Gone Away -### Onboard — Setting Up a New Mirror +After months of operation, HOST2 goes quiet. Maybe a hardware failure. Maybe the +person moved. Maybe a forgotten reboot that never completed. The sync starts failing. +The offline counter increments. But nothing actually happens — the ecosystem just keeps +failing the same sync, incrementing the same counter, sending the same notifications. -Run `--onboard` from HOST1 (owner) to establish the relationship: +Without a defined threshold and an automated response, "partner gone for 30 days" +looks exactly like "partner gone for 3 years" from the ecosystem's perspective. +Both require manual intervention to clean up. + +The fix: `PARTNERSHIP_OFFLINE_THRESHOLD`. After this many days of missed sync cycles, +both servers independently auto-offboard. HOST1 removes HOST2 from Tailscale, disables +critical sync, writes INACTIVE state. HOST2 — if it eventually comes back — reads HOST1's +INACTIVE state and cleans up its own side. The relationship is formally ended from both +sides without anyone needing to be physically present. + +--- + +### 🔴 Ownership Transfer Had No Safe Path + +The arrangement was always intended to be flexible — HOST1 owns the auth stack now, but +circumstances change. HOST2 operator might want to take ownership. Or HOST1 might move +and HOST2's location becomes more appropriate for the primary. Swapping ownership manually +meant reconfiguring WebUIs on both servers, swapping sync direction, updating master.conf +on both servers, and hoping the sequence was correct. + +A misstep midway through — like flipping sync direction before the final sync completed — +could leave both servers with different auth configurations and no clear source of truth. + +The fix: `--transfer` with a required confirmation string, a consecutive health check +system, and a strict sequence. The confirmation string is as long and specific as +`--i-know-what-im-doing` in the arr cleanup scripts — you cannot type it accidentally. +Health strikes require both servers to be healthy on multiple consecutive checks before +the transfer begins. The sequence guarantees a final sync in the current direction before +anything is flipped. + +--- + +## ━━━ THE RELATIONSHIP MODEL ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +``` +HOST1 (owner) HOST2 (mirror) +──────────────── ──────────────────────────── +Source of truth Warm copy — always current +Auth stack config Auth containers running +NPM proxy rules NPM serving his domain +LLDAP users LLDAP — same users +Authelia policies Authelia — same policies +Certs Certs — mirrored, valid +Emby Emby — dirty-synced every 15min + +Changes made here ──→ Propagated every 15 minutes +WebUI management ──→ Redirected to HOST1 via Tailscale +Config → git push ──→ Received via git pull on next cycle +``` + +--- + +### ── What the Mirror's Daily Experience Looks Like ─────────────────────────── ```bash -# Always dry-run first +# ───────────────────────────────────────────────────────────────────────────── +# HOST2 operator opens the unRAID Docker UI. +# Clicks NginxProxyManager. +# Browser opens HOST1's NPM WebUI automatically (via Tailscale redirect). +# Makes a proxy rule change. +# 15 minutes later it's live on HOST2. +# +# HOST2 operator never: +# Opens HOST2's NPM directly (WebUI points to HOST1) +# Manages LLDAP users on HOST2 (WebUI points to HOST1) +# Touches Authelia config on HOST2 (WebUI points to HOST1) +# Renews HOST2's certs manually (Critical-Data sync handles it) +# +# HOST2 operator always benefits from: +# Every proxy rule HOST1 adds +# Every user HOST1 adds to LLDAP +# Every security policy HOST1 configures +# Every cert HOST1 renews +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── What the Owner's Daily Experience Looks Like ──────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# HOST1 operator manages auth as normal: +# Opens NPM → adds a proxy rule → 15min later live on both servers +# Opens LLDAP → adds a user → 15min later exists on both servers +# Renews a cert → 15min later current on both servers +# +# HOST1 operator never needs to: +# SSH to HOST2 to apply changes +# Coordinate with HOST2 operator on auth changes +# Know what HOST2's auth config currently looks like +# Think about HOST2 at all during normal operation +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── What Makes the Redirect Transparent ───────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Three things work together to make the auth stack transparent to HOST2: +# +# 1. Auth stack warm on both servers always +# NPM, LLDAP, Authelia, MariaDB, Redis — all running on HOST2 continuously. +# When HOST2's domain gets traffic, auth is ready immediately. +# No failover delay, no cold start. +# +# 2. WebUI URLs pointing to HOST1 +# unRAID stores WebUI URLs in the Docker template. +# partnership_manage.sh --onboard edits these templates on HOST2. +# "NginxProxyManager|81" → URL becomes http://[HOST1-tailscale-ip]:81 +# HOST2 operator clicks the container → gets HOST1's WebUI. +# All changes land on HOST1, propagate to HOST2 automatically. +# +# 3. Tailscale permanent connection +# Both servers permanently connected via Tailscale encrypted mesh. +# The WebUI redirect works because Tailscale IP is always reachable. +# No VPN configuration needed. No open ports. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── The One WebUI That Stays Local ────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Tailscale WebUI is left pointing to HOST2's own local interface. +# When diagnosing connectivity issues between servers you need to see +# HOST2's network state from its own perspective — not HOST1's view. +# It is the one WebUI that genuinely needs to stay local. +# Every other auth container WebUI redirects to HOST1. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +## ━━━ STATE FILES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Each server writes ONLY its own state file. Neither server writes the other's. +# State propagates via SSH reads — no rsync, no shared filesystem. +# +# /boot/config/partnership_HOST1.db ← HOST1 writes only, HOST2 reads via SSH +# /boot/config/partnership_HOST2.db ← HOST2 writes only, HOST1 reads via SSH +# +# /boot/config — survives reboots, available before array starts, minimal flash wear. +# flock on all writes — prevents concurrent corruption from overlapping --check cycles. +# +# Example state file content: +state=ACTIVE +last_updated=2026-04-14 03:00:00 +last_seen_remote=2026-04-14 03:00:00 +owner_host=HOST1 +offline_days=0 +# ───────────────────────────────────────────────────────────────────────────── +``` + +**Why this design enables deferred offboard:** +HOST2 offboards while HOST1 is temporarily unreachable — writes its own state file to +INACTIVE. When HOST1 can next reach HOST2 (next `--check` cycle), it reads the INACTIVE +state and finalises the offboard from its side automatically. No coordination required. +No message passing. Each server acts on what it can observe. + +--- + +## ━━━ THE LIFECYCLE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🟢 --onboard — Establishing the Partnership +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Run from HOST1 (owner) only. Establishes the mirror relationship. After onboard, +HOST2's auth WebUIs redirect to HOST1 and Critical-Data sync begins keeping them current. + +> **Always dry-run first.** Onboard modifies container templates on HOST2 and verifies +> WebUI connectivity. The dry-run shows exactly which templates would be changed and +> what the redirect URLs would become — review before committing. + +--- + +### ── Onboard Sequence ───────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Step 1 — Pre-flight checks +# Both servers healthy (array started, Docker responding) +# Tailscale connected — HOST2 reachable via Tailscale IP +# unRAID version parity — both servers on compatible versions +# HOST2 auth containers exist (may be stopped — that's fine) +# → Any failure: abort, explain what failed, no changes made +# +# Step 2 — Reconfigure HOST2 WebUI templates +# For each container in PARTNERSHIP_AUTH_WEBUIS: +# Find template in /boot/config/plugins/dockerMan/templates-user/ +# Update WebUI URL → http://[HOST1-TAILSCALE-IP]:PORT +# Container restart not needed — template change takes effect on next open +# +# Step 3 — Verify WebUI connectivity (if PARTNERSHIP_ONBOARD_VERIFY=true) +# curl each configured WebUI via HOST1's Tailscale IP +# Confirms Tailscale routing is working and the service is responding +# A redirect that can't be reached is worse than no redirect +# +# Step 4 — Write state files +# Write partnership_HOST1.db=ACTIVE locally +# SSH to HOST2 → push state, write partnership_HOST2.db=ACTIVE +# Reset offline counter on both sides +# +# Step 5 — Notify (if PARTNERSHIP_ONBOARD_NOTIFY=true) +# Notification to both servers: partnership active +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Configuration ──────────────────────────────────────────────────────────── + +```bash +# master_host1.conf (or master_host2.conf for HOST2 side of the relationship) +# ───────────────────────────────────────────────────────────────────────────── +# Containers whose WebUI URLs are reconfigured on onboard. +# Format: "ContainerName|WebUIPort" +# The port here is what gets redirected — matches the container's WebUI port. +# +HOST1_PARTNERSHIP_AUTH_WEBUIS=( + "NginxProxyManager|81" # NPM admin interface — all proxy rules managed here + "Lldap-Gmer4Lfe|17170" # user directory — all user management here + "Authelia|9091" # SSO authentication — policy management here + "Authelia-Secondary|9092" # secondary Authelia instance +) + +# Paths the mirror should collect during the offboard grace window. +# Files here are accessible to HOST2 for PARTNERSHIP_GRACE_HOURS after offboard. +HOST1_PARTNERSHIP_MIRROR_BACKUPS=( + # "/mnt/user/appdata-Failover/Jayred365-Emby" # HOST2's Emby backup on HOST1 +) +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Always dry-run first — shows exact template changes and redirect URLs +# ───────────────────────────────────────────────────────────────────────────── partnership_manage.sh --onboard --dry-run -# Live onboard +# ───────────────────────────────────────────────────────────────────────────── +# Live onboard — owner (HOST1) only +# ───────────────────────────────────────────────────────────────────────────── partnership_manage.sh --onboard -``` -**What happens:** -1. Pre-flight — both servers healthy, Tailscale connected -2. SSH to HOST2 — reconfigure each auth container's WebUI URL to HOST1's Tailscale IP -3. Verify WebUI connectivity — curl each WebUI, confirm reachable -4. Write state files — `partnership_HOST1.db=ACTIVE`, push to HOST2 -5. Reset offline counter -6. Notify both servers — partnership active - -**After onboard:** -``` -Mirror opens Docker UI -→ clicks NginxProxyManager -→ lands on owner's NPM WebUI -→ makes changes there -→ 15min later synced to his server -→ he never touched his own NPM +# Verbose — show each pre-flight check and template change +partnership_manage.sh --onboard --log ``` --- -### Offboard — Clean Separation +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔴 --offboard — Clean Separation +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -**Either server can initiate offboard.** The process differs slightly depending on who runs it. +Either server can initiate offboard. The process differs slightly depending on who +runs it — but both paths end with both servers fully independent. The mirror leaves +with current auth config, working certs, and full control of its own services. + +> **Always dry-run first.** Offboard stops the sync, reconfigures WebUIs, and +> removes Tailscale access. The dry-run shows exactly what would happen, in order, +> before any of it does. + +--- + +### ── Owner-Initiated Offboard (HOST1 runs --offboard) ─────────────────────── ```bash -# Always dry-run first -partnership_manage.sh --offboard --dry-run - -# Live offboard -partnership_manage.sh --offboard +# ───────────────────────────────────────────────────────────────────────────── +# 1. 10-second countdown shown — Ctrl-C to abort if you changed your mind +# +# 2. Stop any running rsync +# rsync_stop.sh --rsync-only — stops the rsync but leaves container recovery +# to docker_watchdog.sh. Nothing else is interrupted. +# +# 3. Final sync +# Critical-Data pushed to HOST2 one last time — HOST2 leaves with current state +# Emby failover sync pushed — HOST2 has current watch states +# This is the cleanest possible state for HOST2 to inherit. +# +# 4. Reconfigure HOST2 WebUI templates → localhost +# SSH to HOST2 → update each auth container WebUI URL → http://localhost:PORT +# HOST2 operator now clicks NPM → gets HOST2's own NPM +# HOST2 manages its own auth from this point forward +# +# 5. Disable critical sync +# Write CRITICAL_RSYNC_ENABLED=false to master.conf +# Critical-Data sync stops at next 15-minute cycle +# +# 6. Write state files +# partnership_HOST1.db=INACTIVE +# SSH to HOST2 → partnership_HOST2.db=INACTIVE +# +# 7. Grace period — PARTNERSHIP_GRACE_HOURS (default 6 hours) +# HOST2's data on HOST1 remains accessible for this window +# HOST2 can collect anything it needs via Tailscale during this time +# Both access and Tailscale removal happen at the same moment (step 8) +# +# 8. Remove HOST2 from Tailscale tailnet +# API call to revoke HOST2's device — encrypted connection ends +# Backup access ends at the same time by design — no misleading window +# +# 9. Notify HOST2 — "offboard complete, access removed in Xh" +# ───────────────────────────────────────────────────────────────────────────── ``` -#### HOST1 (owner) initiates offboard: - -1. 10 second countdown -2. Stop any running rsync (`rsync_stop.sh --rsync-only`) -3. Final sync — Critical-Data and Emby pushed one last time -4. SSH to HOST2 — reconfigure auth WebUIs back to localhost -5. Disable `CRITICAL_RSYNC_ENABLED=false` in Master.conf -6. Write `partnership_HOST1.db=INACTIVE`, push to HOST2 -7. Wait `PARTNERSHIP_GRACE_HOURS` (6hr) — mirror has access to collect anything -8. Remove HOST2 from Tailscale tailnet -9. Notify HOST2 - -#### HOST2 (mirror) initiates offboard: - -1. Reconfigure own auth WebUIs → localhost -2. Write `partnership_HOST2.db=INACTIVE` -3. SSH to HOST1 — push state file (or write locally if HOST1 unreachable) -4. Notify HOST1 — "mirror has requested offboard" - -**HOST1 finalises on next `--check` cycle:** - -5. Reads HOST2's state file — sees INACTIVE -6. Runs final sync — HOST2 leaves with current state -7. Disables critical sync -8. Writes `partnership_HOST1.db=INACTIVE` -9. Waits grace period → removes HOST2 from Tailscale - -#### What HOST2 leaves with: - -``` -Current auth config ✓ — final sync was clean -His own DDNS + updater ✓ — always was his, unaffected -His own Git mirror ✓ — full ecosystem, always mirrored -His anime source of truth ✓ — always was his -Auth WebUIs → localhost ✓ — managing his own auth now -Full independence ✓ — just stop the sync -Certs valid ~60-90 days ✓ — cert_monitor.sh warns at 30 days -``` - -**Tailscale access and backup window both expire at the same time** (`PARTNERSHIP_GRACE_HOURS=6`). Keeping backups accessible beyond Tailscale removal is meaningless — they expire together by design. - --- -### Auto-Offboard — 30 Day Offline Threshold - -If either server is unreachable for `PARTNERSHIP_OFFLINE_THRESHOLD` days (default: 30), the other server auto-offboards. Works both directions: - -``` -HOST1 sees HOST2 offline 30 days: - AM_OWNER → full offboard from owner side - Tailscale removal ✅ (API call works regardless) - -HOST2 sees HOST1 offline 30 days: - AM_MIRROR → mirror offboard - Reconfigures own WebUIs → localhost ✅ - Writes HOST2.db=INACTIVE ✅ - Fully independent immediately ✅ - When HOST1 comes back → sees INACTIVE → cleans its side ✅ -``` - -This handles the "partner disappeared" scenario without manual intervention. 30 days is long enough to cover extended outages, short enough that you're not waiting forever for resolution. - ---- - -### Transfer — Flipping Ownership - -Run `--transfer` from HOST1 when ownership needs to change: +### ── Mirror-Initiated Offboard (HOST2 runs --offboard) ────────────────────── ```bash -# Always dry-run first — this is significant +# ───────────────────────────────────────────────────────────────────────────── +# HOST2 can offboard at any time, even if HOST1 is temporarily unreachable. +# The deferred finalisation mechanism ensures HOST1 catches up automatically. +# +# 1. Reconfigure own WebUI templates → localhost +# All auth containers now point to HOST2's own services +# HOST2 manages its own auth immediately — no waiting for HOST1 +# +# 2. Write partnership_HOST2.db=INACTIVE locally +# +# 3. SSH to HOST1 (if reachable) → push state file notification +# HOST1 reads INACTIVE → begins owner offboard sequence (steps 3-9 above) +# HOST1 runs final sync → sends current state to HOST2 one more time +# HOST1 handles Tailscale removal and grace period +# +# 4. If HOST1 unreachable — deferred finalisation +# HOST2 has already reconfigured to localhost — fully independent now +# STATE file written — HOST1 will read it on next --check when reachable +# Next time HOST1 runs --check and can reach HOST2: sees INACTIVE → finalises +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── What HOST2 Keeps After Offboard ──────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# What HOST2 leaves with: + Current auth config ✓ final sync was clean — NPM rules, users, policies + Own DDNS + updater ✓ always was his, unaffected by partnership + Own Git mirror ✓ full ecosystem — git pull always ran on HOST2 + Own anime source of truth ✓ Anime_Shows / Anime_Movies — always was his + Auth WebUIs → localhost ✓ managing own auth directly from this point + Full independence ✓ just stop the sync — ecosystem continues + Certs valid ~60-90 days ✓ cert_monitor.sh warns at 30 days — time to renew + +# What HOST1 keeps after offboard: + His hardware ✓ unaffected + His media shares ✓ Movies, Tv_Shows, Music — his source of truth + His services ✓ running on HOST1 as always + His domain ✓ DNS unaffected + +# What HOST1 loses: + Offsite backup ✗ was HOST2's hardware in a different location + Geographic redundancy ✗ cannot be scripted — that was the person +# ───────────────────────────────────────────────────────────────────────────── +# The arrangement was always mutual. Both parties provided something the scripts +# cannot replace — hardware on a different power utility in a different location. +# Everything else can be reconfigured. That cannot. +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Available from either server — always dry-run first +# ───────────────────────────────────────────────────────────────────────────── +partnership_manage.sh --offboard --dry-run # show the complete sequence, no changes + +partnership_manage.sh --offboard # live offboard — 10s countdown before starting +partnership_manage.sh --offboard --log # verbose — show each step as it executes +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔄 --transfer — Flipping Ownership +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Transfers ownership from HOST1 to HOST2 (or back after a previous transfer). After +transfer, HOST2 becomes the auth source of truth and HOST1 receives syncs. +Only the current owner can initiate transfer. + +> **This is the most significant operation in the lifecycle.** It reconfigures both +> servers, flips sync direction, and changes master.conf permanently on both sides. +> The confirmation string and health strike system exist to prevent accidents. + +--- + +### ── Why This Is Protected ─────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Transfer involves: +# Modifying master.conf on both servers simultaneously +# Flipping sync direction (HOST1 → HOST2 becomes HOST2 → HOST1) +# Reconfiguring WebUI templates on both servers +# Making HOST2 the authoritative source for auth decisions +# +# A partial transfer — if interrupted mid-sequence — leaves both servers with +# different auth configurations and no clear source of truth. The health strike +# system prevents starting a transfer when either server is degraded. +# The confirmation string prevents accidental execution entirely. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Transfer Sequence ─────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# 1. Display current and future ownership clearly +# "Current: HOST1 → Mirror: HOST2" +# "After: HOST2 → Mirror: HOST1" +# No ambiguity about what is being confirmed. +# +# 2. Confirmation string check +# Must type exactly: i-understand-this-transfers-ownership +# Long and specific by design — cannot be typed accidentally. +# +# 3. Health strike system +# Both servers must pass PARTNERSHIP_TRANSFER_STRIKES consecutive health checks. +# Max PARTNERSHIP_TRANSFER_MAX_ATTEMPTS attempts (default 20) before aborting. +# Each strike: check both arrays started, Docker daemons responding, Tailscale connected. +# Protects against transferring during a degraded state. +# +# 4. Final sync in current direction +# Push Current-Data from current owner (HOST1) to current mirror (HOST2). +# New mirror (HOST1) starts with current state from the outgoing owner. +# +# 5. Reconfigure WebUI templates on HOST2 +# HOST2 WebUIs → localhost (HOST2 now manages its own auth directly) +# +# 6. Reconfigure WebUI templates on HOST1 +# HOST1 WebUIs → HOST2's Tailscale IP (HOST1 now redirects to HOST2) +# +# 7. Flip PARTNERSHIP_OWNER_HOST in master.conf on BOTH servers +# "HOST1" → "HOST2" written to both servers via SSH +# +# 8. Write updated state files on both servers +# +# 9. Notify both servers — transfer complete, new roles shown +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── After Transfer ─────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Before transfer: HOST1 = owner, HOST2 = mirror +# After transfer: HOST2 = owner, HOST1 = mirror +# +# HOST2 now: +# All auth changes made here directly +# Critical-Data pushed → to HOST1 every 15 minutes +# NPM, LLDAP, Authelia — HOST2's own instances are authoritative +# HOST1's WebUIs redirect to HOST2 via Tailscale +# +# HOST1 now: +# Receives Critical-Data sync from HOST2 +# Auth WebUIs point to HOST2 — clicking NPM opens HOST2's NPM +# Never makes auth configuration changes directly +# +# To transfer back — run --transfer from HOST2 (the new owner) +# PARTNERSHIP_OWNER_HOST="HOST2" after first transfer, "HOST1" after second +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Dry run — ALWAYS first. Shows the full sequence, health check results, +# and what master.conf changes would be made on both servers. +# ───────────────────────────────────────────────────────────────────────────── partnership_manage.sh --transfer --dry-run -# Live transfer — requires explicit confirmation string +# ───────────────────────────────────────────────────────────────────────────── +# Live transfer — owner only — requires exact confirmation string +# ───────────────────────────────────────────────────────────────────────────── partnership_manage.sh --transfer --confirm=i-understand-this-transfers-ownership ``` -**The confirmation string is long and specific by design.** You cannot type it accidentally. - -**What happens:** -1. Display current and future ownership clearly -2. Confirmation string check -3. Health strike system — both servers must pass `PARTNERSHIP_TRANSFER_STRIKES` consecutive checks (max `PARTNERSHIP_TRANSFER_MAX_ATTEMPTS` attempts before giving up) -4. Final sync in current direction — new mirror leaves with current state -5. Reconfigure HOST2 WebUIs → new owner's Tailscale IP -6. Reconfigure HOST1 WebUIs → localhost (now manages directly) -7. Flip `PARTNERSHIP_OWNER_HOST` in Master.conf on both servers -8. Write updated state files -9. Notify both servers - -**After transfer:** -``` -Before: HOST1 = owner, HOST2 = mirror -After: HOST2 = owner, HOST1 = mirror - -HOST2 now: - Makes all auth changes ← source of truth - Pushes Critical-Data sync ← every 15min - Manages NPM, certs, LLDAP ← directly - -HOST1 now: - Receives sync ← warm mirror - WebUIs point to HOST2 ← transparent - Never makes changes directly -``` - --- -### Check — Automated State Monitor +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔁 --check — Automated State Monitor +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Called automatically by `critical_sync_maintenance.sh` every 15 minutes. Never run manually. +Called automatically by `critical_sync_maintenance.sh` every 15 minutes. Never run +manually. Silent when healthy — produces output only when state changes or thresholds +are hit. ```bash -# Called internally — do not run manually -partnership_manage.sh --check --remote-seen # rsync succeeded, HOST2 reachable -partnership_manage.sh --check --remote-unseen # rsync failed, HOST2 unreachable +# Called internally — do not run manually: +partnership_manage.sh --check --remote-seen # rsync succeeded, remote reachable +partnership_manage.sh --check --remote-unseen # rsync failed, remote unreachable ``` -**What it does each cycle:** -- `--remote-seen`: resets offline counter, updates `last_seen_remote` timestamp -- `--remote-unseen`: increments offline counter, checks auto-offboard threshold -- SSHes to remote, reads remote state file -- Both ACTIVE → silent, healthy ✅ -- Remote INACTIVE → owner finalises offboard, mirror cleans up -- Threshold exceeded → auto-offboard - --- -## Role-Based Access - -``` -HOST1 (owner): --onboard ✅ --offboard ✅ --transfer ✅ --status ✅ -HOST2 (mirror): --onboard ❌ --offboard ✅ --transfer ❌ --status ✅ -``` - -HOST2 is blocked from onboard and transfer by design — ownership is granted not taken. Either server can initiate offboard — clean exit is always available to both parties. - ---- - -## Configuration - -All configuration in `Master.conf` under `── PARTNERSHIP ──`, placed immediately after HOST CONFIGURATION. +### ── What It Does Each Cycle ───────────────────────────────────────────────── ```bash -PARTNERSHIP_ENABLED=false -PARTNERSHIP_OWNER_HOST="HOST1" # "HOST1" or "HOST2" — flips on --transfer +# ───────────────────────────────────────────────────────────────────────────── +# --remote-seen path (rsync succeeded → HOST2 was reachable): +# Reset offline counter to 0 +# Update last_seen_remote timestamp +# SSH to read remote state file +# Both ACTIVE → silent ✅ +# Remote INACTIVE → begin offboard finalisation from owner side +# +# --remote-unseen path (rsync failed → HOST2 unreachable): +# Increment offline counter by 1 +# Counter × 15min = approximate offline duration +# At PARTNERSHIP_OFFLINE_THRESHOLD days: auto-offboard begins +# Notify when counter hits threshold — not on every missed cycle +# +# Why the rsync outcome drives this: +# The rsync that runs before --check provides a reliable connectivity signal. +# If rsync to HOST2 succeeded, HOST2 is reachable — reset counter. +# If rsync failed, HOST2 may be unreachable — increment counter. +# No separate ping needed — the sync attempt is the connectivity test. +# ───────────────────────────────────────────────────────────────────────────── +``` -# Auth containers reconfigured on onboard/offboard -# Format: "ContainerName|WebUIPort" -PARTNERSHIP_AUTH_WEBUIS=( - "NginxProxyManager|81" - "Lldap-Gmer4Lfe|17170" - "Authelia|9091" - "Authelia-Secondary|9092" +--- + +### ── Auto-Offboard ──────────────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# After this many days of consecutive missed sync cycles, auto-offboard begins. +# Works both directions independently — neither server needs the other to act. +# + PARTNERSHIP_OFFLINE_THRESHOLD=30 # days — each missed cycle = ~15min +# +# HOST1 sees HOST2 offline for 30 days: +# AM_OWNER → full owner-initiated offboard +# Tailscale removal via API (API works regardless of HOST2 reachability) +# Critical sync disabled +# State: INACTIVE +# +# HOST2 sees HOST1 offline for 30 days: +# AM_MIRROR → mirror-initiated offboard +# Reconfigure own WebUIs → localhost +# Write HOST2.db=INACTIVE +# Fully independent immediately +# When HOST1 eventually comes back → reads INACTIVE → cleans its side +# +# 30 days covers: +# Extended hardware failure (people do fix things eventually) +# Someone moving and being offline for weeks +# But not: a permanent departure +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +## ━━━ ROLE-BASED ACCESS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Which server can run which mode: +# + HOST1 (owner) HOST2 (mirror) + --onboard ✅ ❌ ownership is granted, not taken + --offboard ✅ ✅ clean exit available to both parties + --transfer ✅ ❌ owner only — mirror cannot self-promote + --check ✅ ✅ both servers monitor state + --status ✅ ✅ status is always available +# +# AM_OWNER / AM_MIRROR flags set by detect_hosts() from PARTNERSHIP_OWNER_HOST. +# All routing decisions use these flags — no hostname string comparisons. +# Mirror attempting --onboard or --transfer: blocked with a clear error message. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +## ━━━ INTEGRATION WITH CRITICAL SYNC ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Partnership check runs inside critical_sync_maintenance.sh — the 15-minute +# orchestrator. The rsync outcome directly informs the partnership check. +# +# Schedule: */15 * * * * +# +# Execution order in critical_sync_maintenance.sh: +# 1. Critical-Data rsync → auth stack sync +# 2. emby-failover rsync → dirty Emby sync +# 3. partnership --check → state check, informed by rsync results +# → --remote-seen if rsync succeeded (HOST2 reachable) +# → --remote-unseen if rsync failed (HOST2 unreachable) +# +# When critical sync is disabled (CRITICAL_RSYNC_ENABLED=false): +# partnership --check is skipped automatically +# Offline counter does not increment from a sync that wasn't attempted +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +## ━━━ FULL CONFIGURATION REFERENCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── + +# ── Partnership Gate ─────────────────────────────────────────────────────── + PARTNERSHIP_ENABLED=false # set true once both servers are configured + PARTNERSHIP_OWNER_HOST="HOST1" # "HOST1" or "HOST2" — flips on --transfer + +# ── Tailscale Removal ────────────────────────────────────────────────────── + PARTNERSHIP_REMOVE_TAILSCALE=true # remove mirror from tailnet on offboard + TAILSCALE_API_KEY="" # tskey-api-... from tailscale.com/admin/settings/keys + TAILSCALE_TAILNET="" # your tailnet name (e.g. example.github) + # required when PARTNERSHIP_REMOVE_TAILSCALE=true + +# ── Timing ───────────────────────────────────────────────────────────────── + PARTNERSHIP_GRACE_HOURS=6 # hours before Tailscale removal after offboard + # backup access also expires at this time — by design + PARTNERSHIP_OFFLINE_THRESHOLD=30 # days unreachable before auto-offboard (both servers) + PARTNERSHIP_SYNC_INTERVAL=15 # informational — actual schedule in cron + +# ── Transfer Safety ──────────────────────────────────────────────────────── + PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership" + # must type this exactly to proceed — long by design + PARTNERSHIP_TRANSFER_STRIKES=3 # consecutive health checks required before transfer + PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20 # max health attempts before giving up + +# ── Onboard Behaviour ────────────────────────────────────────────────────── + PARTNERSHIP_ONBOARD_VERIFY=true # curl each WebUI after reconfigure to verify routing + PARTNERSHIP_ONBOARD_NOTIFY=true # notify both servers on successful onboard + +# master_host1.conf / master_host2.conf +# ───────────────────────────────────────────────────────────────────────────── + +# Containers whose WebUI URLs are reconfigured on onboard/offboard + HOST1_PARTNERSHIP_AUTH_WEBUIS=( + "NginxProxyManager|81" + "Lldap-Gmer4Lfe|17170" + "Authelia|9091" + "Authelia-Secondary|9092" + ) + +# Paths accessible to HOST2 during grace window after offboard + HOST1_PARTNERSHIP_MIRROR_BACKUPS=( + # "/mnt/user/appdata-Failover/Jayred365-Emby" + ) +``` + +--- + +## ━━━ INITIAL SETUP REQUIREMENTS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Before `--onboard` can run, five things must be in place: + +--- + +### 1. Tailscale Connected on Both Servers + +```bash +# Verify HOST2 is visible from HOST1: +tailscale ip -4 unRAID-Jayred365 # should return HOST2's Tailscale IP + +# Verify HOST1 is visible from HOST2: +tailscale ip -4 unRAID-Gmer4Lfe # should return HOST1's Tailscale IP + +# Verify reachability: +tailscale ping unRAID-Jayred365 +``` + +HOST1 and HOST2 hostnames must match their exact Tailscale device names — already +enforced by the ecosystem since `REMOTE_SERVER_NAME` is used everywhere. + +--- + +### 2. SSH Keys — Passwordless Both Directions + +```bash +# From HOST1 — should print HOST2's hostname without a password prompt: +ssh -i /root/.ssh/Gmer4Lfe-rsync-key root@[HOST2-tailscale-ip] "hostname" + +# From HOST2 — should print HOST1's hostname without a password prompt: +ssh -i /root/.ssh/Jayred365-rsync-key root@[HOST1-tailscale-ip] "hostname" +``` + +--- + +### 3. Auth Containers Exist on HOST2 + +```bash +# Containers must exist — they can be stopped: +ssh root@[HOST2-ip] "docker inspect NginxProxyManager --format '{{.State.Status}}'" +# Expected: created, exited, or running — NOT "no such container" + +# Template files must exist for WebUI reconfiguration: +ssh root@[HOST2-ip] "ls /boot/config/plugins/dockerMan/templates-user/" +``` + +--- + +### 4. Critical-Data rsync Profile Configured + +```bash +# master.conf — CRITICAL_SYNC_SHARES must include the auth stack path: +CRITICAL_SYNC_SHARES=( + "/mnt/user/appdata-Failover/Critical-Data" # auth config — NPM, Authelia, LLDAP, certs ) - -# Paths on owner that mirror should collect during grace window -PARTNERSHIP_MIRROR_BACKUPS=( - # "/mnt/user/appdata-Failover/Jayred365-Emby" -) - -# Timing -PARTNERSHIP_GRACE_HOURS=6 # hours after offboard before Tailscale removal - # backup access also expires at this time -PARTNERSHIP_OFFLINE_THRESHOLD=30 # days unreachable before auto-offboard (both directions) -PARTNERSHIP_REMOVE_TAILSCALE=true # remove mirror from Tailscale on offboard - -# Tailscale API — required for PARTNERSHIP_REMOVE_TAILSCALE=true -TAILSCALE_API_KEY="" # tskey-api-... -TAILSCALE_TAILNET="" # your tailnet name - -# Transfer safety -PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership" -PARTNERSHIP_TRANSFER_STRIKES=3 -PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20 - -# Onboard -PARTNERSHIP_ONBOARD_VERIFY=true -PARTNERSHIP_ONBOARD_NOTIFY=true -PARTNERSHIP_SYNC_INTERVAL=15 # minutes — informational, actual schedule in cron +# This is what keeps HOST2 current after onboard ``` --- -## critical_sync_maintenance.sh - -The partnership check runs inside `critical_sync_maintenance.sh` — the 15-minute orchestrator that also handles Critical-Data and Emby failover syncs. - -``` -Orchestrators/critical_sync_maintenance.sh -Schedule: */15 * * * * - -Execution order: - 1. Critical-Data rsync ← auth stack sync (15min) - 2. emby-failover rsync ← dirty Emby sync (15min) - 3. partnership --check ← state check, informed by rsync outcome -``` - -The rsync outcome directly informs the partnership check — if rsync succeeded, HOST2 was reachable and the offline counter resets. If rsync failed, the counter increments. - -Controlled by `CRITICAL_RSYNC_ENABLED` in Master.conf — set to `false` automatically on offboard. - ---- - -## Initial Setup Requirements - -Before `--onboard` can run: - -**1. Tailscale connected on both servers** - -HOST1 and HOST2 hostnames must match their exact Tailscale device names — already enforced by the ecosystem since `HOST1` and `HOST2` in Master.conf are used for Tailscale IP resolution everywhere. - -**2. SSH keys configured — passwordless both directions** +### 5. Tailscale API Key (if removing from tailnet on offboard) ```bash -ssh -i /root/.ssh/your_key root@[HOST2-tailscale-ip] "hostname" -``` +# master.conf + PARTNERSHIP_REMOVE_TAILSCALE=true + TAILSCALE_API_KEY="tskey-api-..." # from tailscale.com/admin/settings/keys + TAILSCALE_TAILNET="example.github" # your tailnet name -**3. Auth containers exist on HOST2** - -Containers must exist before onboard. They can be stopped — onboard only reconfigures WebUI URLs. Critical-Data sync brings them live config. - -**4. Critical-Data rsync profile configured** - -The `critical-data` rsync profile must be set up in Master.conf. This is what keeps the mirror current after onboard. Configured in `CRITICAL_SYNC_SHARES`. - -**5. Tailscale API key configured** (if `PARTNERSHIP_REMOVE_TAILSCALE=true`) - -``` -https://login.tailscale.com/admin/settings/keys -Scope: Devices write +# Required scope: Devices write +# Generate at: https://login.tailscale.com/admin/settings/keys ``` --- -## Independence — Always One Stop Away - -The partnership is designed with a clean exit built in from day one. - -**To stop the relationship — from either server:** -```bash -partnership_manage.sh --offboard -``` - -That's it. The mirror's containers keep running. His domain keeps working. His DDNS keeps pointing to him. His git repo has the full ecosystem. His certs run for another 60-90 days. - -**What the owner keeps after separation:** -``` -His hardware ← always was his -His media shares ← his source of truth shares stay his -His services ← unaffected -His domain ← unaffected -His Emby users ← continue as before -``` - -**What the owner loses:** -``` -Offsite backup ← was the mirror's hardware -Geographic redundancy ← can't script that, it was the person -``` - -The arrangement was always mutual. Both parties contributed something the scripts couldn't provide — hardware in a different location on a different power utility. That's irreplaceable. Everything else in the ecosystem can be reconfigured. +## ━━━ TROUBLESHOOTING ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ --- -## Troubleshooting +### 🔴 WebUI Still Pointing to Old IP After Onboard -**WebUI still pointing to old IP after onboard:** -- Check template was found: `partnership_manage.sh --status` -- Some containers may need a restart to pick up the new WebUI URL -- Verify with `--status` on both servers +``` +Check template was found and modified: + partnership_manage.sh --status shows current WebUI redirect state + ssh HOST2 "cat /boot/config/plugins/dockerMan/templates-user/NginxProxyManager.xml" + should show HOST1's Tailscale IP -**Transfer health checks failing:** -- Both arrays must be fully started -- Both Docker daemons must be responding -- Tailscale must be connected on both servers -- Max attempts: `PARTNERSHIP_TRANSFER_MAX_ATTEMPTS` (default 20) before giving up +Some containers need a restart to pick up new WebUI URL: + docker restart NginxProxyManager on HOST2 — picks up template change -**Mirror's domain broken after offboard:** -- Auth WebUIs reconfigured to localhost — mirror manages auth directly now -- Verify auth containers are running: `docker ps` -- Check cert expiry: run `cert_monitor.sh` - -**State mismatch between servers:** -- Run `--status` on both servers to see both state files -- If one shows ACTIVE and other INACTIVE — run `--offboard` to resolve -- If HOST2 unreachable — HOST2 will self-resolve on next `--check` when reachable - -**Auto-offboard triggered unexpectedly:** -- Check `/boot/config/partnership_offline_days.db` for counter value -- Extended outage or Tailscale connectivity issue may have incremented counter -- Re-onboard if partnership should continue: `--onboard` +Verify Tailscale routing is working: + curl http://[HOST1-tailscale-ip]:81 from HOST2 — should reach HOST1's NPM +``` --- -## Design Notes +### 🔴 Transfer Health Checks Failing -**Why HOST1/HOST2 instead of PARTNERSHIP_OWNER/MIRROR?** +``` +Both arrays must be fully started: + ls /mnt/user should show share directories on both servers -HOST1 and HOST2 are already defined in Master.conf with SSH keys, Tailscale names, and all connection details. Duplicating them as PARTNERSHIP_OWNER and PARTNERSHIP_MIRROR would require maintaining the same values twice. A single `PARTNERSHIP_OWNER_HOST` var flips ownership direction — everything else derives from the existing HOST1/HOST2 configuration. +Both Docker daemons responding: + docker ps should return a list, not hang -**Why does transfer require a confirmation string?** +Tailscale connected on both servers: + tailscale status should show the remote peer as online -Transferring ownership touches Master.conf on both servers, reconfigures WebUIs, and flips sync direction. The confirmation string is the same philosophy as `--i-know-what-im-doing` in the arr cleanup scripts — make accidental execution impossible, not just unlikely. +Max attempts before giving up: PARTNERSHIP_TRANSFER_MAX_ATTEMPTS (default 20) +Each attempt waits for next health window — may take several minutes total +``` -**Why do Tailscale removal and backup access expire at the same time?** +--- -Keeping backups accessible beyond Tailscale removal is meaningless — if the mirror can't reach the owner's server via Tailscale, he can't access the backups anyway. `PARTNERSHIP_GRACE_HOURS` controls both simultaneously. One var, consistent behavior, no misleading "30 days available" when access is gone in 6 hours. +### 🔴 Mirror's Domain Broken After Offboard -**Why does either server auto-offboard after 30 days offline?** +``` +Auth WebUIs reconfigured to localhost — mirror manages auth directly now. +The containers are running and functional — they just no longer redirect to HOST1. -30 consecutive days of missed sync cycles means the relationship has effectively ended regardless of intent. The auto-offboard makes the state official and cleans up both sides without requiring manual intervention from a server that may genuinely be gone. Each server acts independently — they don't need to coordinate to offboard. +Verify auth containers running: + docker ps | grep -E "NPM|Authelia|LLDAP" -**Why does WebUI reconfiguration use the unRAID template system?** +Check cert expiry — certs won't auto-renew anymore via HOST1: + cert_monitor.sh --dry-run shows days remaining on each domain's cert -unRAID stores container WebUI URLs in `/boot/config/plugins/dockerMan/templates-user/`. Editing templates is the correct way to change WebUI URLs — it persists across container restarts and array reboots. Direct Docker label manipulation would be lost on the next container recreation. +Set up cert auto-renewal on HOST2 independently if not already configured. +``` -**Why is Tailscale WebUI left on the mirror's local server?** +--- -Tailscale WebUI shows that server's network state from its own perspective. When diagnosing connectivity issues between the servers, you need to see the mirror's network view — not the owner's. It's the one WebUI that genuinely needs to stay local. \ No newline at end of file +### 🔴 State Mismatch Between Servers + +``` +Run --status on both servers to see both state files: + partnership_manage.sh --status on HOST1 + partnership_manage.sh --status on HOST2 + +If one shows ACTIVE and other shows INACTIVE: + The INACTIVE side has already offboarded + Run --offboard on the ACTIVE side to sync the state + +If HOST2 unreachable: + HOST2 will self-resolve on next --check when it becomes reachable + HOST1 reads HOST2's INACTIVE state → finalises from owner side automatically +``` + +--- + +### 🔴 Auto-Offboard Triggered Unexpectedly + +``` +Check the offline counter: + cat /boot/config/partnership_offline_days.db + +Extended Tailscale connectivity issue may have incremented the counter: + tailscale status verify HOST2 appears in HOST1's peer list + +If partnership should continue — re-onboard: + partnership_manage.sh --onboard --dry-run # verify setup is correct + partnership_manage.sh --onboard # re-establish partnership + +Counter resets to 0 on successful --onboard. +``` + +--- + +## ━━━ DESIGN NOTES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +``` +Why HOST1/HOST2 instead of PARTNERSHIP_OWNER/MIRROR? + HOST1 and HOST2 are already defined in master.conf — SSH keys, Tailscale names, + all connection details. A single PARTNERSHIP_OWNER_HOST var flips ownership. + Duplicating everything as PARTNERSHIP_OWNER and PARTNERSHIP_MIRROR would require + maintaining identical values in two places. One source of truth for the host + definition, one var to flip direction. + +Why does transfer require a confirmation string? + Same philosophy as --i-know-what-im-doing in the arr cleanup scripts. + Transfer touches master.conf on both servers, reconfigures WebUIs, and flips + sync direction. The confirmation string makes accidental execution impossible, + not just unlikely. + +Why do Tailscale removal and backup access expire at the same time? + If the mirror can't reach the owner's server via Tailscale, the backup is also + unreachable. PARTNERSHIP_GRACE_HOURS controls both — one var, consistent behaviour, + no misleading "data available for 30 days" when access is gone in 6 hours. + +Why does either server auto-offboard after 30 days offline? + 30 consecutive days of missed sync cycles means the relationship has effectively + ended regardless of intent. Auto-offboard makes the state official. Each server + acts independently — they don't need to coordinate to offboard. + +Why does WebUI reconfiguration use the unRAID template system? + unRAID stores WebUI URLs in /boot/config/plugins/dockerMan/templates-user/. + Editing templates persists across container restarts and array reboots. + Direct Docker label manipulation is lost on next container recreation. +``` \ No newline at end of file diff --git a/Partnership/partnership_manager.sh b/Partnership/partnership_manager.sh index 3c31ce0..7ba85de 100644 --- a/Partnership/partnership_manager.sh +++ b/Partnership/partnership_manager.sh @@ -1,54 +1,93 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# ----------------------------- Partnership Manager -------------------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= Partnership Manager ======================================== +# ============================================================================================== # Manages the relationship lifecycle between two unRAID servers. # HOST1 is always the owner (source of truth). HOST2 is always the mirror. -# PARTNERSHIP_OWNER_HOST flips to "HOST2" after --transfer. +# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a successful --transfer. # -# Modes: -# --onboard ← owner only — set up mirror relationship +# ── MODES ───────────────────────────────────────────────────────────────────────────────────── +# --onboard ← owner only — establish mirror relationship # Reconfigures HOST2 auth WebUIs → HOST1 Tailscale IP # HOST2 clicks NPM → gets HOST1's NPM automatically # # --offboard ← either server — clean separation -# Either party can initiate -# HOST2 offboard: writes state, reconfigures own WebUIs → localhost -# HOST1 sees it on next --check → finalises, runs final sync -# HOST1 offboard: final sync, reconfigures HOST2 WebUIs, removes Tailscale +# Mirror-initiated: writes INACTIVE state, reconfigures own WebUIs → localhost +# Owner finalises on next --check: final sync, Tailscale removal +# Owner-initiated: final sync, reconfigures mirror WebUIs, Tailscale removal # Both leave with current state, clean exit ✅ # -# --transfer ← owner only — flip ownership -# Requires confirmation string + health strike system -# Reconfigures both servers, flips PARTNERSHIP_OWNER_HOST in Master.conf +# --transfer ← owner only — flip ownership between servers +# Requires confirmation string + consecutive health strikes +# Reconfigures both servers, flips PARTNERSHIP_OWNER_HOST in master.conf # # --check ← called by critical_sync_maintenance.sh every 15min # Reads both state files via SSH # Detects offboard requests → finalises from owner side # Updates last_seen_remote timestamp # Increments offline counter → auto-offboards after threshold -# Silent when healthy +# Silent when healthy ← never noisy on clean runs # -# --status ← either server — show current state +# --status ← either server — show current state, both sides # -# State files on /boot/config (survives reboots, available before array): +# ── STATE FILES ─────────────────────────────────────────────────────────────────────────────── +# On /boot/config — survives reboots, available before array starts: # /boot/config/partnership_HOST1.db ← HOST1 writes only # /boot/config/partnership_HOST2.db ← HOST2 writes only # Propagated via SSH — no rsync needed # -# All configuration in Master.conf under PARTNERSHIP section. -# ----------------------------------------------------------------------------------------------- +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# Root check — all operations require root +# validate_unraid_cmd — notify validated before use +# Role enforcement — mirror cannot run owner-only modes +# AM_OWNER / AM_MIRROR — all routing via these flags, not hostname strings +# version parity — onboard checks both servers match unRAID version +# remote docker daemon — onboard verifies remote daemon responsive +# SSH_TIMEOUT — all SSH calls timeout-protected +# flock on state writes — prevents concurrent state file corruption +# SIGTERM trap — grace period sleep interruptible +# Silent by default — only warns/errors produce output (--check is always silent healthy) +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_PARTNERSHIP_AUTH_WEBUIS — containers reconfigured on onboard/offboard +# HOST*_PARTNERSHIP_MIRROR_BACKUPS — paths available after offboard +# All aliased by detect_hosts() — script uses PARTNERSHIP_AUTH_WEBUIS etc. +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# PARTNERSHIP_ENABLED — global enable gate +# PARTNERSHIP_OWNER_HOST — "HOST1" or "HOST2" — flips on --transfer +# PARTNERSHIP_REMOVE_TAILSCALE — remove mirror from tailnet on offboard +# PARTNERSHIP_GRACE_HOURS — hours before Tailscale removal after offboard +# PARTNERSHIP_OFFLINE_THRESHOLD — days unreachable before auto-offboard +# PARTNERSHIP_TRANSFER_CONFIRM — exact string required for --transfer +# PARTNERSHIP_TRANSFER_STRIKES — consecutive health checks required +# PARTNERSHIP_TRANSFER_MAX_ATTEMPTS — max attempts before giving up +# PARTNERSHIP_ONBOARD_VERIFY — verify WebUI connectivity after onboard +# PARTNERSHIP_ONBOARD_NOTIFY — notify both servers on onboard completion +# PARTNERSHIP_SYNC_INTERVAL — informational — actual schedule in cron +# TAILSCALE_API_KEY / TAILSCALE_TAILNET — required when PARTNERSHIP_REMOVE_TAILSCALE=true +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# partnership_manage.sh --onboard +# partnership_manage.sh --offboard +# partnership_manage.sh --transfer --confirm=i-understand-this-transfers-ownership +# partnership_manage.sh --check --remote-seen|--remote-unseen +# partnership_manage.sh --status +# Any mode supports --dry-run and --log +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" -# Parse flags before parse_args +SSH_TIMEOUT=15 + +# ── Parse mode flags before parse_args ──────────────────────────────────────────────────────── MODE="" TRANSFER_CONFIRM_INPUT="" REMOTE_SEEN=false REMOTE_UNSEEN=false +REASON="manual" FILTERED_ARGS=() for arg in "$@"; do @@ -61,51 +100,59 @@ for arg in "$@"; do --confirm=*) TRANSFER_CONFIRM_INPUT="${arg#--confirm=}" ;; --remote-seen) REMOTE_SEEN=true ;; --remote-unseen) REMOTE_UNSEEN=true ;; + --reason=*) REASON="${arg#--reason=}" ;; *) FILTERED_ARGS+=("$arg") ;; esac done parse_args "${FILTERED_ARGS[@]}" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + +# detect_hosts() sets MY_ID and aliases PARTNERSHIP_AUTH_WEBUIS, PARTNERSHIP_MIRROR_BACKUPS detect_hosts -# Derive owner and mirror from PARTNERSHIP_OWNER_HOST -if [[ "${PARTNERSHIP_OWNER_HOST:-HOST1}" == "HOST1" ]]; then - OWNER="$HOST1" - MIRROR="$HOST2" - OWNER_SSH_KEY="$HOST1_SSH_KEY" - MIRROR_SSH_KEY="$HOST2_SSH_KEY" -else - OWNER="$HOST2" - MIRROR="$HOST1" - OWNER_SSH_KEY="$HOST2_SSH_KEY" - MIRROR_SSH_KEY="$HOST1_SSH_KEY" -fi +# ── Derive owner and mirror from PARTNERSHIP_OWNER_HOST ─────────────────────────────────────── +OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}" # e.g. "HOST1" +MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" ) + +OWNER="${!OWNER_ID}" # hostname string +MIRROR="${!MIRROR_ID}" +OWNER_SSH_KEY_VAR="${OWNER_ID}_SSH_KEY" +MIRROR_SSH_KEY_VAR="${MIRROR_ID}_SSH_KEY" +OWNER_SSH_KEY="${!OWNER_SSH_KEY_VAR}" +MIRROR_SSH_KEY="${!MIRROR_SSH_KEY_VAR}" AM_OWNER=false AM_MIRROR=false -[[ "$LOCAL_SERVER_NAME" == "$OWNER" ]] && AM_OWNER=true -[[ "$LOCAL_SERVER_NAME" == "$MIRROR" ]] && AM_MIRROR=true +[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true +[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true # State files LOCAL_STATE_FILE="/boot/config/partnership_${LOCAL_SERVER_NAME}.db" REMOTE_STATE_FILE="/boot/config/partnership_${REMOTE_SERVER_NAME}.db" OWNER_STATE_FILE="/boot/config/partnership_${OWNER}.db" MIRROR_STATE_FILE="/boot/config/partnership_${MIRROR}.db" - -# Offline counter file OFFLINE_COUNTER="/boot/config/partnership_offline_days.db" if [[ -z "$MODE" ]]; then error "No mode specified" echo "Usage:" - echo " partnership_manage.sh --onboard [--dry-run]" - echo " partnership_manage.sh --offboard [--dry-run]" - echo " partnership_manage.sh --transfer --confirm=... [--dry-run]" - echo " partnership_manage.sh --check --remote-seen|--remote-unseen" + echo " partnership_manage.sh --onboard" + echo " partnership_manage.sh --offboard" + echo " partnership_manage.sh --transfer --confirm=..." + echo " partnership_manage.sh --check --remote-seen|--remote-unseen" echo " partnership_manage.sh --status" exit 1 fi @@ -114,32 +161,31 @@ fi if [[ "$AM_MIRROR" == true ]]; then case "$MODE" in onboard|transfer) - error "Only the owner ($OWNER) can run --$MODE" + error "Only the owner ($OWNER / $OWNER_ID) can run --$MODE" error "Run from $OWNER or use --offboard to separate cleanly" exit 1 ;; esac fi -# Acquire lock for all modes except check (check is called frequently, lock would pile up) +# Lock for all modes except --check (check is called frequently, lock would pile up) [[ "$MODE" != "check" ]] && acquire_lock "strict" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" -# ----------------------------------------------------------------------------------------------- -# HELPERS -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== read_state_file() { - local file="$1" - local key="$2" + local file="$1" key="$2" grep "^${key}=" "$file" 2>/dev/null | cut -d= -f2 } write_state_file() { local file="$1" shift - # flock to prevent concurrent writes + # flock prevents concurrent writes to the same state file ( flock -x 200 cat > "$file" << EOF @@ -157,39 +203,30 @@ EOF } push_state_to_remote() { - local local_file="$1" - local remote_ip="$2" - local ssh_key="$3" - + local local_file="$1" remote_ip="$2" ssh_key="$3" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would push state file to remote" return 0 fi - - scp -i "$ssh_key" -o ConnectTimeout=10 \ + timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \ "$local_file" "root@${remote_ip}:${local_file}" 2>/dev/null && \ - success "State file pushed to remote ✅" || \ + log "State file pushed to remote ✅" || \ warn "Could not push state file to remote — will propagate on next sync" } read_remote_state() { - local remote_ip="$1" - local ssh_key="$2" - local remote_file="$3" - - ssh -i "$ssh_key" -o ConnectTimeout=10 root@"$remote_ip" \ + local remote_ip="$1" ssh_key="$2" remote_file="$3" + timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ + -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \ "cat '$remote_file' 2>/dev/null" 2>/dev/null } +# Reconfigure a container's WebUI on the remote server reconfigure_webui() { - local container="$1" - local port="$2" - local target_ip="$3" - local ssh_key="$4" - local remote_ip="$5" - local label="${6:-remote}" + local container="$1" port="$2" target_ip="$3" + local ssh_key="$4" remote_ip="$5" label="${6:-remote}" - info "Reconfiguring $container WebUI → ${target_ip}:${port} on $label..." + log "Reconfiguring $container WebUI → ${target_ip}:${port} on $label..." if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would reconfigure $container WebUI to http://${target_ip}:${port}/" @@ -197,7 +234,8 @@ reconfigure_webui() { fi local template - template=$(ssh -i "$ssh_key" -o ConnectTimeout=10 root@"$remote_ip" \ + template=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ + -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \ "grep -rl '' /boot/config/plugins/dockerMan/templates-user/ 2>/dev/null | \ xargs grep -l '\"$container\"' 2>/dev/null | head -1" 2>/dev/null) @@ -206,18 +244,20 @@ reconfigure_webui() { return 1 fi - ssh -i "$ssh_key" -o ConnectTimeout=10 root@"$remote_ip" \ + timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ + -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \ "sed -i 's|.*|http://${target_ip}:${port}/|g' '$template'" \ 2>/dev/null && \ - success "$container → http://${target_ip}:${port}/ ✅" || { - error "Failed to reconfigure $container WebUI" + log "$container → http://${target_ip}:${port}/ ✅" || { + error "Failed to reconfigure $container WebUI on $label" return 1 } } +# Reconfigure local auth WebUIs to target IP reconfigure_local_webuis() { local target_ip="$1" - info "Reconfiguring local auth WebUIs → ${target_ip}..." + log "Reconfiguring local auth WebUIs → ${target_ip}..." local failures=0 for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do @@ -226,12 +266,13 @@ reconfigure_local_webuis() { local port="${entry##*|}" local template - template=$(grep -rl '' /boot/config/plugins/dockerMan/templates-user/ 2>/dev/null | \ + template=$(grep -rl '' \ + /boot/config/plugins/dockerMan/templates-user/ 2>/dev/null | \ xargs grep -l "\"$container\"" 2>/dev/null | head -1) if [[ -z "$template" ]]; then warn "$container template not found locally" - ((failures++)) + (( failures++ )) continue fi @@ -242,8 +283,8 @@ reconfigure_local_webuis() { sed -i "s|.*|http://${target_ip}:${port}/|g" \ "$template" 2>/dev/null && \ - success "$container → http://${target_ip}:${port}/ ✅" || \ - { error "Failed to reconfigure $container"; ((failures++)); } + log "$container → http://${target_ip}:${port}/ ✅" || \ + { error "Failed to reconfigure $container"; (( failures++ )); } done return $failures } @@ -267,16 +308,16 @@ remove_tailscale_device() { return 1 fi - info "Looking up Tailscale device ID for $hostname..." + log "Looking up Tailscale device ID for $hostname..." local device_id device_id=$(get_tailscale_device_id "$hostname") if [[ -z "$device_id" ]]; then - warn "Device $hostname not found in Tailscale — may already be removed" + warn "$hostname not found in Tailscale — may already be removed" return 0 fi - info "Removing $hostname (device $device_id) from Tailscale..." + log "Removing $hostname (device $device_id) from Tailscale..." if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would remove $hostname from Tailscale tailnet" @@ -286,7 +327,7 @@ remove_tailscale_device() { curl -sf --max-time 10 -X DELETE \ -H "Authorization: Bearer $TAILSCALE_API_KEY" \ "https://api.tailscale.com/api/v2/devices/${device_id}" 2>/dev/null && \ - success "$hostname removed from Tailscale ✅" || \ + warn "$hostname removed from Tailscale ✅" || \ error "Failed to remove $hostname from Tailscale — remove manually" } @@ -297,41 +338,62 @@ check_both_healthy() { mirror_ip=$(tailscale ip -4 "$MIRROR" 2>/dev/null) [[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; } - ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout=10 root@"$mirror_ip" \ + timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \ "mountpoint -q /mnt/user && timeout 10 docker ps" >/dev/null 2>&1 || { - error "Mirror not healthy" + error "Mirror $MIRROR not healthy" return 1 } return 0 } do_final_sync() { - info "Running final Critical-Data sync..." + log "Running final Critical-Data and Emby sync..." if [[ "$DRY_RUN" == false ]]; then + # Use CRITICAL_SYNC_SHARES from HOST* conf if available + # Falls back to known paths — these are the critical ones bash "$SCRIPT_DIR/../Rsync/rsync.sh" \ - "/mnt/user/appdata-Failover/Critical-Data" --log + "/mnt/user/appdata-Failover/Critical-Data" \ + --profile=critical-failover --log bash "$SCRIPT_DIR/../Rsync/rsync.sh" \ - "/mnt/user/Media_Server/Emby" --profile=emby-failover --log + "/mnt/user/Media_Server/Emby" \ + --profile=emby-failover --log else - warn "DRY RUN — would run final Critical-Data and emby-failover sync" + warn "DRY RUN — would run final Critical-Data and Emby sync" fi - success "Final sync complete — mirror has current state ✅" + warn "Final sync complete — mirror has current state ✅" } -# ----------------------------------------------------------------------------------------------- -# ━━━ STATUS ━━━ -# ----------------------------------------------------------------------------------------------- +# Safe master.conf modification with error handling +update_master_conf() { + local key="$1" value="$2" + local conf="$SCRIPT_DIR/../master.conf" + if [[ ! -f "$conf" ]]; then + error "master.conf not found at $conf" + return 1 + fi + if sed -i "s|^[[:space:]]*${key}=.*| ${key}=${value}|" "$conf" 2>/dev/null; then + log "master.conf updated: ${key}=${value}" + return 0 + else + error "Failed to update master.conf: ${key}=${value}" + return 1 + fi +} + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$MODE" == "status" ]]; then echo "" echo "━━━━━ $ICON_SUMMARY PARTNERSHIP STATUS ━━━━━" - - OWNER_IP=$(tailscale ip -4 "$OWNER" 2>/dev/null || echo "unreachable") + OWNER_IP=$(tailscale ip -4 "$OWNER" 2>/dev/null || echo "unreachable") MIRROR_IP=$(tailscale ip -4 "$MIRROR" 2>/dev/null || echo "unreachable") - - echo " Owner: $OWNER ($OWNER_IP)" - echo " Mirror: $MIRROR ($MIRROR_IP)" + echo " $ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)" + echo " Owner: $OWNER_ID ($OWNER — $OWNER_IP)" + echo " Mirror: $MIRROR_ID ($MIRROR — $MIRROR_IP)" + echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )" echo " Enabled: ${PARTNERSHIP_ENABLED:-false}" - echo " Running as: $LOCAL_SERVER_NAME ($( [[ "$AM_OWNER" == true ]] && echo "owner" || echo "mirror"))" echo "" # Local state @@ -341,72 +403,65 @@ if [[ "$MODE" == "status" ]]; then LOCAL_OFFBOARDED=$(read_state_file "$LOCAL_STATE_FILE" "offboarded") LOCAL_LAST_SEEN=$(read_state_file "$LOCAL_STATE_FILE" "last_seen_remote") echo " Local state: $LOCAL_STATE" - [[ -n "$LOCAL_ONBOARDED" ]] && echo " Onboarded: $LOCAL_ONBOARDED" + [[ -n "$LOCAL_ONBOARDED" ]] && echo " Onboarded: $LOCAL_ONBOARDED" [[ -n "$LOCAL_OFFBOARDED" ]] && echo " Offboarded: $LOCAL_OFFBOARDED" - [[ -n "$LOCAL_LAST_SEEN" ]] && echo " Remote last seen: $LOCAL_LAST_SEEN" + [[ -n "$LOCAL_LAST_SEEN" ]] && echo " Remote seen: $LOCAL_LAST_SEEN" else echo " Local state: no state file found" fi echo "" - # Remote state via SSH + # Remote state REMOTE_IP=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) if [[ -n "$REMOTE_IP" ]]; then REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE") if [[ -n "$REMOTE_CONTENT" ]]; then REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2) REMOTE_ONBOARDED=$(echo "$REMOTE_CONTENT" | grep "^onboarded=" | cut -d= -f2) - REMOTE_OFFBOARDED=$(echo "$REMOTE_CONTENT" | grep "^offboarded=" | cut -d= -f2) echo " Remote state: $REMOTE_STATE" - [[ -n "$REMOTE_ONBOARDED" ]] && echo " Onboarded: $REMOTE_ONBOARDED" - [[ -n "$REMOTE_OFFBOARDED" ]] && echo " Offboarded: $REMOTE_OFFBOARDED" - - # Check agreement + [[ -n "$REMOTE_ONBOARDED" ]] && echo " Onboarded: $REMOTE_ONBOARDED" if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]]; then echo "" echo " ✅ Both servers agree: $LOCAL_STATE" else echo "" echo " ⚠️ State mismatch — local: $LOCAL_STATE remote: $REMOTE_STATE" - echo " Run --onboard or --offboard to resolve" fi else - echo " Remote state: state file not found on $REMOTE_SERVER_NAME" + echo " Remote state: not found on $REMOTE_SERVER_NAME" fi else echo " Remote state: $REMOTE_SERVER_NAME unreachable" fi echo "" - echo " Auth WebUIs configured:" + echo " Auth WebUIs:" for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do [[ -z "$entry" ]] && continue echo " ${entry%%|*} → port ${entry##*|}" done - # Offline counter if [[ -f "$OFFLINE_COUNTER" ]]; then OFFLINE_DAYS=$(cat "$OFFLINE_COUNTER" 2>/dev/null || echo 0) [[ "$OFFLINE_DAYS" -gt 0 ]] && \ echo "" && \ - echo " ⚠️ Remote offline counter: ${OFFLINE_DAYS}/${PARTNERSHIP_OFFLINE_THRESHOLD} days" + echo " ⚠️ Remote offline counter: ${OFFLINE_DAYS}/${PARTNERSHIP_OFFLINE_THRESHOLD}" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -# ----------------------------------------------------------------------------------------------- -# ━━━ CHECK (called every 15min by critical_sync_maintenance.sh) ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Check ━━━ +# Called every 15min by critical_sync_maintenance.sh — must be silent when healthy +# ============================================================================================== if [[ "$MODE" == "check" ]]; then # Update last_seen_remote and offline counter based on rsync outcome if [[ "$REMOTE_SEEN" == true ]]; then - OFFLINE_COUNT=0 echo "0" > "$OFFLINE_COUNTER" - # Update last_seen_remote in local state file if [[ -f "$LOCAL_STATE_FILE" ]]; then sed -i "s|^last_seen_remote=.*|last_seen_remote=$(date '+%Y-%m-%d %H:%M:%S')|" \ "$LOCAL_STATE_FILE" 2>/dev/null @@ -416,18 +471,16 @@ if [[ "$MODE" == "check" ]]; then OFFLINE_COUNT=$(( OFFLINE_COUNT + 1 )) echo "$OFFLINE_COUNT" > "$OFFLINE_COUNTER" - # Check auto-offboard threshold - # Convert 15min intervals to days: threshold_days * 96 intervals/day + # Auto-offboard threshold: threshold_days × 96 intervals/day (every 15min) THRESHOLD_INTERVALS=$(( ${PARTNERSHIP_OFFLINE_THRESHOLD:-30} * 96 )) if [[ "$OFFLINE_COUNT" -ge "$THRESHOLD_INTERVALS" ]]; then warn "Remote offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days — triggering auto-offboard" - notify "Partnership auto-offboard on $(hostname) — $REMOTE_SERVER_NAME offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days. Both servers will offboard independently." \ + notify "Partnership auto-offboard on $(hostname) — $REMOTE_SERVER_NAME offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days" \ "Partnership" "warning" - bash "$0" --offboard --reason="auto-offboard-timeout" + bash "$0" --offboard --reason=auto-offboard-timeout exit 0 fi - - log "Partnership check — remote unseen (count: $OFFLINE_COUNT/$THRESHOLD_INTERVALS)" + log "Partnership check — remote unseen ($OFFLINE_COUNT/$THRESHOLD_INTERVALS)" fi # Read remote state file @@ -443,7 +496,7 @@ if [[ "$MODE" == "check" ]]; then REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2) LOCAL_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state" 2>/dev/null || echo "UNKNOWN") - # Both agree — healthy + # Both agree and active — healthy, silent if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then log "Partnership check — ACTIVE, both servers agree ✅" exit 0 @@ -451,38 +504,41 @@ if [[ "$MODE" == "check" ]]; then # Remote requested offboard if [[ "$REMOTE_STATE" == "INACTIVE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then - info "Partnership check — $REMOTE_SERVER_NAME requested offboard" + warn "Partnership check — $REMOTE_SERVER_NAME requested offboard" if [[ "$AM_OWNER" == true ]]; then - info "Owner finalising offboard request from mirror..." - # Final sync — mirror leaves with current state + warn "Owner finalising offboard request from mirror..." do_final_sync - # Update local state + NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" \ "INACTIVE" "" "$NOW" "$REMOTE_SERVER_NAME" "mirror-requested" - # Disable critical rsync + if [[ "$DRY_RUN" == false ]]; then - sed -i 's/CRITICAL_RSYNC_ENABLED=true/CRITICAL_RSYNC_ENABLED=false/' \ - "$SCRIPT_DIR/../Master.conf" 2>/dev/null + update_master_conf "CRITICAL_RSYNC_ENABLED" "false" fi - # Tailscale removal after grace period + if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then - local grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-3} * 3600 )) - info "Waiting ${PARTNERSHIP_GRACE_HOURS:-3}hr grace period for mirror to see notification..." - [[ "$DRY_RUN" == false ]] && sleep "$grace_seconds" + local grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-6} * 3600 )) + warn "Waiting ${PARTNERSHIP_GRACE_HOURS:-6}hr grace period..." + # Grace sleep — interruptible via SIGTERM trap + if [[ "$DRY_RUN" == false ]]; then + trap 'warn "Partnership check interrupted during grace sleep"; exit 0' SIGTERM SIGINT + sleep "$grace_seconds" + trap - SIGTERM SIGINT + fi remove_tailscale_device "$MIRROR" fi - notify "Partnership offboard finalised on $(hostname) — $MIRROR requested separation" \ + notify "Partnership offboard finalised — $MIRROR requested separation" \ "Partnership" "normal" else # Mirror sees owner is INACTIVE — clean up own side - info "Owner has offboarded — cleaning up mirror side..." + warn "Owner has offboarded — cleaning up mirror side..." reconfigure_local_webuis "localhost" NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" \ "INACTIVE" "" "$NOW" "$OWNER" "owner-offboarded" - notify "Partnership ended on $(hostname) — $OWNER has offboarded. Auth WebUIs reconfigured to localhost." \ + notify "Partnership ended — $OWNER offboarded. Auth WebUIs → localhost." \ "Partnership" "normal" fi exit 0 @@ -498,9 +554,9 @@ if [[ "$MODE" == "check" ]]; then exit 0 fi -# ----------------------------------------------------------------------------------------------- -# ━━━ ONBOARD ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Onboard ━━━ +# ============================================================================================== if [[ "$MODE" == "onboard" ]]; then echo "" echo "━━━ $ICON_FAILOVER Onboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━" @@ -509,14 +565,13 @@ if [[ "$MODE" == "onboard" ]]; then if [[ -f "$LOCAL_STATE_FILE" ]]; then CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state") if [[ "$CURRENT_STATE" == "ACTIVE" ]]; then - info "Partnership already ACTIVE — no action needed" - info "Use --status for detail or --offboard to separate" + warn "Partnership already ACTIVE — use --status for detail or --offboard to separate" exit 0 fi fi - info "Owner: $OWNER" - info "Mirror: $MIRROR" + log "Owner: $OWNER_ID ($OWNER)" + log "Mirror: $MIRROR_ID ($MIRROR)" # Pre-flight echo "" @@ -524,17 +579,22 @@ if [[ "$MODE" == "onboard" ]]; then resolve_remote_ip check_connectivity - check_remote_array || exit 1 - check_remote_docker || exit 1 - OWNER_IP=$(tailscale ip -4 "$OWNER" 2>/dev/null) + # Version parity — both servers must agree on unRAID version + check_unraid_version_parity || exit 1 + + # Remote array and Docker daemon + check_remote_array || exit 1 + check_remote_docker_daemon || exit 1 + + OWNER_IP=$(tailscale ip -4 "$OWNER" 2>/dev/null) MIRROR_IP=$(tailscale ip -4 "$MIRROR" 2>/dev/null) - [[ -z "$OWNER_IP" ]] && error "Cannot resolve $OWNER Tailscale IP" && exit 1 - [[ -z "$MIRROR_IP" ]] && error "Cannot resolve $MIRROR Tailscale IP" && exit 1 + [[ -z "$OWNER_IP" ]] && { error "Cannot resolve $OWNER Tailscale IP"; exit 1; } + [[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; exit 1; } - success "Owner IP: $OWNER_IP" - success "Mirror IP: $MIRROR_IP" + log "Owner IP: $OWNER_IP" + log "Mirror IP: $MIRROR_IP" # Reconfigure mirror WebUIs → owner IP echo "" @@ -546,19 +606,19 @@ if [[ "$MODE" == "onboard" ]]; then container="${entry%%|*}" port="${entry##*|}" reconfigure_webui "$container" "$port" "$OWNER_IP" \ - "$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || ((WEBUI_FAILURES++)) + "$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ )) done # Verify WebUI connectivity if [[ "${PARTNERSHIP_ONBOARD_VERIFY:-true}" == true ]]; then echo "" - echo "━━━ $ICON_HEALTH Verify WebUI Connectivity ━━━" + echo "━━━ $ICON_VERIFY WebUI Connectivity ━━━" for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do [[ -z "$entry" ]] && continue container="${entry%%|*}" port="${entry##*|}" if curl -sf --max-time 10 "http://${OWNER_IP}:${port}/" >/dev/null 2>&1; then - success "$container reachable at http://${OWNER_IP}:${port}/ ✅" + log "$container reachable at http://${OWNER_IP}:${port}/ ✅" else warn "$container not reachable at http://${OWNER_IP}:${port}/ — may not be running" fi @@ -571,32 +631,28 @@ if [[ "$MODE" == "onboard" ]]; then NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "onboard" - success "Local state: ACTIVE ✅" + log "Local state: ACTIVE ✅" - # Push to remote push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY" - - # Reset offline counter echo "0" > "$OFFLINE_COUNTER" # Summary echo "" echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━" - echo " Owner: $OWNER ($OWNER_IP)" - echo " Mirror: $MIRROR ($MIRROR_IP)" + echo " Owner: $OWNER_ID ($OWNER_IP)" + echo " Mirror: $MIRROR_ID ($MIRROR_IP)" echo " WebUI failures: $WEBUI_FAILURES" echo " Sync interval: ${PARTNERSHIP_SYNC_INTERVAL}min" echo "" - if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no changes made" + warn "DRY RUN — no changes made" elif [[ "$WEBUI_FAILURES" -eq 0 ]]; then - echo "$ICON_DONE Status: $ICON_SUCCESS DONE" + warn "$ICON_DONE DONE — $MIRROR is now mirroring $OWNER via Tailscale ✅" [[ "${PARTNERSHIP_ONBOARD_NOTIFY:-true}" == true ]] && \ notify "Partnership onboard complete — $MIRROR is now mirroring $OWNER via Tailscale" \ "Partnership" "normal" else - echo "$ICON_WARN Status: DONE with $WEBUI_FAILURES WebUI warning(s)" + warn "DONE with $WEBUI_FAILURES WebUI warning(s) — check manually" notify "Partnership onboard complete with $WEBUI_FAILURES WebUI warning(s) on $(hostname)" \ "Partnership" "warning" fi @@ -604,9 +660,9 @@ if [[ "$MODE" == "onboard" ]]; then exit 0 fi -# ----------------------------------------------------------------------------------------------- -# ━━━ OFFBOARD ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Offboard ━━━ +# ============================================================================================== if [[ "$MODE" == "offboard" ]]; then echo "" echo "━━━ $ICON_FAILOVER Offboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━" @@ -615,19 +671,16 @@ if [[ "$MODE" == "offboard" ]]; then if [[ -f "$LOCAL_STATE_FILE" ]]; then CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state") if [[ "$CURRENT_STATE" == "INACTIVE" ]]; then - info "Partnership already INACTIVE" - info "Use --status to verify both servers agree" + warn "Partnership already INACTIVE — use --status to verify both servers agree" exit 0 fi fi - REASON="${REASON:-manual}" - - # ── MIRROR-initiated offboard ──────────────────────────────────────────────────────────── + # ── Mirror-initiated offboard ───────────────────────────────────────────────────────────── if [[ "$AM_MIRROR" == true ]]; then - warn "$MIRROR is initiating offboard" - warn "This will reconfigure your local auth WebUIs → localhost" - warn "HOST1 will finalise the offboard on its next --check cycle" + warn "$MIRROR_ID ($MIRROR) is initiating offboard" + warn "Local auth WebUIs will be reconfigured → localhost" + warn "$OWNER_ID ($OWNER) will finalise on its next --check cycle" if [[ "$DRY_RUN" == false ]]; then echo "" @@ -635,22 +688,19 @@ if [[ "$MODE" == "offboard" ]]; then sleep 10 fi - # Reconfigure own WebUIs → localhost echo "" echo "━━━ $ICON_CONTAINERS Reconfigure Local WebUIs → localhost ━━━" reconfigure_local_webuis "localhost" - # Write own state file NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" \ "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON" - success "Local state: INACTIVE ✅" + log "Local state: INACTIVE ✅" - # Try to push state to owner immediately OWNER_IP=$(tailscale ip -4 "$OWNER" 2>/dev/null) if [[ -n "$OWNER_IP" ]]; then push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$MIRROR_SSH_KEY" - notify "Partnership offboard requested by $MIRROR — owner will finalise on next check cycle" \ + notify "Partnership offboard requested by $MIRROR — $OWNER will finalise on next check" \ "Partnership" "normal" else warn "$OWNER unreachable — state written locally, owner will see it when reachable" @@ -658,17 +708,16 @@ if [[ "$MODE" == "offboard" ]]; then echo "" echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY ━━━━━" - echo " Your WebUIs: reconfigured → localhost ✅" - echo " State: INACTIVE written ✅" - echo " Owner: will finalise + run final sync on next --check ✅" + echo " Your WebUIs: reconfigured → localhost ✅" + echo " State: INACTIVE ✅" + echo " Owner: will finalise + final sync on next --check ✅" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi - # ── OWNER-initiated offboard ───────────────────────────────────────────────────────────── - warn "Offboarding $MIRROR from partnership" + # ── Owner-initiated offboard ────────────────────────────────────────────────────────────── + warn "Offboarding $MIRROR_ID ($MIRROR) from partnership" warn "Final sync will run — mirror leaves with current state" - warn "$MIRROR auth WebUIs will be reconfigured → localhost" if [[ "$DRY_RUN" == false ]]; then echo "" @@ -682,7 +731,7 @@ if [[ "$MODE" == "offboard" ]]; then # Stop any running rsync first echo "" echo "━━━ $ICON_STOP Stop Running Rsync ━━━" - bash "$SCRIPT_DIR/../unRAID_Essentials/rsync_stop.sh" --rsync-only 2>/dev/null || true + bash "$SCRIPT_DIR/../Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true # Final sync echo "" @@ -704,21 +753,20 @@ if [[ "$MODE" == "offboard" ]]; then container="${entry%%|*}" port="${entry##*|}" reconfigure_webui "$container" "$port" "localhost" \ - "$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || ((WEBUI_FAILURES++)) + "$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ )) done else warn "$MIRROR unreachable — WebUI reconfiguration skipped" - warn "$MIRROR will reconfigure its own WebUIs when it sees INACTIVE state on next --check" - ((WEBUI_FAILURES++)) + warn "$MIRROR will reconfigure its own WebUIs when it sees INACTIVE state on --check" + (( WEBUI_FAILURES++ )) fi # Disable critical rsync echo "" echo "━━━ $ICON_GEAR Disable Critical Sync ━━━" if [[ "$DRY_RUN" == false ]]; then - sed -i 's/CRITICAL_RSYNC_ENABLED=true/CRITICAL_RSYNC_ENABLED=false/' \ - "$SCRIPT_DIR/../Master.conf" 2>/dev/null - success "CRITICAL_RSYNC_ENABLED=false ✅" + update_master_conf "CRITICAL_RSYNC_ENABLED" "false" + warn "CRITICAL_RSYNC_ENABLED=false ✅" else warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false" fi @@ -730,9 +778,8 @@ if [[ "$MODE" == "offboard" ]]; then write_state_file "$LOCAL_STATE_FILE" \ "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON" - success "Local state: INACTIVE ✅" + log "Local state: INACTIVE ✅" - # Push to mirror if [[ "$MIRROR_REACHABLE" == true ]]; then push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY" fi @@ -742,9 +789,13 @@ if [[ "$MODE" == "offboard" ]]; then echo "" echo "━━━ $ICON_NET Tailscale Separation ━━━" if [[ "$MIRROR_REACHABLE" == true ]]; then - local grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-3} * 3600 )) - info "Waiting ${PARTNERSHIP_GRACE_HOURS:-3}hr grace period for mirror to see state..." - [[ "$DRY_RUN" == false ]] && sleep "$grace_seconds" + grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-6} * 3600 )) + warn "Waiting ${PARTNERSHIP_GRACE_HOURS:-6}hr grace — mirror can collect backups..." + if [[ "$DRY_RUN" == false ]]; then + trap 'warn "Offboard interrupted during grace sleep"; exit 0' SIGTERM SIGINT + sleep "$grace_seconds" + trap - SIGTERM SIGINT + fi fi remove_tailscale_device "$MIRROR" fi @@ -753,51 +804,42 @@ if [[ "$MODE" == "offboard" ]]; then if [[ ${#PARTNERSHIP_MIRROR_BACKUPS[@]} -gt 0 ]]; then echo "" echo "━━━ $ICON_DISK Backup Handover ━━━" - info "Backups available for $MIRROR:" + log "Backups available for $MIRROR:" for path in "${PARTNERSHIP_MIRROR_BACKUPS[@]}"; do [[ -z "$path" ]] && continue echo " $path" done - info "Available for ${PARTNERSHIP_GRACE_HOURS:-3} hours — Tailscale access removed after this window" - notify "$MIRROR offboard complete — backups available for ${PARTNERSHIP_GRACE_HOURS:-3}hr. Tailscale access expires then. Certs valid ~60-90 days. Run tailscale up to rejoin." \ + notify "$MIRROR offboard complete — backups available for ${PARTNERSHIP_GRACE_HOURS:-6}hr. Tailscale access expires then." \ "Partnership" "warning" fi # Summary echo "" echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY ━━━━━" - echo " Owner: $OWNER" - echo " Mirror: $MIRROR" - echo " Final sync: complete ✅" + echo " Owner: $OWNER_ID ($OWNER)" + echo " Mirror: $MIRROR_ID ($MIRROR)" + echo " Final sync: complete ✅" echo " WebUI failures: $WEBUI_FAILURES" echo " Critical rsync: disabled ✅" - echo " State: INACTIVE ✅" + echo " State: INACTIVE ✅" [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]] && \ - echo " Tailscale: $MIRROR removed ✅" + echo " Tailscale: $MIRROR removed ✅" echo "" echo " $MIRROR leaves with:" echo " ✓ Current auth config (final sync)" - echo " ✓ His own DDNS and updater" - echo " ✓ His own Git mirror" echo " ✓ Auth WebUIs → localhost" - echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-3}hr to collect backups before Tailscale removed" - echo " ✓ Certs valid ~60-90 more days" + echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-6}hr to collect backups" echo " ✓ Full ecosystem — just stop the sync" - - if [[ "$DRY_RUN" == true ]]; then - echo "" - echo "$ICON_WARN Status: DRY RUN — no changes made" - else - echo "" - echo "$ICON_DONE Status: $ICON_SUCCESS DONE — clean separation complete" - fi + echo "" + [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \ + warn "$ICON_DONE DONE — clean separation complete ✅" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -# ----------------------------------------------------------------------------------------------- -# ━━━ TRANSFER ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Transfer ━━━ +# ============================================================================================== if [[ "$MODE" == "transfer" ]]; then echo "" echo "━━━ $ICON_FAILOVER Transfer Ownership — $(date '+%Y-%m-%d %H:%M:%S') ━━━" @@ -805,19 +847,19 @@ if [[ "$MODE" == "transfer" ]]; then echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "⚠️ WARNING — OWNERSHIP TRANSFER" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo " Current owner: $OWNER" - echo " Current mirror: $MIRROR" + echo " Current owner: $OWNER_ID ($OWNER)" + echo " Current mirror: $MIRROR_ID ($MIRROR)" echo "" echo " After transfer:" - echo " New owner: $MIRROR" - echo " New mirror: $OWNER" + echo " New owner: $MIRROR_ID ($MIRROR)" + echo " New mirror: $OWNER_ID ($OWNER)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - # Confirmation string + # Confirmation check if [[ "$DRY_RUN" == false ]]; then if [[ -z "$TRANSFER_CONFIRM_INPUT" ]]; then echo "" - echo "To proceed type exactly:" + echo "To proceed pass exactly:" echo " --confirm=${PARTNERSHIP_TRANSFER_CONFIRM}" echo "" error "Transfer cancelled — confirmation required" @@ -827,7 +869,7 @@ if [[ "$MODE" == "transfer" ]]; then error "Confirmation string does not match — transfer cancelled" exit 1 fi - success "Confirmation accepted" + log "Confirmation accepted" else warn "DRY RUN — confirmation check skipped" fi @@ -835,14 +877,14 @@ if [[ "$MODE" == "transfer" ]]; then # Health strike system echo "" echo "━━━ $ICON_SHIELD Health Verification ━━━" - info "Both servers must pass ${PARTNERSHIP_TRANSFER_STRIKES} consecutive health checks" + log "Both servers must pass ${PARTNERSHIP_TRANSFER_STRIKES} consecutive health checks" STRIKES=0 ATTEMPTS=0 MAX_ATTEMPTS="${PARTNERSHIP_TRANSFER_MAX_ATTEMPTS:-20}" while [[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]]; do - ((ATTEMPTS++)) + (( ATTEMPTS++ )) if [[ "$ATTEMPTS" -gt "$MAX_ATTEMPTS" ]]; then error "Health checks failed after $MAX_ATTEMPTS attempts — servers not stable" error "Transfer cancelled — try again when both servers are healthy" @@ -850,8 +892,8 @@ if [[ "$MODE" == "transfer" ]]; then fi if check_both_healthy; then - ((STRIKES++)) - success "Health check passed ($STRIKES/${PARTNERSHIP_TRANSFER_STRIKES})" + (( STRIKES++ )) + log "Health check passed ($STRIKES/${PARTNERSHIP_TRANSFER_STRIKES})" [[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]] && sleep 10 else warn "Health check failed — resetting (attempt $ATTEMPTS/$MAX_ATTEMPTS)" @@ -859,14 +901,22 @@ if [[ "$MODE" == "transfer" ]]; then sleep 30 fi done - success "Both servers healthy — proceeding" + warn "Both servers healthy — proceeding with transfer" + NEW_OWNER_ID="$MIRROR_ID" + NEW_MIRROR_ID="$OWNER_ID" NEW_OWNER="$MIRROR" NEW_MIRROR="$OWNER" - NEW_OWNER_IP=$(tailscale ip -4 "$NEW_OWNER" 2>/dev/null) + NEW_OWNER_SSH_KEY_VAR="${NEW_OWNER_ID}_SSH_KEY" + NEW_MIRROR_SSH_KEY_VAR="${NEW_MIRROR_ID}_SSH_KEY" + NEW_OWNER_SSH_KEY="${!NEW_OWNER_SSH_KEY_VAR}" + NEW_MIRROR_SSH_KEY="${!NEW_MIRROR_SSH_KEY_VAR}" + + NEW_OWNER_IP=$(tailscale ip -4 "$NEW_OWNER" 2>/dev/null) NEW_MIRROR_IP=$(tailscale ip -4 "$NEW_MIRROR" 2>/dev/null) - [[ -z "$NEW_OWNER_IP" ]] && error "Cannot resolve new owner Tailscale IP" && exit 1 + [[ -z "$NEW_OWNER_IP" ]] && { error "Cannot resolve new owner Tailscale IP"; exit 1; } + [[ -z "$NEW_MIRROR_IP" ]] && { error "Cannot resolve new mirror Tailscale IP"; exit 1; } # Final sync in current direction echo "" @@ -876,12 +926,7 @@ if [[ "$MODE" == "transfer" ]]; then # Reconfigure new mirror WebUIs → new owner echo "" echo "━━━ $ICON_CONTAINERS Reconfigure WebUIs ━━━" - info "New mirror ($NEW_MIRROR) WebUIs → new owner ($NEW_OWNER_IP)" - - NEW_MIRROR_SSH="${HOST1_SSH_KEY}" - [[ "$NEW_MIRROR" == "$HOST2" ]] && NEW_MIRROR_SSH="${HOST2_SSH_KEY}" - NEW_OWNER_SSH="${HOST1_SSH_KEY}" - [[ "$NEW_OWNER" == "$HOST2" ]] && NEW_OWNER_SSH="${HOST2_SSH_KEY}" + log "New mirror ($NEW_MIRROR) WebUIs → new owner ($NEW_OWNER_IP)" WEBUI_FAILURES=0 for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do @@ -889,53 +934,50 @@ if [[ "$MODE" == "transfer" ]]; then container="${entry%%|*}" port="${entry##*|}" reconfigure_webui "$container" "$port" "$NEW_OWNER_IP" \ - "$NEW_MIRROR_SSH" "$NEW_MIRROR_IP" "$NEW_MIRROR" || ((WEBUI_FAILURES++)) + "$NEW_MIRROR_SSH_KEY" "$NEW_MIRROR_IP" "$NEW_MIRROR" || (( WEBUI_FAILURES++ )) done # New owner WebUIs → localhost (now manages directly) - info "New owner ($NEW_OWNER) WebUIs → localhost" + log "New owner ($NEW_OWNER) WebUIs → localhost" reconfigure_local_webuis "localhost" - # Flip PARTNERSHIP_OWNER_HOST in Master.conf on both servers + # Flip PARTNERSHIP_OWNER_HOST in master.conf on both servers echo "" echo "━━━ $ICON_GEAR Update Ownership ━━━" - NEW_OWNER_HOST="HOST1" - [[ "$NEW_OWNER" == "$HOST2" ]] && NEW_OWNER_HOST="HOST2" if [[ "$DRY_RUN" == false ]]; then - sed -i "s/PARTNERSHIP_OWNER_HOST=.*/PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_HOST\"/" \ - "$SCRIPT_DIR/../Master.conf" 2>/dev/null - success "Local Master.conf updated: PARTNERSHIP_OWNER_HOST=$NEW_OWNER_HOST ✅" + update_master_conf "PARTNERSHIP_OWNER_HOST" "\"$NEW_OWNER_ID\"" - # Update remote - ssh -i "$NEW_MIRROR_SSH" -o ConnectTimeout=10 root@"$NEW_MIRROR_IP" \ - "sed -i 's/PARTNERSHIP_OWNER_HOST=.*/PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_HOST\"/' \ - '$SCRIPT_DIR/../Master.conf'" 2>/dev/null && \ - success "Remote Master.conf updated ✅" || \ - error "Failed to update remote Master.conf — update manually" + # Update remote master.conf + timeout "$SSH_TIMEOUT" ssh -i "$NEW_MIRROR_SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" root@"$NEW_MIRROR_IP" \ + "sed -i 's|^[[:space:]]*PARTNERSHIP_OWNER_HOST=.*| PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\"|' \ + '$SCRIPT_DIR/../master.conf'" 2>/dev/null && \ + log "Remote master.conf updated ✅" || \ + error "Failed to update remote master.conf — update manually" else - warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_HOST on both servers" + warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID on both servers" fi # Write state files NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "transfer" - push_state_to_remote "$LOCAL_STATE_FILE" "$NEW_MIRROR_IP" "$NEW_MIRROR_SSH" + push_state_to_remote "$LOCAL_STATE_FILE" "$NEW_MIRROR_IP" "$NEW_MIRROR_SSH_KEY" # Summary echo "" echo "━━━━━ $ICON_SUMMARY TRANSFER SUMMARY ━━━━━" - echo " New owner: $NEW_OWNER ($NEW_OWNER_IP)" - echo " New mirror: $NEW_MIRROR ($NEW_MIRROR_IP)" - echo " WebUI failures: $WEBUI_FAILURES" - echo " Sync direction: $NEW_OWNER → $NEW_MIRROR" - echo " Master.conf: PARTNERSHIP_OWNER_HOST=$NEW_OWNER_HOST" - + echo " New owner: $NEW_OWNER_ID ($NEW_OWNER_IP)" + echo " New mirror: $NEW_MIRROR_ID ($NEW_MIRROR_IP)" + echo " WebUI failures: $WEBUI_FAILURES" + echo " Sync direction: $NEW_OWNER → $NEW_MIRROR" + echo " Owner host: PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID" + echo "" if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no changes made" + warn "DRY RUN — no changes made" else - echo "$ICON_DONE Status: $ICON_SUCCESS DONE — ownership transferred" - notify "Partnership ownership transferred — new owner: $NEW_OWNER new mirror: $NEW_MIRROR" \ + warn "$ICON_DONE DONE — ownership transferred to $NEW_OWNER_ID ✅" + notify "Partnership ownership transferred — new owner: $NEW_OWNER ($NEW_OWNER_ID)" \ "Partnership" "normal" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" diff --git a/README-User_Script_Plug-in.md b/README-User_Script_Plug-in.md index 3a73ca7..22e68aa 100644 --- a/README-User_Script_Plug-in.md +++ b/README-User_Script_Plug-in.md @@ -1,363 +1,1711 @@ -# User Scripts Setup +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# 📋 USER SCRIPTS PLUGIN — COMPLETE SETUP REFERENCE +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -How to configure the unRAID User Scripts plugin to match the repository structure exactly. Matching names and categories between the plugin and the repo means your eyes find what they're looking for instantly — no mental translation between "what did I call it in the plugin" and "what's it called in git". +**Every script entry for the User Scripts plugin — what it does, when it runs, +how to set it up, and what to watch for.** No cross-referencing other READMEs to +understand the basics. The deeper documentation exists — this is your daily reference. --- -## Required Plugins +## ━━━ REQUIRED PLUGINS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Install both from Community Applications: - -**User Scripts** — Andrew Zawadzki -The base plugin. Runs scripts via the unRAID UI without command line access. Required. - -**User Scripts Enhanced** — Kurotaku -Adds category support, UI customization, and additional features. Required for category organization. - -Both must be installed before setting up the script entries below. - ---- - -## Category Structure - -Categories match the repository folder names exactly: +Install both from Community Applications before creating any entries: ``` -Failover -Monitors -Orchestrators -Rsync -Docker_Essentials -Media -Transcodes -Tools -unRAID_Essentials -Git +User Scripts — Andrew Zawadzki + Core plugin. Runs scripts via unRAID UI without terminal. + Required — nothing works without this. + +User Scripts Enhanced — Kurotaku + Adds categories, drag-to-reorder, UI improvements. + Required for the category organisation used here. ``` -Create these categories in User Scripts Enhanced before adding scripts. When the plugin category and the repo folder name are identical, navigating between VS Code and the plugin requires zero mental overhead. +--- + +## ━━━ GLOBAL RULES — READ BEFORE CREATING ENTRIES ━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +``` +1. Every script entry → Run in Background: YES + Without background mode the plugin UI freezes waiting for the script to complete. + Long-running scripts and continuous loops (failover, watchdogs) require background. + There are no exceptions — set background on everything. + +2. Run as root: YES + All scripts require root. Docker operations, rsync, kernel tuning, reboots. + +3. Category names match repo folder names exactly + When the plugin category and the git folder are identical, you never wonder + which folder a script lives in. No mental translation. + +4. Script name matches the script filename (without .sh) + daily_sync_maintenance.sh → name: daily_sync_maintenance + When logs reference a script name, you find the plugin entry immediately. + +5. --dry-run exists on every script + Test anything unfamiliar with --dry-run first. Pre-flight checks still run. + It shows what would happen without making changes. + +6. Most scripts never need to be run directly + The orchestrators handle timing and order. Individual script entries exist + for manual testing and one-off maintenance — not as replacement cron jobs. +``` --- -## Script Entries +## ━━━ CATEGORY STRUCTURE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -For each entry below: -1. Create a new User Script -2. Set the **Name** exactly as shown -3. Set the **Category** as shown -4. Paste the **Script** line into the script body -5. Set the **Schedule** as shown +Create these categories in User Scripts Enhanced before adding any entries: + +``` +Orchestrators — scheduled maintenance windows and startup launcher +Failover — mutual failover state machine and test tools +Monitors — weekly reports, health checks, trend tracking +Rsync — manual per-profile sync triggers +Docker_Essentials — container management tools (manual use) +Media — library maintenance tools (manual use) +Transcodes — ramdisk and transcode tools (manual use) +unRAID_Essentials — server management tools (manual use) +Partnership — two-server relationship tools (manual use) +Tools — repair, recovery, and one-time utilities +Git — script deployment +``` --- -### Failover +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🎯 ORCHESTRATORS +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -| Name | Schedule | Background | -|------|----------|------------| -| `failover` | At Startup of Array | ✅ Yes — runs continuously | -| `failover_test` | None — run manually | ❌ No | +Orchestrators are the only scripts that run on a schedule. They coordinate multiple +child scripts in the correct order. Adding or removing a job means editing master.conf +— never the orchestrator script itself. + +--- + +### ── array_start ───────────────────────────────────────────────────────────── + +``` +Category: Orchestrators +Name: array_start +Schedule: At Startup of Array +Background: YES — required (starts continuous background processes) +``` ```bash -# failover -/mnt/user/appdata/unraid_scripts/Failover/failover.sh - -# failover_test — always --dry-run first -/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh --dry-run +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh ``` -> ⚠️ `failover.sh` **must** run as a background task. Set "Run in background" in the script settings. If run in foreground the plugin UI will hang waiting for it to complete — it never does, it loops forever. +**What it does:** +The single "At Startup of Array" entry for the entire ecosystem. Launches every +startup script in the order defined by ARRAY_START_SCRIPTS in master.conf. You +never add other startup scripts directly to User Scripts — add them to master.conf. + +**What it starts (in order):** +``` +inotify_tuning.sh — raises kernel inotify limits before containers start + containers inherit limits at startup — must be first +docker_syslog_filter.sh — suppresses Docker veth noise before containers create interfaces +php_fpm_max_children.sh — sets PHP-FPM workers to 250 before WebGUI serves requests +ramdisk_setup.sh — creates /mnt/ramdisk_transcodes and symlink before Emby starts +docker_network_connect.sh— ensures custom networks exist + containers connected +system_watchdog.sh ← continuous — three-tier server stability watchdog +docker_watchdog.sh ← continuous — two-tier container healing watchdog +failover.sh ← continuous — mutual failover state machine +``` + +**After running:** +One-shot scripts exit cleanly. Three continuous processes run in the background until +the array stops. Check with: `pgrep -f "system_watchdog\|docker_watchdog\|failover"` --- -### Monitors +### ── transcode_management ───────────────────────────────────────────────────── -| Name | Schedule | -|------|----------| -| `backup_verify` | `0 10 * * 0` (Sunday 10am) | -| `bandwidth_monitor_report` | `0 11 * * 0` (Sunday 11am) | -| `cert_monitor` | `0 9 * * 0` (Sunday 9am) | -| `emby_session_report` | `0 11 * * 0` (Sunday 11am) | -| `smart_health` | `0 7 * * 0` (Sunday 7am) | -| `weekly_health_digest` | `0 8 * * *` (8am daily) | -| `zfs_memory_snapshot` | `0 6 * * 0` (Sunday 6am) | - -```bash -# backup_verify -/mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh - -# bandwidth_monitor_report -/mnt/user/appdata/unraid_scripts/Monitors/bandwidth_monitor.sh --report - -# cert_monitor -/mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh - -# emby_session_report -/mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh - -# smart_health -/mnt/user/appdata/unraid_scripts/Monitors/smart_health.sh - -# weekly_health_digest -/mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh - -# zfs_memory_snapshot -/mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh +``` +Category: Orchestrators +Name: transcode_management +Schedule: */3 * * * * (every 3 minutes) +Background: YES ``` ---- - -### Orchestrators - -| Name | Schedule | -|------|----------| -| `daily_sync` | `0 1 * * *` (1am daily) | -| `media_management` | `0 2 * * *` (2am daily) | -| `transcode_management` | `*/3 * * * *` (every 3 min) | - ```bash -# daily_sync -/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync.sh - -# media_management -/mnt/user/appdata/unraid_scripts/Orchestrators/media_management.sh - -# transcode_management +#!/bin/bash /mnt/user/appdata/unraid_scripts/Orchestrators/transcode_management.sh ``` +**What it does:** +Runs transcode_cleanup.sh then transcode_manager.sh — in that exact order every +3 minutes. Order is non-negotiable: cleanup removes stale files first so the manager +sees real active session usage, not inflated numbers from ended sessions. + +**transcode_cleanup.sh:** +Removes HLS segment files older than TRANSCODE_MAX_AGE minutes that are not currently +open by any process. Uses one lsof call per location — not one per file. Covers both +ramdisk and SSD fallback. Never touches the transcoding-temp directory. + +**transcode_manager.sh:** +Checks ramdisk usage against RAMDISK_WARN_GB. Flips the /mnt/ram-transcode symlink +between ramdisk and SSD fallback based on current usage. Shows active Emby sessions +with their play method. Writes one daily stats entry to TRANSCODE_DAILY_LOG. +Runs safety checks every cycle: missing symlink → recreate, ramdisk disappeared → +auto-flip to SSD, missing transcoding-temp → recreate immediately. + +**Output when healthy:** session list only — no flip messages = ramdisk managing fine + +> ⚠️ The Emby Docker container must have +> `--mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared` +> in Extra Parameters. Without `shared` propagation, symlink flips are silently +> ignored after the first flip and all sessions land permanently on SSD. + --- -### Rsync +### ── critical_sync_maintenance ─────────────────────────────────────────────── -| Name | Schedule | -|------|----------| -| `rsync_arrs_stack` | As needed | -| `rsync_critical_data` | As needed | -| `rsync_important_data` | As needed | -| `rsync_emby` | As needed | -| `rsync_gmer4lfe` | As needed | +``` +Category: Orchestrators +Name: critical_sync_maintenance +Schedule: */15 * * * * (every 15 minutes) +Background: YES +``` ```bash -# rsync_arrs_stack -/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Orchestrators/critical_sync_maintenance.sh +``` -# rsync_critical_data -/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Critical-Data +**What it does:** +Keeps the auth stack and Emby watch state current on HOST2 — 15 minutes is the +maximum staleness for failover-critical data. -# rsync_important_data -/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Important-Data +**Jobs in order:** +``` +rsync Critical-Data — pushes auth stack to HOST2 (NPM rules, certs, LLDAP users, + Authelia config). HOST2's auth is always within 15min of HOST1. +rsync Emby dirty sync — pushes watch states, user activity, library delta to HOST2 + with Emby running. WAL files excluded — safe mid-write. + HOST2 Emby restarts after sync to pick up config changes. +partnership --check — reads remote state file, increments offline counter if HOST2 + unreachable, triggers auto-offboard at PARTNERSHIP_OFFLINE_THRESHOLD +``` -# rsync_emby -/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Emby +**Silent when healthy:** no output on clean cycles — runs 96 times per day. -# rsync_gmer4lfe -/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Gmer4Lfe +--- + +### ── arrs_failed_stalled_recovery ─────────────────────────────────────────── + +``` +Category: Orchestrators +Name: arrs_failed_stalled_recovery +Schedule: 0 */6 * * * (every 6 hours) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Orchestrators/arrs_failed_stalled_recovery.sh +``` + +**What it does:** +Automatically recovers failed imports and stalled downloads across Sonarr, Radarr, +and Lidarr. A release that downloaded but failed to import sits in the queue +permanently without this — no arr retries importFailed items on its own. + +**What it finds and acts on:** +``` +importFailed — downloaded but couldn't import → blocklist + remove + new search +importPending — stuck in import queue → blocklist + remove + new search +error — serious failure state → blocklist + remove + new search +stalled — no seeders or no progress → blocklist + remove + new search +``` + +**What it never touches:** items with status "downloading" or "imported". +Age gate: items newer than ARR_IMPORT_RECOVERY_AGE (default: 6 hours) are skipped — +the arr may still retry on its own. + +**Result:** failed imports are automatically replaced while you sleep. + +--- + +### ── daily_sync_maintenance ────────────────────────────────────────────────── + +``` +Category: Orchestrators +Name: daily_sync_maintenance +Schedule: 0 1 * * * (1am daily) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh +``` + +**What it does:** +The full 1am daily maintenance window — everything that needs to happen overnight +in one orchestrated sequence. + +**Execution order:** +``` +1. git_pull_execute.sh — pull latest scripts FIRST, everything below runs on + current code + +2. rsync DAILY_SYNC_SHARES — push this server's truth shares to the other server + HOST1 pushes: Movies, Tv_Shows, Music, Kids content, Sports... + HOST2 pushes: Anime_Shows, Anime_Movies + Personal encrypted shares appended after — encrypted blocks, remote can't read + + Drive temp exit codes: + exit 1 (WARN) → skip this share, continue to next + exit 2 (CRIT) → abort ALL remaining syncs, notify immediately + +3. media_shares_permissions.sh — apply nobody:users 755/664 to all media shares + arr cleanup depends on correct ownership + +4. media_cleaner.sh anime — remove .sfv .nfo .rar .sample and other scene junk + media_cleaner.sh media from anime and media share folders + junk removal BEFORE orphan detection — cleaner results + +5. lidarr_cleanup.sh — identify and delete orphaned music files not tracked + sonarr_cleanup.sh by the arr API. 7 safety checks must all pass before + radarr_cleanup.sh a single file is deleted. notify_emby_scan() called + after any deletion — Emby library cleaned immediately + +6. docker_daily_restart.sh — restart connection-heavy containers that degrade over time + NginxProxyManager, Authelia, Dispatcharr, ErsatzTV-Emby + Running → docker restart. Stopped → left stopped. +``` + +**Notify:** silent on success. One notification on failure with count of failed jobs. +**Both servers run this independently at 1am** — same script, correct direction via detect_hosts(). + +--- + +### ── rsync_emby_failover ───────────────────────────────────────────────────── + +``` +Category: Orchestrators +Name: rsync_emby_failover +Schedule: */30 * * * * (every 30 minutes) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ + /mnt/user/Media_Server/Emby --profile=emby-failover +``` + +**What it does:** +Keeps HOST2's Emby within 30 minutes of HOST1 on watch states, play positions, and +library changes — with Emby running on both sides. Not a full sync (that's weekly) — +this is continuous coverage of what matters for failover. + +**What syncs (emby-failover profile):** +``` +Syncs: users.db, library.db, authentication.db, config/ +Skips: *.wal *.shm — WAL files being written by Emby, unsafe to copy mid-write + transcodes/ — volatile, never needed on HOST2 + logs/ — local diagnostics only + cache/ — rebuilt automatically +``` + +**HOST2 Emby restarts after each sync** to pick up config changes. +**Result:** if HOST1 goes down, users resume from a position at most 30min stale. + +--- + +### ── weekly_sync_maintenance ───────────────────────────────────────────────── + +``` +Category: Orchestrators +Name: weekly_sync_maintenance +Schedule: 30 2 * * 0 (Sunday 2:30am) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Orchestrators/weekly_sync_maintenance.sh +``` + +**What it does:** +The full Sunday maintenance window — clean sync with containers stopped, image +updates at zero extra downtime, weekly container restarts. + +**Execution order:** +``` +1. Pre-flight — connectivity, remote Docker daemon, remote rootfs +2. Stop local containers — Emby + auth stack stopped on this server +3. Stop remote containers— Emby + auth stack stopped on remote via SSH +4. Pull images locally — if WEEKLY_SYNC_UPDATES=true (containers already stopped, + free downtime, both servers start on identical image versions) +5. Pull images remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true (SSH to remote) +6. rsync WEEKLY_SYNC_SHARES: + /mnt/user/Media_Server/Emby — full clean mirror, databases checkpointed + /mnt/user/appdata-Failover/Critical-Data — auth stack clean state +7. Start remote containers— dependency order, new image, settle + verify each +8. Start local containers — same +9. docker_weekly_restart.sh — restarts less-critical services that benefit from + weekly clean start: NextCloud, AdGuard, Immich +``` + +**Why containers stop for Emby sync:** +Weekly sync is a full clean mirror including databases. Emby must be stopped to +checkpoint WAL — ensures a consistent database copy, not a snapshot mid-write. +~30 seconds of downtime. The 30-minute dirty sync covers users until this clean sync. + +**Why weekly and not nightly:** +Emby builds a warm image thumbnail cache on HOST2 throughout the week. Nightly sync +resets this cache — cold loads every morning. Weekly sync: cache stays warm 6 days, +resets Sunday night while users sleep. + +--- + +### ── sunday_morning_coffee_report ──────────────────────────────────────────── + +``` +Category: Orchestrators +Name: sunday_morning_coffee_report +Schedule: 0 7 * * 0 (Sunday 7am) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Orchestrators/sunday_morning_coffee_report.sh +``` + +**What it does:** +Full weekly system overview — everything that happened this week in one clean read. +Designed to be read over coffee after the 2:30am maintenance window has completed +and run all the Sunday morning monitor scripts. + +**Sections:** +``` +🖥️ System — uptime, memory, boot drive, cache usage, reboots this week +📀 Array — disk count, parity status, ZFS health, drive temperatures +🎬 Transcodes — weekly ramdisk peak, flip count, session split (ramdisk vs SSD) +🎵 Media Activity — arr cleanup stats (orphans removed, space freed), queue depth +🌐 Rsync — weekly transfer totals, per-share breakdown, failures +🛡️ Watchdog — system + docker watchdog state, active strikes, skip list +🔐 Security — SSL cert expiry days remaining per domain (live TLS check) +📊 Emby — version, active streams now, weekly play count, top users +⚙️ System Health — inotify peak/avg/warnings, php-fpm peak/avg, SMART, git sync +⚠️ Issues — anything requiring attention from any section above +``` + +**Delivery:** unRAID notification + Discord webhook (if configured). +**Silent when healthy:** footer shows ✅ if no issues, ⚠️ with count if something needs attention. + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 📊 MONITORS +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Monitor scripts observe and report — they never act. Read-only except for two bounded +log writers. All are safe to run manually at any time. + +--- + +### ── weekly_health_digest ──────────────────────────────────────────────────── + +``` +Category: Monitors +Name: weekly_health_digest +Schedule: 0 8 * * * (8am daily — DIGEST_PROFILE in master.conf controls notify behaviour) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh +``` + +**What it does:** +Reads state files from across the ecosystem — watchdog strikes, failover state, skip +list, bandwidth history, transcode stats, cert expiry — and compiles them into a +digest notification. Reads only, writes nothing. + +**DIGEST_PROFILE controls when it notifies (set in master.conf):** +``` +always — sends every run regardless of content (daily digest, even when healthy) +smart — sends only when something worth reporting is found + triggers: watchdog strikes, non-NORMAL failover, cert warnings, bandwidth spikes +weekly — sends once per week on DIGEST_DAY (Sunday), silent all other days +``` + +Schedule never changes (8am daily). Switching between profiles requires only one +line change in master.conf — no cron edit needed. + +--- + +### ── system_tuning_monitor ─────────────────────────────────────────────────── + +``` +Category: Monitors +Name: system_tuning_monitor +Schedule: 0 */6 * * * (every 6 hours) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Monitors/system_tuning_monitor.sh +``` + +**What it does:** +Snapshots inotify and PHP-FPM utilisation every 6 hours. Writes one bounded log +entry. The sunday_morning_coffee_report reads this log for peak, average, and +warning counts over the week. + +**inotify monitoring:** +Current instances in use vs kernel limit. Warns above INOTIFY_WARN_PCT (80%). +Shows top 5 consumers. inotify exhaustion = downloads complete but arrs don't detect +them, Live TV stutters, library updates stop — no obvious error anywhere. + +**PHP-FPM monitoring:** +Active workers vs PHP_MAX_CHILDREN. Warns above PHP_FPM_WARN_PCT (80%). +Saturation = WebGUI slows, settings saves hang, container starts from UI timeout. + +**Log:** TUNING_MONITOR_LOG — bounded, trimmed to TUNING_LOG_RETENTION days on write. + +--- + +### ── zfs_memory_snapshot ───────────────────────────────────────────────────── + +``` +Category: Monitors +Name: zfs_memory_snapshot +Schedule: 0 6 * * 0 (Sunday 6am — first Sunday script, before everything else) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh +``` + +**What it does:** +Weekly ZFS pool health and memory diagnostic. Runs before other Sunday monitors so +pool state is captured while the system is post-maintenance and settled. + +**Reports:** +ZFS pool status (ONLINE/DEGRADED/FAULTED) per pool. ARC size vs max, hit rate, metadata +ratio. Total and available RAM. Top 10 Docker containers by memory. Kernel pressure +via vmstat. Warns if ARC > ZFS_REPORT_ARC_WARN_PCT or RAM < ZFS_REPORT_FREE_WARN_GB. +Output goes to both console and /var/log/zfs-weekly-health.log for week-over-week comparison. + +--- + +### ── smart_health ──────────────────────────────────────────────────────────── + +``` +Category: Monitors +Name: smart_health +Schedule: 0 7 * * 0 (Sunday 7am) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Monitors/smart_health.sh +``` + +**What it does:** +Queries SMART attributes for every /dev/sd* and /dev/nvme* drive weekly. Drives +in HOST*_SMART_IGNORE_DRIVES (typically sda — the boot USB) are skipped. + +**What triggers a notification:** +``` +Reallocated_Sector_Ct > 0 — drive found and swapped a bad sector — warning sign +Current_Pending_Sector > 0 — drive suspects sectors bad, not yet confirmed +Offline_Uncorrectable > 0 — sectors that couldn't be corrected — data loss risk +Temperature > thresholds — reads from dynamix.cfg (same thresholds as unRAID UI) +SMART overall-health FAILED — get data off this drive immediately +``` + +**Silent when healthy:** no notification if all drives pass all checks. + +--- + +### ── cert_monitor ───────────────────────────────────────────────────────────── + +``` +Category: Monitors +Name: cert_monitor +Schedule: 0 9 * * 0 (Sunday 9am) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh +``` + +**What it does:** +Connects to each domain via live `openssl s_client` TLS connection and reads the +certificate actually being served — not NPM's API, not the certificate file. + +**Why live TLS and not an API check:** +NPM can report a cert as renewed while the web server is still serving the old +expired cert (nginx not reloaded after renewal). A live TLS check catches this. +If a user would see a certificate error, this script catches it first. + +**Thresholds (master.conf):** +``` +> CERT_WARN_DAYS (30) → silent ✅ +≤ CERT_WARN_DAYS (30) → notification — renew soon +≤ CERT_CRIT_DAYS (7) → urgent notification — renew now +connection failed → notification — cert unreachable ``` --- -### Docker_Essentials +### ── backup_verify ─────────────────────────────────────────────────────────── -| Name | Schedule | -|------|----------| -| `docker_watchdog` | `*/15 * * * *` (every 15 min) | -| `docker_daily_restart` | `0 3 * * *` (3am daily) | -| `docker_weekly_restart` | `0 3 * * 0` (Sunday 3am) | -| `docker_network_connect` | At Startup of Array | +``` +Category: Monitors +Name: backup_verify +Schedule: 0 10 * * 0 (Sunday 10am) +Background: YES +``` ```bash -# docker_watchdog -/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_watchdog.sh +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh +``` -# docker_daily_restart -/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_daily_restart.sh +**What it does:** +Randomly samples BACKUP_VERIFY_SAMPLE (default: 10) files per share, computes MD5 +checksums on both sides via SSH, and compares. Does not trust rsync's exit code — +verifies actual content matches. -# docker_weekly_restart -/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh +**Why verify independently:** +rsync exit 0 does not mean the copy is correct. Silent data corruption from faulty +hardware produces a file that exists on both sides with matching size and mtime but +different content. rsync never detects this. MD5 does. -# docker_network_connect -/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh +**Pre-flight guards:** +``` +check_connectivity() — aborts if remote Tailscale IP unreachable + prevents all files showing as MISSING from network failure +check_remote_array() — aborts if /mnt/user not mounted on remote + prevents all files showing as MISSING from array not started ``` --- -### Media +### ── emby_session_report ───────────────────────────────────────────────────── -| Name | Schedule | -|------|----------| -| `media_shares_permissions` | None — called by media_management | -| `media_cleaner_anime` | None — called by media_management | -| `media_cleaner_media` | None — called by media_management | -| `lidarr_cleanup` | None — called by media_management | -| `sonarr_cleanup` | None — called by media_management | -| `radarr_cleanup` | None — called by media_management | - -```bash -# media_shares_permissions -/mnt/user/appdata/unraid_scripts/Media/media_shares_permissions.sh - -# media_cleaner_anime -/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh anime - -# media_cleaner_media -/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh media - -# lidarr_cleanup — always --dry-run first -/mnt/user/appdata/unraid_scripts/Media/lidarr_cleanup.sh --dry-run - -# sonarr_cleanup — always --dry-run first -/mnt/user/appdata/unraid_scripts/Media/sonarr_cleanup.sh --dry-run - -# radarr_cleanup — always --dry-run first -/mnt/user/appdata/unraid_scripts/Media/radarr_cleanup.sh --dry-run +``` +Category: Monitors +Name: emby_session_report +Schedule: 0 11 * * 0 (Sunday 11am) +Background: YES ``` -> These scripts are called automatically by `media_management.sh`. Individual entries exist so you can run them manually for testing or one-off maintenance without triggering the full orchestrator. +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh +``` + +**What it does:** +Queries the Emby API for the past EMBY_REPORT_DAYS (7) days and reports streaming +activity. Each server reports on its own Emby instance via HOST*_EMBY_URL. + +**Reports:** +Server info + version. Active streams now with direct play vs transcode breakdown. +Library counts (movies, episodes, songs). Weekly play count. Top EMBY_REPORT_TOP_N +most-played items. Most active users. Transcode ratio — notifies if > 80% transcoding +(usually means a client/format compatibility issue worth fixing). --- -### Transcodes +### ── bandwidth_monitor_report ──────────────────────────────────────────────── -| Name | Schedule | -|------|----------| -| `ramdisk_setup` | At Startup of Array | -| `transcode_manager` | None — called by transcode_management | -| `transcode_cleanup` | None — called by transcode_management | - -```bash -# ramdisk_setup -/mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh - -# transcode_manager — run individually if needed -/mnt/user/appdata/unraid_scripts/Transcodes/transcode_manager.sh - -# transcode_cleanup — run individually if needed -/mnt/user/appdata/unraid_scripts/Transcodes/transcode_cleanup.sh +``` +Category: Monitors +Name: bandwidth_monitor_report +Schedule: 0 11 * * 0 (Sunday 11am — runs alongside emby_session_report) +Background: YES ``` -> ⚠️ `ramdisk_setup.sh` must run **before Emby starts** at array start. Ensure it is ordered before any Emby-related startup scripts in the User Scripts plugin. If Emby starts before the ramdisk is created it will write `transcoding-temp` to the SSD fallback path and all sessions will land on SSD until Emby restarts. +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Monitors/bandwidth_monitor.sh --report +``` + +**What it does:** +Reads the accumulated BANDWIDTH_LOG (written by rsync.sh after each sync) and +generates a weekly transfer summary. + +**The log is written automatically by rsync.sh** — you never call +`--log-transfer` manually. This `--report` entry reads what rsync already logged. + +**Reports:** +Per-profile run count, total transferred, average duration, failures. +Last 7 days timeline. Large transfer flags (> BANDWIDTH_WARN_GB). +Log format: DATE|TIME|profile|duration|status|bytes — version-stable field names +that survive rsync updates without parser changes. --- -### Tools +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔄 FAILOVER +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -| Name | Schedule | -|------|----------| -| `failover_state_reset` | None — run manually | -| `watchdog_skip_list_manager` | None — run manually | -| `bulk_permissions_repair` | None — run manually | -| `container_data_export` | None — run manually | -| `emby_database_repair` | None — run manually | -| `recreate_shares` | None — run manually | -| `zfs_pool_scrub` | None — run manually | +> **failover.sh is started by array_start.sh — not scheduled separately.** +> These entries exist for manual control and testing only. + +--- + +### ── failover_status ───────────────────────────────────────────────────────── + +``` +Category: Failover +Name: failover_status +Schedule: None — run manually +Background: NO (runs and exits — status check) +``` ```bash -# failover_state_reset — check status first -/mnt/user/appdata/unraid_scripts/Tools/failover_state_reset.sh --status +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Failover/failover.sh --status +``` -# watchdog_skip_list_manager — check status first -/mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --status +**What it does:** +Shows current failover state at a glance without affecting the running loop. +State (NORMAL/FAILOVER/NO_INTERNET/DARK). Which tier containers are active. +Outage duration if in FAILOVER. Handback strike count if remote has returned. +DDNS containers currently running. Remote server Tailscale visibility. -# bulk_permissions_repair — specify share path -/mnt/user/appdata/unraid_scripts/Tools/bulk_permissions_repair.sh /mnt/user/ShareName +**Use before any manual failover intervention.** Understanding the current state +before acting prevents making things worse. -# container_data_export — specify container and paths -/mnt/user/appdata/unraid_scripts/Tools/container_data_export.sh ContainerName /path/to/appdata /path/to/output +--- -# emby_database_repair +### ── failover_test ──────────────────────────────────────────────────────────── + +``` +Category: Failover +Name: failover_test +Schedule: None — run manually (during a planned maintenance window) +Background: YES — test takes several minutes +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh --dry-run +``` + +**What it does:** +Validates the full failover lifecycle — detection, Tier 1 container start, handback, +container stop, DNS return — using an iptables rule to simulate remote unreachability. +No real outage. Safety trap removes the iptables rule on any exit condition including +ctrl-c and crashes — remote is always reachable after the test. + +> ⚠️ **Always run with --dry-run first.** Dry run walks all 8 phases with full +> output but no iptables changes and no container starts/stops. Review timing and +> configuration before committing to a live test. + +**For a live test** (real container lifecycle, brief service interruption): +Change `--dry-run` to nothing and run during a maintenance window. + +**8 phases:** pre-flight → block remote → detect FAILOVER → verify Tier 1 started +→ restore remote → wait for handback → verify NORMAL restored → report pass/fail. + +--- + +### ── failover_state_reset ──────────────────────────────────────────────────── + +``` +Category: Failover +Name: failover_state_reset +Schedule: None — run manually (after testing or failed handback) +Background: NO +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Failover/failover_state_reset.sh --status +``` + +**What it does:** +Shows current state file then prompts for confirmation before resetting to NORMAL. +Resets state file only — does NOT start or stop any containers. + +**When to use:** +State file stuck in FAILOVER after a test that didn't complete cleanly. After killing +failover.sh directly (not via plugin Abort button). After a failed handback. + +**Verify before resetting:** +Right containers on right server. DDNS correct (`nslookup Gmer4Lfe.com`). +Both servers Tailscale connected. failover.sh not currently running. + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔄 RSYNC +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Manual trigger entries for individual rsync profiles. Used when you need to sync +one specific share outside the normal nightly window — after a bulk import, before a +planned migration, or to verify connectivity. + +> **These are NOT scheduled.** Daily and weekly orchestrators handle all scheduled syncs. +> These exist purely for on-demand manual triggers. + +**Every rsync run includes:** +Pre-flight checks (connectivity, version parity, remote daemon, remote rootfs, share exists). +Profile-matched container stop/start (if profile requires it). Bandwidth logging. +Drive temperature check — exit 1 on WARN, exit 2 on CRITICAL. + +--- + +### ── rsync_critical_data ───────────────────────────────────────────────────── + +``` +Category: Rsync +Name: rsync_critical_data +Schedule: None — run manually +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ + /mnt/user/appdata-Failover/Critical-Data +``` + +**What syncs:** auth stack — NPM proxy rules, TLS certificates, LLDAP user data, +Authelia policies and config, MariaDB database. Profile: critical-data. Stops auth +containers on both sides, syncs, restarts with dependency ordering (databases before auth). + +--- + +### ── rsync_emby_clean ───────────────────────────────────────────────────────── + +``` +Category: Rsync +Name: rsync_emby_clean +Schedule: None — run manually (also runs via weekly_sync_maintenance.sh) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ + /mnt/user/Media_Server/Emby --profile=emby +``` + +**What syncs:** full clean Emby mirror — all databases checkpointed, metadata, plugins, +config. Both Emby instances stopped during sync. Excludes transcodes, logs, cache, crash +files. This is the full-fidelity sync (vs emby-failover which is the dirty 30-minute sync). + +--- + +### ── rsync_arrs_stack ───────────────────────────────────────────────────────── + +``` +Category: Rsync +Name: rsync_arrs_stack +Schedule: None — run manually +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ + /mnt/user/appdata-Failover/Arrs_Stack +``` + +**What syncs:** all arr appdata — Sonarr, Radarr, Lidarr, Prowlarr, Bazarr, Pinchflat +databases and config. Profile: arrs_stack. Arr containers stopped on both sides during +sync to ensure clean SQLite snapshot (no mid-write database corruption). + +--- + +### ── rsync_important_data ──────────────────────────────────────────────────── + +``` +Category: Rsync +Name: rsync_important_data +Schedule: None — run manually +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ + /mnt/user/appdata-Failover/Important-Data +``` + +**What syncs:** NextCloud, Postgres-NextCloud, Immich, PostgreSQL-Immich appdata. +Profile: important-data. Postgres containers stop first, then NextCloud after delay. + +--- + +### ── rsync_gmer4lfe ─────────────────────────────────────────────────────────── + +``` +Category: Rsync +Name: rsync_gmer4lfe +Schedule: None — run manually +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ + /mnt/user/appdata-Failover/Gmer4Lfe +``` + +**What syncs:** HOST1 server appdata backup — everything in the Gmer4Lfe appdata share. +No profile container stop required. Incremental — safe to run at any time. + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🐳 DOCKER_ESSENTIALS +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +> **docker_watchdog.sh is started by array_start.sh — not scheduled separately.** +> docker_daily_restart.sh is called by daily_sync_maintenance.sh. +> docker_weekly_restart.sh is called by weekly_sync_maintenance.sh. +> docker_network_connect.sh is called by array_start.sh. +> These entries exist for manual control only. + +--- + +### ── docker_watchdog_status ────────────────────────────────────────────────── + +``` +Category: Docker_Essentials +Name: docker_watchdog_status +Schedule: None — run manually +Background: NO +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_watchdog.sh --status +``` + +**What it shows:** +Strike counts for all monitored containers. Current skip list with running/stopped state. +Whether startup grace period is active and how long remains. Whether RAM emergency +deferral is active (mem_shutdown_active from system_watchdog). Last cycle timing. + +--- + +### ── watchdog_skip_list_manager ────────────────────────────────────────────── + +``` +Category: Docker_Essentials +Name: watchdog_skip_list_manager +Schedule: None — run manually (after fixing a skip-listed container) +Background: NO +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Docker_Essentials/watchdog_skip_list_manager.sh --status +``` + +**What it does:** +Views and manages the persistent container skip list. When docker_watchdog.sh restarts +the same container 3 times in 1 hour it adds it to the skip list and stops trying. +A critical notification is sent. This tool clears the list after you've fixed the cause. + +**Recovery workflow:** +``` +1. Run --status to see what's skip-listed and its restart history +2. Fix the underlying problem (check docker logs, disk space, database) +3. --clear ContainerName to remove from skip list + clear restart history +4. docker start ContainerName to confirm your fix works +5. Watchdog resumes monitoring on next cycle automatically +``` + +**Actions:** +``` +--status — show skip list, running/stopped state, restart counts +--clear ContainerName — clear specific container (prompts YES) +--clear ContainerName --force — no prompt +--clear-all — clear everything +--dry-run — preview any clear action +``` + +--- + +### ── docker_daily_restart_manual ───────────────────────────────────────────── + +``` +Category: Docker_Essentials +Name: docker_daily_restart_manual +Schedule: None — run manually (called automatically by daily_sync_maintenance.sh) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_daily_restart.sh --dry-run +``` + +**What it does:** +Restarts containers in HOST*_DAILY_RESTART_CONTAINERS. Running → docker restart +(graceful stop + start). Stopped → left stopped (state respected). Missing → skip. +Dependency ordering: databases restart before applications. CONTAINER_DELAY wait +between dependency and dependent restarts. + +**Default configured containers:** NginxProxyManager, Authelia, Dispatcharr, +Dispatcharr-Basic, ErsatzTV-Emby — services that degrade with uptime. + +> Remove `--dry-run` to run live. Default is dry-run for safety when run manually. + +--- + +### ── docker_network_connect_manual ─────────────────────────────────────────── + +``` +Category: Docker_Essentials +Name: docker_network_connect_manual +Schedule: None — run manually (runs automatically via array_start.sh) +Background: NO +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh --status +``` + +**What it does:** +Ensures custom Docker networks exist and containers are connected to them. Idempotent +— safe to run any time. If a network is missing (typically wiped by a unRAID update), +it creates it and notifies. + +**Use after unRAID updates** if containers can't reach each other — the update may have +wiped custom networks. Run this to recreate them without restarting the array. + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🎬 MEDIA +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +> **All media scripts are called by daily_sync_maintenance.sh via media_management.sh.** +> These entries exist for manual testing and targeted one-off runs. +> Run --dry-run --log for any new system or after configuration changes. + +--- + +### ── media_shares_permissions_manual ──────────────────────────────────────── + +``` +Category: Media +Name: media_shares_permissions_manual +Schedule: None — run manually (runs automatically via daily_sync_maintenance.sh) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Media/media_shares_permissions.sh --dry-run +``` + +**What it does:** +Applies `nobody:users` ownership and 755 (dirs) / 664 (files) permissions to every +file in HOST*_MEDIA_PERMISSION_SHARES. Two separate passes — directories and files +get different modes. Idempotent — only changes what's wrong. + +**When ownership drifts:** scp/cp from another machine leaves root:root. New container +before PUID/PGID is set. rsync without --chown. This fixes it all in one pass. + +> Remove `--dry-run` to run live. Dry run shows count of files that would be corrected. + +--- + +### ── media_cleaner_anime_manual ────────────────────────────────────────────── + +``` +Category: Media +Name: media_cleaner_anime_manual +Schedule: None — run manually (runs automatically via daily_sync_maintenance.sh) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh anime --dry-run +``` + +**What it does:** +Removes junk files from anime share folders using ANIME_FILE_PATTERNS. Patterns +include: *.sfv *.nfo *.rar *.sample* *.torrent *.url *.log and other scene release +debris. Runs before arr cleanup so orphan detection only sees actual media files. + +--- + +### ── media_cleaner_media_manual ────────────────────────────────────────────── + +``` +Category: Media +Name: media_cleaner_media_manual +Schedule: None — run manually +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh media --dry-run +``` + +**What it does:** +Same as above but for media share folders using MEDIA_FILE_PATTERNS — all anime +patterns plus *.iso and *.lrc. + +--- + +### ── lidarr_cleanup_manual ─────────────────────────────────────────────────── + +``` +Category: Media +Name: lidarr_cleanup_manual +Schedule: None — run manually (runs automatically via daily_sync_maintenance.sh) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Media/lidarr_cleanup.sh --dry-run --log +``` + +**What it does:** +Queries Lidarr API for all tracked file paths. Walks /mnt/user/Music. Files on disk +not in the API response that are older than LIDARR_ORPHAN_AGE (7 days) and not +currently open by any process = orphans = deleted. + +**7 safety checks must ALL pass before any deletion:** +Lidarr running + healthy. API reachable. API version matches. Artist count > 0. +Tracked file count > 0. Count >= 80% of last known (detects database issues). +Total deletion size < LIDARR_MAX_DELETE_GB (50GB) — stops catastrophic path mismatches. + +**Protected always:** *.jpg *.png (cover art), *.nfo (metadata), *.lrc (lyrics) +**After deletion:** notify_emby_scan() triggers "Clean Missing Files" in Emby immediately. + +> Remove `--dry-run` only after reviewing the full log output. See README-Media.md +> for the complete safe testing procedure. + +--- + +### ── sonarr_cleanup_manual ─────────────────────────────────────────────────── + +``` +Category: Media +Name: sonarr_cleanup_manual +Schedule: None — run manually +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Media/sonarr_cleanup.sh --dry-run --log +``` + +**What it does:** +Same logic as lidarr_cleanup but for TV files. Walks HOST*_SONARR_TV_ROOT. +Protected: *.jpg *.png (artwork), *.nfo, *.srt *.sub *.ass *.ssa (Bazarr subtitles). +Host-aware: HOST1 manages Tv_Shows, HOST2 manages Anime_Shows — correct root via detect_hosts(). + +--- + +### ── radarr_cleanup_manual ─────────────────────────────────────────────────── + +``` +Category: Media +Name: radarr_cleanup_manual +Schedule: None — run manually +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Media/radarr_cleanup.sh --dry-run --log +``` + +**What it does:** +Same logic for movie files. Walks HOST*_RADARR_MOVIES_ROOT. +Protected: *.jpg *.png (movie artwork), *.nfo, subtitles. +Host-aware: HOST1 manages Movies, HOST2 manages Anime_Movies. + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🎬 TRANSCODES +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +> **ramdisk_setup.sh is called by array_start.sh — not scheduled separately.** +> **transcode_manager.sh and transcode_cleanup.sh are called by transcode_management.sh.** +> These entries exist for manual status checks and emergency use. + +--- + +### ── ramdisk_status ──────────────────────────────────────────────────────────── + +``` +Category: Transcodes +Name: ramdisk_status +Schedule: None — run manually +Background: NO +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh --status +``` + +**What it shows:** +Whether ramdisk is mounted at RAMDISK_PATH. Current symlink target (ramdisk or SSD). +Whether transcoding-temp exists on ramdisk. SSD fallback directory state. +Current usage on both ramdisk and SSD. + +--- + +### ── transcode_manager_manual ──────────────────────────────────────────────── + +``` +Category: Transcodes +Name: transcode_manager_manual +Schedule: None — run manually (runs via transcode_management.sh every 3 min) +Background: NO +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Transcodes/transcode_manager.sh --status +``` + +**What it shows:** +Current mode (smart/ramdisk/ssd). Ramdisk usage vs thresholds. Current symlink target. +Active sessions per configured TRANSCODE_SERVERS. Daily log stats (peak this week, +flip count, session split ramdisk vs SSD). + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🖥️ UNRAID_ESSENTIALS +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +> **system_watchdog.sh is started by array_start.sh — not scheduled separately.** +> **webgui_restart.sh, system_tuning_monitor.sh, clear_logs.sh are scheduled below.** +> **inotify_tuning.sh, php_fpm_max_children.sh, docker_syslog_filter.sh run via array_start.sh.** + +--- + +### ── webgui_restart ─────────────────────────────────────────────────────────── + +``` +Category: unRAID_Essentials +Name: webgui_restart +Schedule: */10 * * * * (every 10 minutes) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/unRAID_Essentials/webgui_restart.sh +``` + +**What it does:** +Checks WebGUI responsiveness and recovers it automatically. Silent when healthy — +runs 144 times per day with no output when the WebGUI is responding normally. + +**Escalation path (each step only runs if previous failed):** +``` +1. curl http://localhost → unresponsive for WEBGUI_TIMEOUT (5s)? +2. Restart nginx → wait 15s → recheck +3. Restart php-fpm → wait 10s → recheck (handles worker saturation) +4. Restart emhttp (core daemon) → wait 30s → recheck (most disruptive, last resort) +5. All failed → notify warning — manual investigation needed +``` + +A notification fires on any restart so you know what recovered and when. + +--- + +### ── clear_logs ─────────────────────────────────────────────────────────────── + +``` +Category: unRAID_Essentials +Name: clear_logs +Schedule: 0 5 * * 0 (Sunday 5am — before Sunday maintenance window) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/unRAID_Essentials/clear_logs.sh +``` + +**What it does:** +Clears system and Docker container logs using size thresholds — only logs large +enough to be worth clearing. + +**System logs** (/var/log/syslog, messages, dmesg): cleared only if > LOG_MIN_SIZE_MB (10MB). +Smaller logs contain useful recent history — not cleared. + +**Docker container logs**: cleared per-container only if > LOG_DOCKER_MAX_MB (100MB). +Active containers like Emby and SABnzbd grow fastest. Truncation not rotation — +`> file` keeps the file descriptor valid while emptying. Safe for running services. + +--- + +### ── system_watchdog_status ────────────────────────────────────────────────── + +``` +Category: unRAID_Essentials +Name: system_watchdog_status +Schedule: None — run manually +Background: NO +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/unRAID_Essentials/system_watchdog.sh --status +``` + +**What it shows:** +All three tiers with current thresholds. Active strike counts per check. RAM tier +status. Whether mem_shutdown_active is set (RAM emergency in progress). Recent +watchdog-triggered reboots. All 18 per-host toggle states. + +--- + +### ── mover_stop ─────────────────────────────────────────────────────────────── + +``` +Category: unRAID_Essentials +Name: mover_stop +Schedule: None — run manually (before array operations needing mover stopped) +Background: NO +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/unRAID_Essentials/mover_stop.sh +``` + +**What it does:** +Stops the unRAID mover cleanly. Wall message warning to all logged-in users. Waits +MOVER_STOP_TIMEOUT seconds. SIGTERM (allows mover to finish current file — no partial +files). Waits + verifies. SIGKILL if still running (warns that partial files are possible). + +--- + +### ── rsync_stop ──────────────────────────────────────────────────────────────── + +``` +Category: unRAID_Essentials +Name: rsync_stop +Schedule: None — run manually (when rsync needs stopping) +Background: NO +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/unRAID_Essentials/rsync_stop.sh --status +``` + +**What it does:** +Stops rsync intelligently. Auto-detects if an orchestrator is running: +Smart mode (default): kills rsync subprocess only — orchestrator sees rsync died and +exits cleanly. No orphaned lock files. Containers recover via docker_watchdog.sh. +`--full-stop`: kills orchestrator first then rsync — use for emergency stop. + +--- + +### ── user_scripts_stop ──────────────────────────────────────────────────────── + +``` +Category: unRAID_Essentials +Name: user_scripts_stop +Schedule: None — run manually (before reboots, or when a script hangs) +Background: NO +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/unRAID_Essentials/user_scripts_stop.sh --status +``` + +**What it does:** +Finds all running User Script processes by /tmp/user.scripts path signature. Shows +script names (not just PIDs — you know what you're stopping). SIGTERM → verify → +SIGKILL. Self-exclusion prevents the script from killing itself. Called automatically +by server_reboot.sh before reboot. + +--- + +### ── server_reboot ──────────────────────────────────────────────────────────── + +``` +Category: unRAID_Essentials +Name: server_reboot +Schedule: None — run manually (planned maintenance reboots) +Background: YES — sequence takes 1-2 minutes +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/unRAID_Essentials/server_reboot.sh --dry-run +``` + +**What it does:** +Graceful reboot with full pre-flight and clean shutdown. Wall message + unRAID +notification with MY_ID. Waits REBOOT_SLEEP (30s). virsh shutdown each running VM +then waits REBOOT_VM_WAIT for graceful exit. Stops Docker. Stops libvirt. sync. +/sbin/reboot. + +Pre-flight warnings (not blocks): rsync running, mover running, active Emby sessions. +These show in the summary — you chose to reboot, these are context. + +> Remove `--dry-run` to execute. Dry run walks the full sequence without rebooting — +> use it to verify what processes are active before committing. + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🤝 PARTNERSHIP +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Manages the two-server auth-sharing relationship lifecycle. Automated check runs +inside critical_sync_maintenance.sh. These entries are for manual operations only. + +--- + +### ── partnership_status ──────────────────────────────────────────────────────── + +``` +Category: Partnership +Name: partnership_status +Schedule: None — run manually +Background: NO +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --status +``` + +**What it shows:** +Both server state files (ACTIVE/INACTIVE). Offline counter (days since last successful +sync). Which server is the auth stack owner. Current WebUI redirect state on HOST2. +Whether CRITICAL_RSYNC_ENABLED is true. + +--- + +### ── partnership_onboard ────────────────────────────────────────────────────── + +``` +Category: Partnership +Name: partnership_onboard +Schedule: None — run manually (HOST1/owner only, once at relationship start) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --onboard --dry-run +``` + +**What it does:** +Establishes the auth-sharing partnership. Reconfigures HOST2 Docker template WebUI +URLs to redirect to HOST1 via Tailscale. Verifies routing works. Writes ACTIVE state +on both servers. Enables Critical-Data sync. + +> Remove `--dry-run` to run live. Dry run shows exactly which templates would be +> changed and what URLs they'd become — review before committing. + +--- + +### ── partnership_offboard ───────────────────────────────────────────────────── + +``` +Category: Partnership +Name: partnership_offboard +Schedule: None — run manually (either server, when ending the partnership) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --offboard --dry-run +``` + +**What it does:** +Clean partnership separation. Stops rsync. Final sync to HOST2. Reconfigures HOST2 +WebUI URLs back to localhost (HOST2 manages its own auth). Disables critical sync. +Grace period (PARTNERSHIP_GRACE_HOURS). Removes HOST2 from Tailscale tailnet. + +Both servers can initiate. If HOST1 is unreachable when HOST2 offboards, deferred +finalisation completes automatically when HOST1 can next reach HOST2. + +> Remove `--dry-run` to run live. 10-second countdown shown before any changes. + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔧 TOOLS +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Situational utilities — run when something needs fixing, not on a schedule. + +--- + +### ── bulk_permissions_repair ───────────────────────────────────────────────── + +``` +Category: Tools +Name: bulk_permissions_repair +Schedule: None — run manually +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Tools/bulk_permissions_repair.sh /mnt/user/Movies --dry-run +``` + +**What it does:** +Applies nobody:users 755/664 to a specific path immediately — without waiting for +the nightly media_shares_permissions.sh run. Use when an admin copy, scp, or bad +container left root-owned files that are blocking arr operations right now. +Replace /mnt/user/Movies with the actual path that needs fixing. + +--- + +### ── container_data_export ──────────────────────────────────────────────────── + +``` +Category: Tools +Name: container_data_export +Schedule: None — run manually (before risky updates or migrations) +Background: YES +``` + +```bash +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Tools/container_data_export.sh \ + Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/ +``` + +**What it does:** +Stops the container cleanly. Archives the entire appdata directory to a timestamped +.tar.gz. Verifies archive integrity. Restarts the container (always, even if archiving +failed). Space check before stopping — won't start if output drive is too small. + +Replace ContainerName, /path/to/appdata, and /output/dir for each use. +Output filename: ContainerName_YYYY-MM-DD_HH-MM.tar.gz + +--- + +### ── emby_database_repair ───────────────────────────────────────────────────── + +``` +Category: Tools +Name: emby_database_repair +Schedule: None — run manually (after power loss or repeated Emby crashes) +Background: YES +``` + +```bash +#!/bin/bash /mnt/user/appdata/unraid_scripts/Tools/emby_database_repair.sh +``` -# recreate_shares -/mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh +**What it does:** +Stops Emby. Runs SQLite `PRAGMA integrity_check` on every Emby database. Restarts +Emby. Reports per-database with specific recovery guidance. -# zfs_pool_scrub +**Per-database recovery guidance:** +``` +library.db — CORRUPT → safe to delete, Emby fully rebuilds from media files +users.db — CORRUPT → deleting resets ALL user accounts + watch history + check for recent backup (weekly_sync_maintenance.sh) +authentication.db — CORRUPT → safe to delete, API keys regenerated on restart +activity.db — CORRUPT → safe to delete, it's a log only +``` + +**This script checks and reports only** — it does not automatically delete anything. +Recovery is a human decision based on whether you have backups. + +--- + +### ── zfs_pool_scrub ──────────────────────────────────────────────────────────── + +``` +Category: Tools +Name: zfs_pool_scrub +Schedule: None — run manually (monthly or after any disk/power event) +Background: YES +``` + +```bash +#!/bin/bash /mnt/user/appdata/unraid_scripts/Tools/zfs_pool_scrub.sh ``` +**What it does:** +Triggers ZFS scrub on all pools (except ZFS_REPORT_IGNORE_POOLS), waits for completion, +notifies with results. ZFS checksums every block and verifies — catches silent data +corruption before you discover it on reads. Safe to run while system is active. +For a specific pool: add the pool name as argument (bypasses ignore list). + --- -### unRAID_Essentials +### ── recreate_shares ────────────────────────────────────────────────────────── -| Name | Schedule | -|------|----------| -| `system_watchdog` | `*/15 * * * *` (every 15 min) | -| `webgui_restart` | `*/10 * * * *` (every 10 min) | -| `docker_syslog_filter` | At Startup of Array | -| `php_fpm_max_children` | At Startup of Array | -| `clear_logs` | `0 5 * * 0` (Sunday 5am) | -| `mover_stop` | None — run manually | -| `rsync_stop` | None — run manually | -| `server_reboot` | None — run manually | -| `user_scripts_stop` | None — run manually | +``` +Category: Tools +Name: recreate_shares +Schedule: None — run manually (after fresh install or disk replacement) +Background: YES +``` ```bash -# system_watchdog -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/system_watchdog.sh - -# webgui_restart -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/webgui_restart.sh - -# docker_syslog_filter -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/docker_syslog_filter.sh - -# php_fpm_max_children -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/php_fpm_max_children.sh - -# clear_logs -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/clear_logs.sh - -# mover_stop -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/mover_stop.sh - -# rsync_stop -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/rsync_stop.sh - -# server_reboot -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/server_reboot.sh - -# user_scripts_stop -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/user_scripts_stop.sh +#!/bin/bash +/mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh --dry-run ``` +**What it does:** +Reads every .cfg in /boot/config/shares/, parses the shareInclude disk list, and +creates /mnt/diskN/sharename on each included disk. Run after restoring share .cfg +files to a fresh install — shares appear in the UI but directories are missing until +this creates them. chown nobody:users on creation. + --- -### Git +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔀 GIT +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -| Name | Schedule | -|------|----------| -| `git_pull_execute` | None — run manually after pushing changes | +--- + +### ── git_pull ───────────────────────────────────────────────────────────────── + +``` +Category: Git +Name: git_pull +Schedule: None — run manually (called automatically by daily_sync_maintenance.sh) +Background: NO +``` ```bash -# git_pull_execute -/mnt/user/appdata/unraid_scripts/git_pull_execute.sh +#!/bin/bash +/mnt/user/appdata/unraid_scripts/unRAID_Essentials/git_pull_execute.sh ``` -This is the deployment mechanism for the entire ecosystem. After pushing changes to Gitea from VS Code — run this on both servers. Both pull the latest scripts and set executable permissions automatically. +**What it does:** +Pulls latest commits from Gitea. Sets execute permission on all .sh files. This is +the deployment mechanism for the ecosystem — push changes from VS Code to Gitea, +then run this on both servers to deploy. daily_sync_maintenance.sh runs it +automatically at 1am as the first job in the window. --- -## At Startup of Array — Order Matters +## ━━━ AT STARTUP OF ARRAY — ORDER ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Scripts that run at array start must be ordered correctly. In User Scripts Enhanced you can drag to reorder. The required order: +**Only one entry needed at startup:** `array_start`. It handles everything. + +`array_start.sh` is the single "At Startup of Array" entry. It reads ARRAY_START_SCRIPTS +from master.conf and launches each in order. No other startup entries. No dragging to +reorder individual scripts. Change the order by editing master.conf. ``` -1. ramdisk_setup — ramdisk must exist before Emby starts -2. docker_syslog_filter — suppress veth noise before containers start -3. php_fpm_max_children — WebGUI tuning before anyone accesses it -4. docker_network_connect — connect containers after Docker is ready -5. failover — background task, starts monitoring loop +Correct startup setup: + 1 entry: array_start — At Startup of Array, Background: YES + +Wrong startup setup (old approach): + ramdisk_setup — At Startup of Array + docker_syslog_filter — At Startup of Array + system_watchdog — At Startup of Array + docker_watchdog — At Startup of Array + failover — At Startup of Array + (unpredictable order, no coordination) ``` --- -## After Rebuilding +## ━━━ COMPLETE SCHEDULE REFERENCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Once all entries are created and scheduled, verify by running each script once manually from the plugin UI before relying on the scheduled runs. Use `--dry-run` for anything destructive before running live. - -The scheduled scripts that matter most to verify first: ``` -transcode_management — runs cleanup then manager, check output is clean -docker_watchdog — check all required containers show running -system_watchdog — check no threshold alerts on first run -daily_sync — run manually, verify remote connectivity +At Startup of Array: + array_start BG: YES — single entry, handles everything + +*/3 * * * * (every 3 min): + transcode_management BG: YES — cleanup then manager + +*/10 * * * * (every 10 min): + webgui_restart BG: YES — silent when healthy + +*/15 * * * * (every 15 min): + critical_sync_maintenance BG: YES — auth stack + Emby dirty + partnership check + +*/30 * * * * (every 30 min): + rsync_emby_failover BG: YES — Emby watch state sync + +0 */6 * * * (every 6 hours): + arrs_failed_stalled_recovery BG: YES — blocklist + re-search failed imports + system_tuning_monitor BG: YES — inotify + php-fpm snapshot + +0 1 * * * (1am daily): + daily_sync_maintenance BG: YES — full nightly window + +0 5 * * 0 (Sunday 5am): + clear_logs BG: YES — size-threshold log cleanup + +0 6 * * 0 (Sunday 6am): + zfs_memory_snapshot BG: YES — ZFS + memory report + +0 7 * * 0 (Sunday 7am): + sunday_morning_coffee_report BG: YES — full weekly digest + +0 8 * * * (8am daily): + weekly_health_digest BG: YES — profile-controlled notification + +0 7 * * 0 (Sunday 7am): + smart_health BG: YES — SMART attribute check + +0 9 * * 0 (Sunday 9am): + cert_monitor BG: YES — live TLS cert expiry check + +0 10 * * 0 (Sunday 10am): + backup_verify BG: YES — MD5 checksum mirror verification + +0 11 * * 0 (Sunday 11am): + emby_session_report BG: YES — Emby weekly usage stats + bandwidth_monitor_report BG: YES — rsync transfer summary + +30 2 * * 0 (Sunday 2:30am): + weekly_sync_maintenance BG: YES — clean sync + image updates +``` + +--- + +## ━━━ AFTER CREATING ALL ENTRIES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Run each of these once manually before relying on scheduled runs: + +``` +array_start --dry-run verify all startup scripts found and sequenced correctly +transcode_management --status confirm ramdisk mounted and symlink pointing at ramdisk +critical_sync_maintenance verify Critical-Data sync completes to HOST2 +daily_sync_maintenance --dry-run verify remote connectivity and share list +failover_status confirm NORMAL state on both servers +docker_watchdog_status confirm required containers all running +system_watchdog_status confirm no threshold alerts on first check ``` \ No newline at end of file diff --git a/README.md b/README.md index 0d8fcc4..b51cf77 100644 --- a/README.md +++ b/README.md @@ -1,709 +1,740 @@ -# unRAID Script Ecosystem +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# 🏠 UNRAID SCRIPT ECOSYSTEM +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -A modular, git-managed automation ecosystem for two unRAID servers. One configuration file. Both servers stay in sync with a single git pull. Everything from daily media syncing to mutual container failover runs automatically — and when something goes wrong, the system tries to fix itself before waking you up. +**A complete self-healing, self-maintaining, mutually-redundant two-server unRAID +ecosystem.** One codebase runs on both servers. Both servers run their own lives +independently. When one goes down the other covers it — not because of conventional +failover infrastructure, but because both servers already have copies of everything +that matters and both know how to run it. + +> **This is not conventional failover.** There is no primary and standby. There is no +> shared storage. There is no cluster manager. There are two fully independent unRAID +> servers, living in different locations on different power utilities, each running +> their own household's media stack — and a layer of scripts that keeps them +> synchronised, coordinated, and mutually covering. The goal is minimal disruption to +> a media stack, not enterprise high availability. --- -## What This Is - -Two friends. Hardware they already own. A mutual agreement to help each other. - -This ecosystem delivers practical high availability and offsite backup between two unRAID servers at zero additional hardware cost. It is not enterprise-grade clustering. It does not promise zero disruption. What it delivers is **minimal disruption** — and for a homelab, that's genuinely good enough. - -**The alternative:** -``` -Proxmox cluster — minimum 3 nodes -Ceph storage — dedicated hardware -Enterprise networking — specialised equipment -Offsite infrastructure — at least 3 servers minimum -Cost — thousands in hardware, ongoing licensing -``` - -**This ecosystem:** -``` -Two friends who already run unRAID -Hardware they already own -Mutual agreement — each covers the other -Cost: $0 beyond what they were already running -``` - -**The honest user experience during failover:** -``` -Library transcode users — buffer absorbs the cutover, usually seamless -Library direct play — may notice, needs to resume -Live TV direct play — no buffer, notices every time, has to resume -Watch history — worst case 15-30min window behind - reality: a show or two, movies likely unaffected - -Could force transcode on Live TV to get the buffer benefit — -not worth the permanent CPU/RAM overhead for an event that rarely happens. -Live TV disruption is the known, accepted tradeoff. -``` - -**The real world math:** - -How often does a homelab server actually go down? How many of those times are during peak viewing? How many viewers notice vs just resume? Is the occasional minor annoyance worth thousands in enterprise hardware? - -For most people — absolutely not. This ecosystem exists for those people. - -**Minimal disruption, not no disruption. Free, mutual, and genuinely useful.** - ---- - -## The Goal - -A self-hosted infrastructure that runs itself. - -``` -Step away → come back to a healthy system -Something breaks → system self-heals -Something can't be fixed → you get notified -Servers stay in sync → one edit propagates everywhere -``` - -Before this ecosystem existed, the same problems were solved by 60+ standalone scripts across two servers — different coding styles, no shared standards, every change applied twice. This ecosystem standardises everything into one codebase with one config file and one deployment mechanism. - ---- - -## The Servers +## ━━━ THE TWO SERVERS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ``` HOST1 — unRAID-Gmer4Lfe - Hardware: Threadripper 1950X, 128GB RAM - Storage: Multiple ZFS pools + cache - Domain: Gmer4Lfe.com - Runs: Full media stack, auth, live TV, arrs, downloaders - Source of truth: Movies, Tv_Shows, Music - Tdarr — runs anime transcoding (stronger hardware) + Hardware: Threadripper 1950X, 128GB RAM, ZFS cache pools + Location: Primary site + Domain: Gmer4Lfe.com + Owns: Movies, Tv_Shows, Music — source of truth + Auth stack — source of truth + Emby — primary media server -HOST2 — unRAID-Jayred365 (Buddy server) - Hardware: Intel i5 10th gen — completely different hardware - Storage: Different disk count, different pool layout - RAM: 64GB - Location: Remote — 50 miles from HOST1 - Domain: Gmer4Lfe.us (his own domain, his own services) - Runs: His own independent Emby, his own containers, his own users - Source of truth: Anime_Shows, Anime_Movies - His arrs manage anime — syncs to HOST1 as mirror - -Network: Tailscale — encrypted tunnel between both servers -Repo: Self-hosted Gitea on HOST1 -Deployment: git pull on either server → both stay current +HOST2 — unRAID-Jayred365 + Hardware: Intel i5 10th gen, 64GB RAM + Location: Remote — different building, different power utility + Domain: Gmer4Lfe.us + Owns: Anime_Shows, Anime_Movies — source of truth + His own Emby — for his household ``` -**This is a genuine mutual failover agreement between two friends, each running a fully independent server.** Both servers are fully active in normal operation — each with their own service stack, their own domain, their own users. HOST2 is not a standby. He runs his own Emby, his own containers, and his own domain while sharing access to the mirrored media library. - -``` -Normal operation: - HOST1 — Gmer4Lfe.com — full stack, live TV, arrs, everything - Source of truth for all config, DNS, auth, certs - HOST2 — Gmer4Lfe.us — his Emby, his containers, his domain, his users - Mirror of HOST1's auth stack — changes on HOST1 propagate here -``` - -**The shared production Emby:** - -All users — from both servers — connect to one Emby instance running on HOST1. HOST1 has the stronger hardware and better bandwidth. Both domains route through NPM to the same Emby. This is intentional — one library, one watch history, one set of users. - -``` -Gmer4Lfe.com → NPM → Emby (my users) -Gmer4Lfe.us → NPM → Emby (his users) -Both point at the same container on HOST1 -``` - -**Auth stack — runs on both servers simultaneously:** - -NPM, LLDAP, and Authelia run actively on both servers at all times. HOST2 needs them running to serve his users through his domain in normal operation. HOST1 is source of truth — all changes made there, mirrored to HOST2 every 15 minutes. One username and password works across all services on both domains. Group-based access controls what each user can see. - -``` -Change made on HOST1 LLDAP → mirrored to HOST2 → works on both domains -Cert renewed on HOST1 → mirrored to HOST2 → valid on both domains -NPM rule added on HOST1 → mirrored to HOST2 → routes on both servers -``` - -**Why the auth stack runs on both — a year of testing:** - -Early versions started the auth stack from cold on failover. Testing showed 30-60 seconds of broken authentication after DNS cut over — Emby clients hit HOST2 before LLDAP, Authelia, and NPM were ready. For library transcode users with buffer this was invisible. For Live TV and direct play users this meant the stream died and reconnect failed, forcing a manual exit and resume. - -Running the auth stack warm on both servers at all times eliminates this window entirely. When failover triggers, the auth stack is already running and ready. What actually starts from stopped: - -``` -Tier 1 — immediate: - Emby ← starts from stopped on covering server - DDNS updater ← remote domain updater starts on covering server - - Everything else verified healthy — NOT cold started: - NPM, LLDAP, Authelia, certs — already running, already warm on both servers - -Tier 4 — 18hr+ outage: - Arrs_Stack ← starts from stopped on covering server - has mirrored Arrs_Stack appdata — starts in known good state -``` - -**When HOST1 goes down:** -``` -HOST2 detects HOST1 unreachable + internet up -→ Starts HOST1's DDNS updater — Gmer4Lfe.com now points at HOST2 -→ Starts Emby — only real Tier 1 container starting from stopped -→ Auth stack already running on HOST2 — users authenticate immediately -→ DNS TTL 1 minute — cutover fast -→ Library transcode users — buffer absorbs, usually seamless -→ Live TV / direct play users — will notice, need to resume -→ Watch history — worst case 15-30min behind, reality a show or two - movies may be unaffected given their length -→ Tiers 2/3/4 escalate if outage extends -→ HOST2's own services keep running unaffected throughout -``` - -**When HOST2 goes down:** -``` -HOST1 detects HOST2 unreachable + internet up -→ Starts HOST2's DDNS updater — Gmer4Lfe.us now points at HOST1 -→ Starts his Emby — his users served from HOST1 -→ His auth stack already running on HOST1 — serving his domain continuously -→ Tiers 2/3 escalate if outage extends -→ Tier 4 (18hr+): his Arrs_Stack starts from stopped on HOST1 - mirrored appdata — starts in known good state -→ HOST1's own services keep running unaffected throughout -``` - -**Handback — waits for stability:** -``` -Remote returns → FAILOVER_HANDBACK_STRIKES consecutive stable checks -→ Prevents handing back during a brief network blip -→ Stops remote DDNS first — no split brain during transition -→ Rsync writeback — userdata, watch states, auth changes during outage -→ Starts containers on correct server -→ Starts DDNS last — only after containers confirmed healthy -→ Returns to NORMAL -``` - -**The hardware doesn't need to match.** `/mnt/user/` abstracts everything. A share called `Movies` is `/mnt/user/Movies` on both servers regardless of what drives or pools back it. rsync syncs the content. Container mounts use the same path. The hardware underneath is irrelevant. - -**Media library configurations — multiple valid setups:** - -The ecosystem supports several library arrangements. The right choice depends on how the two servers and their users relate to each other. All configurations use the same scripts — `HOST1_DAILY_SYNC_SHARES` and `HOST2_DAILY_SYNC_SHARES` in `Master.conf` define who owns what. +**Hardware does not need to match.** Everything is accessed through `/mnt/user/` — +unRAID's unified share layer. HOST1 has a Threadripper with ZFS. HOST2 has a completely +different CPU and fewer drives. A failover container on HOST2 mounts `/mnt/user/Movies` +and sees mirrored data because the share names match. The hardware underneath is +irrelevant. --- -**Configuration A — Single source, one owner (simplest):** +## ━━━ CONTAINER NAMING — YOU NEED TO KNOW THIS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -One server owns all media. The other mirrors everything and piggybacks. - -``` -HOST1 owns everything: - Movies, Tv_Shows, Music, Anime — all managed by HOST1's arrs - HOST2 mirrors all shares — read only in normal operation - HOST2 arrs disabled or not running - -HOST1_DAILY_SYNC_SHARES=(Movies Tv_Shows Music Anime_Shows Anime_Movies ...) -HOST2_DAILY_SYNC_SHARES=() ← nothing — HOST2 owns nothing - -Best for: - One person running both servers - Secondary server is pure failover + piggybacking - Simpler arr management — one set of arrs, one library -``` +This is the most important convention in the ecosystem. Get it wrong and the scripts +break silently. --- -**Configuration B — Split source of truth (this setup):** - -Each server owns specific shares — managed by their own arrs. The other mirrors those shares read only. - -``` -HOST1 source of truth: HOST2 source of truth: - Movies ← Radarr Anime_Shows ← his Sonarr - Tv_Shows ← Sonarr Anime_Movies ← his Radarr - Music ← Lidarr - -HOST1 mirrors ← HOST2 anime HOST2 mirrors ← HOST1 movies/shows/music -``` - -Both servers run arrs simultaneously — no conflict because they manage completely different shares. Scheduling keeps them further separated: - -``` -HOST2 arrs: midnight → noon downloading and managing anime -HOST1 Tdarr: 12:30 → 23:00 transcoding, syncs as source of truth -``` - -**The rule is not "don't run arrs on both servers" — it's:** -**never run two instances of the same arr against the same share simultaneously.** - -Each Emby instance on both servers can access all media — movies, shows, anime — because everything is mirrored in both directions. HOST1 mirrors anime from HOST2. HOST2 mirrors movies and shows from HOST1. - -``` -HOST1 /mnt/user/Movies → rsync → HOST2 /mnt/user/Movies -HOST2 /mnt/user/Anime_Shows → rsync → HOST1 /mnt/user/Anime_Shows -``` - -At Tier 4 failover (18hr+ outage) each server's arr copies spin up to cover the other's shares — but only when the truth holder has genuinely been down long enough to need it. - ---- - -**Configuration C — Weighted split:** - -One server owns most shares, the other owns a smaller subset. Same mechanics as B — just an unequal split based on what each person actually manages. - -``` -HOST1 owns: Movies, Tv_Shows, Music, Kids_Movies, Stand-up, Sports -HOST2 owns: Anime_Shows, Anime_Movies - -HOST1_DAILY_SYNC_SHARES=(Movies Tv_Shows Music Kids_Movies ...) -HOST2_DAILY_SYNC_SHARES=(Anime_Shows Anime_Movies) -``` - -Most real-world split setups will land here — one person has a larger collection, the other a specific niche. - ---- - -**Configuration D — Fully shared, no split:** - -Both servers have access to all media but neither "owns" it in a managed sense. No arrs running on either for the shared shares. Content added manually or by one designated manager. - -``` -HOST1_DAILY_SYNC_SHARES=(Movies Tv_Shows Music ...) -HOST2_DAILY_SYNC_SHARES=() - -One-way push — HOST1 is the single truth holder -HOST2 is a read-only mirror for all media -Simpler than split — no arr coordination needed -``` - ---- - -**What all configurations have in common:** - -``` -Same share names on both servers ← /mnt/user/Movies everywhere -Same container names on both servers ← Emby, NginxProxyManager everywhere -HOST*_DAILY_SYNC_SHARES defines truth ← who pushes what, automatically -daily_sync.sh reads the correct list ← direction determined at runtime -Failover Tier 4 uses opposing list ← writeback always correct direction -``` - -The scripts don't need to know which configuration you're running. The arrays in `Master.conf` define the truth holders — everything else follows automatically. - -**Container naming convention:** - -``` -Server-specific containers — always on their own server, unique names: - Emby-Gmer4Lfe ← HOST1's personal Emby, always on HOST1 - Emby-Jayred365 ← HOST2's personal Emby, always on HOST2 - VaultWarden-Gmer4Lfe ← HOST1's password manager - VaultWarden-Jayred365 ← HOST2's password manager - -Shared failover containers — same name on both servers: - Emby ← HOST1's main Emby, starts on HOST2 during HOST1 outage - NginxProxyManager ← same name, same proxy rules, same networks - Authelia ← same name -``` - -**Docker custom networks** — containers communicate by name within custom networks, not by IP. When a container restarts and gets a new IP, NPM still reaches it by name. Failover containers join the same named networks on either server — no proxy rule changes, no IP reconfiguration, no downtime from IP drift. - -**Personal shares — backup without failover:** - -Beyond the shared media library, each user can sync personal shares to the other server purely for offsite backup — no failover container involvement, just data protection. - -``` -HOST1: /mnt/user/Gmer4Lfe-Personal → rsync nightly → HOST2 (encrypted backup) -HOST2: /mnt/user/Jayred365-Personal → rsync nightly → HOST1 (encrypted backup) -``` - -Configure in `Master.conf`: -```bash -HOST1_PERSONAL_SHARES=( - "/mnt/user/Gmer4Lfe-Personal" -) -HOST2_PERSONAL_SHARES=( - "/mnt/user/Jayred365-Personal" -) -``` - -`daily_sync.sh` automatically picks up the personal shares for the local host and syncs them alongside that host's truth shares. - -**Encrypting personal shares — unRAID 7 ZFS:** - -ZFS native encryption in unRAID 7 means both admins can see the share exists and file sizes but neither can read content without your passphrase or keyfile. rsync copies encrypted blocks as-is — the remote server never needs your key. - -**Setup on HOST1 (your personal share):** - -**Step 1 — Create an encrypted ZFS dataset:** -``` -unRAID UI → Main tab → click your ZFS pool -→ Click "+ Dataset" -→ Name: Gmer4Lfe-Personal -→ Enable Encryption: Yes -→ Encryption type: passphrase (simplest) or keyfile (auto-unlock capable) -→ Enter your passphrase — write it down, if lost data is unrecoverable -→ Create -``` - -**Step 2 — Create the share:** -``` -Settings → Shares → Add Share -→ Name: Gmer4Lfe-Personal -→ Primary storage: your ZFS pool -→ Use cache: Only (keeps data on ZFS pool, not array) -→ Add -``` - -**Step 3 — Verify encryption is active:** -```bash -zfs get encryption poolname/Gmer4Lfe-Personal -# Should show: encryption aes-256-gcm -``` - -**Step 4 — Add to Master.conf and sync:** -```bash -HOST1_PERSONAL_SHARES=( - "/mnt/user/Gmer4Lfe-Personal" -) -``` - -**Auto-unlock on boot (keyfile approach):** - -If you want the share to mount automatically after reboot without entering a passphrase: -```bash -# Create keyfile — on HOST1 only, never sync this file -mkdir -p /root/.zfs-keys -dd if=/dev/urandom bs=32 count=1 | base64 > /root/.zfs-keys/personal.key -chmod 400 /root/.zfs-keys/personal.key - -# Set dataset to use keyfile -zfs change-key -o keylocation=file:///root/.zfs-keys/personal.key \ - -o keyformat=raw poolname/Gmer4Lfe-Personal - -# Add to array start script (unRAID_Essentials or User Scripts) -zfs load-key poolname/Gmer4Lfe-Personal -zfs mount poolname/Gmer4Lfe-Personal -``` - -**Manual unlock (most secure — you control when it's readable):** -```bash -zfs load-key poolname/Gmer4Lfe-Personal # prompts for passphrase -zfs mount poolname/Gmer4Lfe-Personal -``` - -**What the remote admin sees:** - -The share directory exists on HOST2. File names and sizes are visible (ZFS encrypts content, not metadata by default). File contents are unreadable without your key. To hide filenames too, enable `zfs set encryption=aes-256-gcm` with `dnodesize=auto` — this is a more advanced setup. - -**Current status:** The infrastructure supports encrypted personal share syncing. The ZFS dataset setup is a one-time manual step per server. Once set up it syncs automatically like any other share. - ---- - -## How It Works - -Every script sources two files at startup: +### ── Shared Containers — No Suffix ─────────────────────────────────────────── ```bash -source "$SCRIPT_DIR/../Master.conf" # all user configuration -source "$SCRIPT_DIR/../common.sh" # shared library +# ───────────────────────────────────────────────────────────────────────────── +# Shared containers — meaning both servers run this container and scripts treat +# them as the same logical service — use the plain name with no suffix. +# +# The convention: "no suffix = this container participates in the shared stack" +# + Emby # the shared Emby — runs on HOST1, picked up by HOST2 on failover + NginxProxyManager # shared NPM — runs on both, same config via sync + Lldap-Gmer4Lfe # shared LLDAP — same user directory, both servers + Authelia # shared auth — both servers, same config + Mariadb-Authelia # Authelia's database — same on both + Redis-Authelia # Authelia's cache — same on both + VaultWarden # shared password manager + Dispatcharr # shared Live TV scheduler ``` -**`Master.conf`** is the single source of truth. Container names, thresholds, paths, API keys, rsync profiles, failover tiers — everything configurable lives here. Change a value, push to git, both servers pull — done. - -**`common.sh`** provides shared functions used by every script — host detection, notifications, rsync helpers, output formatting, icon set. Scripts never duplicate this logic. - -**`git_pull_execute.sh`** pulls the latest scripts from Gitea and sets executable permissions. Schedule it or run it manually on either server. +When `failover.sh` on HOST2 reads `FAILOVER_HOST2_RUNS_FOR_HOST1_TIER1=(..."Emby"...)`, +it looks for a container called exactly `Emby` and starts it. HOST2 must have a +container called `Emby` — not `Emby-Gmer4Lfe`, not `Emby-HOST1`, just `Emby` — with +its volume mounts pointing at the mirrored share paths. This is what makes one +codebase work on both servers without modification. --- -## Repository Structure +### ── User-Specific Containers — Suffixed ──────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# User-specific containers — things that belong to one person and only ever +# run on their server — use a suffix identifying the user/owner. +# + Emby-Gmer4Lfe # Gmer4Lfe's personal Emby instance (HOST1, never covered) + Emby-Jayred365 # Jayred365's personal Emby instance (HOST2, never covered) + Organizer-Gmer4Lfe # Gmer4Lfe's dashboard — personal, stays on HOST1 + Sonarr-Gmer4Lfe # Gmer4Lfe's Sonarr — managing his Tv_Shows share + Radarr-Gmer4Lfe # Gmer4Lfe's Radarr — managing his Movies share + Immich-Gmer4Lfe # Gmer4Lfe's photo library — personal +``` + +User-specific containers are covered during failover (see Tiered Failover below) but +are distinct from the shared stack. They are never confused with each other because +the suffix makes the owner unambiguous. Scripts targeting `Emby` and scripts targeting +`Emby-Gmer4Lfe` are targeting different containers for different purposes. + +--- + +### ── Why This Matters ───────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# The entire ecosystem uses container names as identifiers. +# docker_watchdog.sh: WATCHDOG_REQUIRED_CONTAINERS=("NginxProxyManager" "Authelia") +# → looks for exactly "NginxProxyManager" on the running server +# failover.sh: FAILOVER_HOST2_RUNS_FOR_HOST1_TIER1=("Emby" "VaultWarden") +# → looks for exactly "Emby" and "VaultWarden" on HOST2 +# docker_daily_restart.sh: DAILY_RESTART_CONTAINERS=("Dispatcharr" "Authelia") +# → looks for exactly those names in docker ps +# +# A container named differently than expected is not found — no error, no restart, +# no failover coverage. It silently does not happen. +# Keep names consistent on both servers for shared containers. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +## ━━━ SHARE NAMING — ALSO YOU NEED TO KNOW THIS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Shares must have **identical names on both servers**. The ecosystem uses share paths +as sync targets and mount points — a mismatch is a broken path. + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# The rule: if a share exists on HOST1 and HOST2 needs a mirror of it, +# the share on HOST2 must have the exact same name. +# +# Same name on both servers: + /mnt/user/Movies # HOST1 source of truth, HOST2 mirror + /mnt/user/Tv_Shows # HOST1 source of truth, HOST2 mirror + /mnt/user/Music # HOST1 source of truth, HOST2 mirror + /mnt/user/Anime_Shows # HOST2 source of truth, HOST1 mirror + /mnt/user/Anime_Movies # HOST2 source of truth, HOST1 mirror +# +# rsync.sh syncs /mnt/user/Movies on HOST1 → /mnt/user/Movies on HOST2. +# If HOST2 has it at /mnt/user/Movies-Mirror → rsync aborts: path not found. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +## ━━━ THE MEDIA LIBRARY MODEL ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +### ── Common Libraries — Shared Pool ───────────────────────────────────────── + +Common media like Movies, Music, and TV Shows are a **shared pool**. Every container +that needs music — Emby, LidaTube, Navidrome, whatever — mounts the same `/mnt/user/Music` +share. There is one share, one location, one source of truth. No duplication of content +across separate shares per-user or per-application. + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# One share for the library — all containers that need it mount the same path. +# Emby container: /mnt/user/Music → /media/Music (read) +# Lidarr: /mnt/user/Music → /music (read+write) +# LidaTube: /mnt/user/Music → /downloads (write) +# Navidrome: /mnt/user/Music → /music (read) +# +# All pointing at the same /mnt/user/Music. No per-user music copies. +# When Lidarr downloads and Emby scans — they're looking at the same files. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Never Two Arrs Against the Same Share Simultaneously ─────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# This rule is absolute. No exceptions. +# +# Correct: HOST1 Sonarr manages /mnt/user/Tv_Shows +# HOST2 Sonarr manages /mnt/user/Anime_Shows +# Different shares — no conflict +# +# Correct: HOST1 Tdarr transcodes from /mnt/user/Anime_Shows +# HOST2 Sonarr downloads into /mnt/user/Anime_Shows +# But: they run in non-overlapping time windows +# +# WRONG: HOST1 Sonarr AND HOST2 Sonarr both managing /mnt/user/Tv_Shows +# Two arrs managing the same library = rename conflicts, double imports, +# deletion battles, corrupted databases. +# +# This is why source of truth is split: + HOST1 owns: Movies (Radarr), Tv_Shows (Sonarr), Music (Lidarr) + HOST2 owns: Anime_Movies (Radarr), Anime_Shows (Sonarr) +# +# When HOST2 enters failover and starts HOST1's arr containers — those arrs +# point at /mnt/user/Movies etc. HOST2's own arrs point at /mnt/user/Anime_*. +# Different shares, no conflict. This is by design. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Source of Truth ────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Every share has exactly one server that owns it. +# That server's arr manages it. That server pushes it to the other via daily sync. +# The receiving server treats it as read-only. +# +# Rule: never add content to a mirrored share on the non-owning server. +# It will be overwritten on the next sync. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +## ━━━ WHAT SYNCS AND WHAT DOESN'T ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Not everything needs to sync. Not everything should sync. Understanding what goes +where determines whether you need encryption and whether sync failure matters. + +--- + +### ── Shared Failover Shares — Sync Required ───────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# These shares enable failover — HOST2 must have current copies for the covered +# services to work correctly when they start on HOST2. +# + /mnt/user/Media_Server/Emby # Emby userdata — watch history, playstates + /mnt/user/appdata-Failover/Critical-Data # auth stack — NPM rules, certs, user accounts + /mnt/user/Movies # media library content + /mnt/user/Tv_Shows # media library content + /mnt/user/Music # media library content +# +# Sync failure here = failover runs but with stale data (acceptable) or missing +# data (problematic). These sync every night via daily_sync_maintenance.sh. +# Critical-Data and Emby also sync every 15 minutes via critical_sync_maintenance.sh. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Non-Failover Shares — Sync as Backup, No Encryption Required ─────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Some shares don't participate in failover — their container only runs on one +# server and is covered during failover from scratch (or not at all). +# These shares can sync as offsite backup without encryption. +# Content is not sensitive — anyone with access to HOST2 could read it. +# +# Examples: + /mnt/user/appdata-Failover/Gmer4Lfe # Gmer4Lfe server appdata backup + /mnt/user/appdata-Failover/Important-Data # NextCloud, Immich appdata + /mnt/user/Kids_Movies # kids content — not sensitive + /mnt/user/Sports # sports content — not sensitive +# +# These sync in DAILY_SYNC_SHARES — straight rsync, no special handling. +# If HOST2 is compromised, the content is readable. That is acceptable. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Personal Shares — Always Encrypted ───────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Personal shares contain content only the owner should see — documents, +# photos, financial records, private files. +# ZFS encrypts at the dataset level — HOST2 receives encrypted blocks. +# Without your passphrase or keyfile, HOST2 cannot read the content. +# + HOST1_PERSONAL_SHARES=( + "/mnt/user/Gmer4Lfe-Personal" # encrypted ZFS dataset + ) + HOST2_PERSONAL_SHARES=( + "/mnt/user/Jayred365-Personal" # encrypted ZFS dataset + ) +# +# Append to HOST*_DAILY_SYNC_SHARES — synced nightly with everything else. +# Same rsync, same timing — but the content on HOST2 is ciphertext only you can unlock. +# +# Setup: README-Rsync_Setup.md → Step 8 — Personal Encrypted Shares +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +## ━━━ THIS IS NOT CONVENTIONAL FAILOVER ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +> **Understanding this distinction prevents a lot of confusion.** + +Conventional failover (Pacemaker, DRBD, Proxmox HA) operates on the premise of a +primary server and a standby server with shared storage or synchronous replication. +The standby exists only to take over when the primary fails. It runs nothing +independently. Its purpose is 100% failover coverage with no service interruption. + +**This is not that.** + +--- + +### ── What This Actually Is ───────────────────────────────────────────────────── + +``` +Two fully independent servers, each running their own household's services. +Each server has its own domain, its own users, its own media library management. +Each server is the primary for its own services every single day. + +The "failover" component is additive: + When HOST1 goes down → HOST2 starts HOST1's containers on top of its own + When HOST2 comes back → HOST1's containers hand back cleanly + HOST2 never stopped running its own services during this entire time + +This is not standby. This is mutual coverage between peers. +``` + +--- + +### ── The Actual Goal: Minimal Disruption to a Media Stack ──────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# "Minimal disruption" means: +# Live TV keeps playing → Dispatcharr is Tier 1 (starts immediately) +# Passwords are accessible → VaultWarden is Tier 1 (people lock themselves out) +# Emby keeps serving → Emby is Tier 1 +# Auth works → NPM, Authelia, LLDAP already warm on both servers +# +# "Minimal disruption" does NOT mean: +# Zero downtime (DDNS TTL = 1 minute, brief gap expected) +# Perfect consistency (Emby userdata is 15min stale at most) +# All services covered immediately (tiered start over 24 hours) +# +# The system is designed around the real experience of a household: +# "The TV stopped working" → unacceptable, fixed in Tier 1 (< 2 minutes) +# "NextCloud is down" → annoying, fixed in Tier 2 (4 hours) +# "Gitea is down" → inconvenient, fixed in Tier 3 (12 hours) +# "Sonarr isn't downloading" → fine, fixed in Tier 4 (24 hours) +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── What You Give Up With This Approach ───────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Things conventional failover provides that this does NOT: +# Zero RPO (recovery point objective) — there is ~15min of potential staleness +# Synchronous state replication — changes don't reach HOST2 instantly +# Automatic failure detection in seconds — failover.sh checks every 2 minutes +# Transparent IP migration — DNS TTL means up to 1 minute of routing to old IP +# +# What you get instead: +# Both servers run fully independently — no single point of failure in software +# Geographic redundancy — different buildings, different power, different ISP +# Both households stay operational during most disruption scenarios +# Simple, auditable scripts — no cluster manager black box +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +## ━━━ HOW FAILOVER ACTUALLY WORKS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +`failover.sh` runs on both servers as a continuous background process. Every 2 minutes +it pings two things: the other server's Tailscale IP, and `8.8.8.8`. The combination +of those two answers determines the current state. + +``` +Remote ping PASS + Internet PASS → NORMAL — both servers running independently + +Remote ping FAIL + Internet PASS → FAILOVER — remote is down, start covering it + Tier 1 immediately: DDNS + Emby + Auth + VaultWarden + Live TV + Tier 2 at 4 hours: NextCloud + Immich + Jellyseerr + Tier 3 at 12 hours: AdGuard + Gitea + Uptime monitoring + Tier 4 at 24 hours: Arrs + download clients + +Internet FAIL (either) → NO_INTERNET — stop own DDNS, wait + Can't serve anyone without internet — stop claiming DNS ownership + +Remote FAIL + Internet FAIL → DARK — conservative, same as NO_INTERNET +``` + +The DDNS sequencing is critical — one domain, one DDNS container, at all times. The +handback sequence stops remote DDNS first, rsyncs, starts containers, starts local +DDNS last. Split-brain DNS is prevented by design. + +**→ Full detail: [README-Failover.md](Failover/README-Failover.md)** + +--- + +## ━━━ THE AUTH STACK — WARM ON BOTH SERVERS ALWAYS ━━━━━━━━━━━━━━━━━━━━━━━━━━ + +NPM, LLDAP, Authelia, MariaDB, Redis — all running actively on both servers at all +times. Not cold standby. Not started at failover. Running right now, serving each +server's own domain. + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Why this eliminates the failover dead zone: +# +# Cold standby approach: HOST1 goes down → DNS switches to HOST2 → HOST2 starts +# auth containers → 30-60 seconds before auth is ready → users hit the auth wall +# before the service behind it responds → failed logins, broken sessions +# +# Warm approach: HOST1 goes down → DNS switches to HOST2 → auth is ALREADY +# running → users are served immediately with no dead zone +# +# HOST1 is source of truth for the auth configuration. All changes to NPM proxy +# rules, LLDAP users, Authelia policies, and certs happen on HOST1 and propagate +# to HOST2 via Critical-Data sync every 15 minutes. +# +# HOST2 operator's experience: click NginxProxyManager in Docker UI → +# browser opens HOST1's NPM via Tailscale redirect → make changes there → +# 15 minutes later live on both servers. Zero auth management overhead on HOST2. +# ───────────────────────────────────────────────────────────────────────────── +``` + +**→ Full detail: [README-Partnership.md](Partnership/README-Partnership.md)** + +--- + +## ━━━ THE COMPLETE PICTURE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +``` + HOST1 (Gmer4Lfe.com) HOST2 (Gmer4Lfe.us) + ────────────────── ────────────────────── + +Always running: + Own stack Full service stack Full service stack (anime focus) + Auth NPM, Authelia, LLDAP NPM, Authelia, LLDAP (mirrored) + +Source of truth: + Media Movies, Tv_Shows, Music ←── Anime_Movies, Anime_Shows + ───────────────────────→ (bidirectional daily push) + Config auth stack config ──→ mirrored every 15 min + Emby state watch history, playstates──→ dirty sync every 30 min + +Failover coverage: + If HOST1 down: — starts Emby, VaultWarden, + Dispatcharr, all arr stack + serves Gmer4Lfe.com via DDNS + + If HOST2 down: starts his Emby, his arrs, — + serves Gmer4Lfe.us via DDNS + +Monitoring: + Both servers system_watchdog.sh system_watchdog.sh + docker_watchdog.sh docker_watchdog.sh + failover.sh failover.sh + Sunday morning coffee report Sunday morning coffee report +``` + +--- + +## ━━━ FOLDER STRUCTURE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ``` Unraid_Scripts/ -├── Master.conf # All user configuration — edit here only -├── common.sh # Shared library — used by all scripts -├── git_pull_execute.sh # Pull latest scripts from Gitea -├── User_Script_Template.sh # Paste into unRAID User Scripts plugin │ -├── Failover/ # Mutual container failover -│ ├── README-Failover.md -│ ├── failover.sh # State machine — runs continuously -│ └── failover_test.sh # Controlled simulation harness +├── master.conf ← All shared configuration — the only file you edit regularly +├── master_host1.conf ← HOST1-specific: share lists, container names, API keys +├── master_host2.conf ← HOST2-specific: same structure, different values +├── common.sh ← Shared library — all functions used by every script +├── load_config.sh ← Sources all conf files and common.sh │ -├── Monitors/ # Health reporting — watch and report only -│ ├── README-Monitors.md -│ ├── cert_monitor.sh # SSL certificate expiry -│ ├── smart_health.sh # Drive SMART attributes -│ ├── zfs_memory_snapshot.sh # ZFS pool health + memory report -│ ├── backup_verify.sh # Random checksum verification vs remote -│ ├── bandwidth_monitor.sh # Rsync transfer history -│ ├── weekly_health_digest.sh # Aggregated system health summary -│ └── emby_session_report.sh # Emby usage statistics +├── Orchestrators/ ← Scheduled job runners — what User Scripts plugin executes +│ README: README-Orchestrators.md │ -├── Orchestrators/ # Sequential job runners -│ ├── README-Orchestrators.md -│ ├── daily_sync.sh # All media shares synced nightly -│ ├── media_management.sh # Permissions + cleaners + arr cleanup -│ └── transcode_management.sh # Cleanup then manager every 3min + daily stats +├── Rsync/ ← Core rsync script — called per share with profile +│ README: README-Rsync_Setup.md │ -├── Rsync/ # Core sync engine -│ ├── README-Rsync.md -│ ├── rsync.sh # Per-share or per-profile sync -│ └── README_Rsync_Setup.md # Initial setup guide +├── Failover/ ← Mutual automatic failover — continuous background process +│ README: README-Failover.md │ -├── Docker_Essentials/ # Container lifecycle management -│ ├── README-Docker_Essentials.md -│ ├── docker_watchdog.sh # Two-tier self-healing monitoring -│ ├── docker_daily_restart.sh # Daily container restarts -│ ├── docker_weekly_restart.sh # Weekly container restarts -│ └── docker_network_connect.sh # Extra network connections at boot +├── Docker_Essentials/ ← Container lifecycle: watchdog, restarts, networks +│ README: README-Docker_Essentials.md │ -├── Media/ # Media library maintenance -│ ├── README-Media.md -│ ├── media_shares_permissions.sh # Recursive permission application -│ ├── media_cleaner.sh # Junk file removal (anime + media profiles) -│ ├── lidarr_cleanup.sh # Orphaned music file cleanup -│ ├── sonarr_cleanup.sh # Orphaned TV file cleanup -│ └── radarr_cleanup.sh # Orphaned movie file cleanup +├── unRAID_Essentials/ ← Server-level: system watchdog, WebGUI, log hygiene, tuning +│ README: README-Unraid_Essentials.md │ -├── Transcodes/ # Emby ramdisk transcode management -│ ├── README-Transcoding.md -│ ├── ramdisk_setup.sh # Create ramdisk + symlink at array start -│ ├── transcode_manager.sh # Monitor usage, manage symlink, display sessions -│ └── transcode_cleanup.sh # Remove old inactive transcode files +├── Media/ ← Library health: permissions, junk cleanup, orphan removal +│ README: README-Media.md │ -├── Tools/ # Situational utilities — run when needed -│ ├── README-Tools.md -│ ├── recreate_shares.sh # Recreate share dirs after incident -│ ├── failover_state_reset.sh # Reset failover state file to NORMAL -│ ├── watchdog_skip_list_manager.sh # Manage container watchdog skip lists -│ ├── bulk_permissions_repair.sh # Targeted permission repair for one share -│ ├── container_data_export.sh # Export container appdata to tar archive -│ ├── emby_database_repair.sh # SQLite integrity check on Emby databases -│ └── zfs_pool_scrub.sh # Trigger ZFS scrub with completion report +├── Transcodes/ ← Ramdisk transcode management with SSD fallback +│ README: README-Transcoding.md │ -└── unRAID_Essentials/ # Server-level system management - ├── README-Unraid_Essentials.md - ├── system_watchdog.sh # Last line of defense — controlled reboot - ├── webgui_restart.sh # WebGUI nginx + emhttp auto-restart - ├── docker_syslog_filter.sh # Suppress Docker veth noise from syslog - ├── php_fpm_max_children.sh # WebGUI PHP-FPM concurrency tuning - ├── clear_logs.sh # Weekly system log clearing - ├── mover_stop.sh # Graceful mover termination - ├── rsync_stop.sh # Stop rsync + recover containers - ├── server_reboot.sh # Graceful reboot with user warning - └── user_scripts_stop.sh # Stop all running User Scripts jobs +├── Monitors/ ← Read-only observation: certs, SMART, bandwidth, digests +│ README: README-Monitors.md +│ +├── Partnership/ ← Two-server relationship lifecycle: onboard, offboard, transfer +│ README: README-Partnership.md +│ +└── Tools/ ← Situational utilities: repair, export, emergency tools + README: README-Tools.md ``` --- -## Folders at a Glance - -| Folder | What it does | Key scripts | -|--------|-------------|-------------| -| **Failover** | Mutual container failover — autonomous, tiered, DDNS-safe | `failover.sh` | -| **Monitors** | Watch and report — never act, minimal flash writes | `weekly_health_digest.sh` | -| **Orchestrators** | Sequential job runners with unified reporting | `daily_sync.sh`, `media_management.sh` | -| **Rsync** | Core sync engine with profile system | `rsync.sh` | -| **Docker_Essentials** | Two-tier self-healing container management | `docker_watchdog.sh` | -| **Media** | Permissions, junk cleanup, arr orphan cleanup | `media_cleaner.sh`, arr scripts | -| **Transcodes** | Ramdisk symlink routing for Emby | `transcode_manager.sh` | -| **Tools** | Recovery and situational utilities | `failover_state_reset.sh`, `zfs_pool_scrub.sh` | -| **unRAID_Essentials** | Server-level maintenance and last-resort recovery | `system_watchdog.sh` | - ---- - -## Key Design Principles - -**One config file.** `Master.conf` is the only file you edit. No hunting through scripts to change a container name or a threshold. - -**Bidirectional.** Both servers run identical scripts. `detect_hosts()` determines local vs remote at runtime. One codebase covers both directions. - -**Exact naming consistency across both servers — non-negotiable.** - -This is the most important operational principle and the one most likely to cause problems if ignored. The ecosystem is built on the assumption that containers and shares have identical names on both servers. - -``` -Container names: - Emby ← HOST1 and HOST2 - Mariadb-Authelia ← HOST1 and HOST2 - Lldap-Gmer4Lfe ← HOST1 and HOST2 - NginxProxyManager ← HOST1 and HOST2 - -Share names: - /mnt/user/Movies ← HOST1 and HOST2 - /mnt/user/Tv_Shows ← HOST1 and HOST2 - /mnt/user/Anime_Shows ← HOST1 and HOST2 -``` - -Why this matters: - -``` -rsync.sh: pushes /mnt/user/Movies → remote /mnt/user/Movies - same path, both sides — no mapping, no translation - -Container stops: - PROFILE_CRITICAL_CONTAINER_NAMES stops "Emby" locally - then stops "Emby" on remote - one list, both servers, no duplication - -failover.sh: starts "Emby" on covering server - same name it always has everywhere - -detect_hosts(): compares hostname against HOST1/HOST2 in Master.conf - exact string match — hostnames must match exactly -``` - -Years of running with custom per-user share paths (`/mnt/user/me/movies`, `/mnt/user/jayred/movies`) taught the hard lesson — every difference becomes a maintenance burden. Every script needs to know about it. Every rsync needs a custom path. Every container stop list diverges. - -Shared media shares should be shared — identical paths, identical names, no ownership prefix. User-specific data belongs in personal encrypted shares that are explicitly separate from the shared ecosystem. That boundary keeps the common infrastructure simple and the private data private. - -If a container or share has a different name on one server — the script either skips it gracefully or fails to find it. It will not error catastrophically, but it also will not do what you expect. Consistent naming is what makes one codebase work on both servers without modification. - -**Self-healing layers.** Problems are addressed at the most targeted level first: -``` -docker_watchdog.sh — container level, minimal disruption -system_watchdog.sh — system level, last resort -failover.sh — infrastructure level, other server covers -``` - -**Strike systems, not hair triggers.** Single spikes don't cause restarts or reboots. Sustained problems do. The strike system filters noise from genuine issues. - -**Notifications when action is needed, silence otherwise.** The ecosystem is designed to run without daily attention. You hear from it when something needs human intervention — not as a regular occurrence. - -**Flash drive friendly.** Scripts that write to `/boot/` use bounded files with auto-purge. `/tmp/` is used for ephemeral state that resets on reboot. Most scripts write nothing at all. - -**--dry-run everywhere.** Every script supports `--dry-run`. Test before you schedule. - -**Hardware utilization without script changes.** Any unRAID system can participate regardless of hardware. The stronger server runs the heavier jobs — not because the scripts force it, but because app scheduling and rsync timing make it happen naturally. No script changes needed to reconfigure who does what: - -``` -HOST1 — Threadripper, 128GB - Tdarr transcoding: 12:30 → 23:00 ← heavy job on stronger hardware - -HOST2 — i5, 64GB - His arrs: midnight → noon ← lighter load, scheduled around Tdarr - Benefits from HOST1's transcoding - without the CPU/RAM cost -``` - -Change a schedule in an app. Adjust an rsync timing. The ecosystem adapts. The scripts never need to know which server is doing what — they just do their jobs on whatever server they're running on. This makes the ecosystem applicable to any two unRAID servers in any configuration — not just this specific hardware pairing. - -**Mutually beneficial, not one-way.** Both servers contribute what their hardware does best. Both benefit from what the other provides. The arrangement works because it's genuinely useful to both sides — not because one server is just a passive backup for the other. - ---- - -## Scheduling Overview +## ━━━ CONFIGURATION — THREE FILES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ```bash -# ━━━ At Startup of Array ━━━ -failover.sh # background task — continuous loop -ramdisk_setup.sh -docker_syslog_filter.sh -php_fpm_max_children.sh -docker_network_connect.sh - -# ━━━ Frequent ━━━ -*/3 * * * * transcode_management.sh # replaces separate manager + cleanup entries -*/10 * * * * webgui_restart.sh -*/15 * * * * docker_watchdog.sh -*/15 * * * * system_watchdog.sh - -# ━━━ Daily ━━━ -0 1 * * * daily_sync.sh -0 2 * * * media_management.sh -0 3 * * * docker_daily_restart.sh -0 8 * * * weekly_health_digest.sh - -# ━━━ Weekly — Sunday ━━━ -0 3 * * 0 docker_weekly_restart.sh -0 5 * * 0 clear_logs.sh -0 6 * * 0 zfs_memory_snapshot.sh -0 7 * * 0 smart_health.sh -0 9 * * 0 cert_monitor.sh -0 10 * * 0 backup_verify.sh -0 11 * * 0 emby_session_report.sh -0 11 * * 0 bandwidth_monitor.sh --report +# ───────────────────────────────────────────────────────────────────────────── +# master.conf — shared across both servers +# Everything that applies equally to both: thresholds, schedules, profile +# definitions, watchdog settings, arr cleanup config, DDNS timing, etc. +# Pushed to both servers via git. Never contains server-specific values. +# +# master_host1.conf — sourced only on HOST1 +# HOST1_* prefixed variables: share lists, container names, API keys, +# ramdisk size, specific paths, per-server toggle overrides. +# +# master_host2.conf — sourced only on HOST2 +# HOST2_* prefixed variables: same structure, different values. +# +# detect_hosts() in common.sh: +# Reads the current server's hostname. Matches it against HOST1 and HOST2. +# Aliases HOST1_DAILY_SYNC_SHARES → DAILY_SYNC_SHARES (if on HOST1) +# Aliases HOST2_DAILY_SYNC_SHARES → DAILY_SYNC_SHARES (if on HOST2) +# Every script uses the unprefixed name. The same script on either server +# automatically uses the correct values. No hostname comparisons in scripts. +# ───────────────────────────────────────────────────────────────────────────── ``` --- -## Folder READMEs +## ━━━ WHAT RUNS WHEN ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Each folder has a detailed README covering setup, configuration, usage, and the reasoning behind design decisions. +The User Scripts plugin runs only orchestrators — never individual scripts directly. -| README | Contents | -|--------|----------| -| [README-Failover.md](Failover/README-Failover.md) | DDNS split brain prevention, tiered failover, handback sequence, initial setup, troubleshooting | -| [README-Monitors.md](Monitors/README-Monitors.md) | All monitor scripts, flash drive write policy, scheduling | -| [README-Orchestrators.md](Orchestrators/README-Orchestrators.md) | Why orchestrators exist, job ordering, how to add jobs | -| [README-Rsync.md](Rsync/README-Rsync.md) | Profile system, SSH key setup, Tailscale requirements | -| [README-Docker_Essentials.md](Docker_Essentials/README-Docker_Essentials.md) | Two-tier watchdog, startup grace, dependency ordering, skip list management | -| [README-Media.md](Media/README-Media.md) | Execution order, cleaner profiles, arr cleanup safety procedure | -| [README-Transcoding.md](Transcodes/README-Transcoding.md) | Symlink indirection design, Docker mount warning, mode switching, sizing | -| [README-Tools.md](Tools/README-Tools.md) | All utility scripts, when to use each, how to add new tools | -| [README-Unraid_Essentials.md](unRAID_Essentials/README-Unraid_Essentials.md) | system_watchdog tiers, startup sequence, scheduled maintenance | +``` +At Startup of Array: + array_start.sh launches all startup scripts in order + → inotify_tuning.sh raise kernel limits before containers start + → docker_syslog_filter.sh suppress veth noise before containers create interfaces + → php_fpm_max_children.sh WebGUI tuning before first request + → ramdisk_setup.sh create ramdisk before Emby starts + → docker_network_connect.sh connect containers to extra networks + → system_watchdog.sh continuous — last resort server stability + → docker_watchdog.sh continuous — two-tier container healing + → failover.sh continuous — mutual failover state machine + +Every 3 minutes: + transcode_management.sh cleanup → manager (order non-negotiable) + +Every 15 minutes: + critical_sync_maintenance.sh Critical-Data + Emby dirty sync + partnership check + +Every 30 minutes: + rsync.sh --profile=emby-failover Emby watch state sync (separate cron entry) + +Every 6 hours: + arrs_failed_stalled_recovery.sh blocklist + re-search failed imports + +1am daily: + daily_sync_maintenance.sh git pull → sync shares → media maintenance → restarts + +2:30am Sunday: + weekly_sync_maintenance.sh stop → update images → clean sync → start → weekly restarts + +8am daily: + weekly_health_digest.sh DIGEST_PROFILE controls when it actually notifies + +Every 6 hours: + system_tuning_monitor.sh inotify + php-fpm utilisation snapshot + +Sunday morning block (6–11am): + zfs_memory_snapshot.sh, smart_health.sh, cert_monitor.sh, + backup_verify.sh, emby_session_report.sh, bandwidth_monitor.sh --report +``` + +**→ Complete schedule: [README-Orchestrators.md](Orchestrators/README-Orchestrators.md)** --- -## v2 Roadmap +## ━━━ THE DAILY CYCLE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -The v1 ecosystem is production-proven and stable. These are the features planned for v2 — some require additional infrastructure, some are extensions of existing systems. +What actually happens on a typical day, from the ecosystem's perspective: -### Transcode Manager — Advanced Mode +``` +Throughout the day: + system_watchdog.sh watching: RAM, CPU temp, rootfs, kernel, daemon + docker_watchdog.sh watching: memory, CPU, HTTP health, required containers + failover.sh watching: remote server, internet connectivity + transcode_management.sh managing: ramdisk ↔ SSD, session cleanup (every 3min) + critical_sync_maintenance.sh keeping: auth stack + Emby current (every 15min) -The current `smart` / `ramdisk` / `ssd` modes route all sessions to one location. v2 adds an `advanced` mode that routes by media type: +1am: + git pull (latest scripts first) + rsync Movies, Tv_Shows, Music → HOST2 (HOST1 pushes its truth shares) + rsync Anime_Shows, Anime_Movies → HOST1 (HOST2 pushes its truth shares) + media_shares_permissions.sh (fix ownership before arr cleanup) + media_cleaner.sh anime + media (remove scene junk) + lidarr/sonarr/radarr_cleanup.sh (remove orphaned files) + docker_daily_restart.sh (NPM, Authelia, Dispatcharr fresh start) + +2:30am Sunday: + Stop Emby + auth stack both sides + Pull updated Docker images (zero extra downtime — already stopped) + rsync Emby + Critical-Data clean mirror + Start containers on new images + docker_weekly_restart.sh (NextCloud, AdGuard, Immich) + +7am Sunday: + sunday_morning_coffee_report.sh (everything that happened this week) +``` + +--- + +## ━━━ THE SELF-HEALING STACK ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Problems are handled at the most appropriate level. Scripts escalate only when the +level below them has failed: + +``` +Problem: Emby container using 22GB RAM (limit: 18GB) + → docker_watchdog.sh Tier 1: restart Emby immediately + → System continues, no reboot + +Problem: Authelia crashed and won't restart + → docker_watchdog.sh Tier 1: attempt restart (dependency check: DB + Redis first) + → Fails 3 times in 1 hour → skip list → critical notification + → Tools: watchdog_skip_list_manager.sh to clear after fixing + +Problem: RAM drops to 3GB + → system_watchdog.sh: stop non-essential containers + → Wait for RAM to recover to 30GB + → docker_watchdog.sh defers (reads mem_shutdown_active=true) + → If RAM drops to 4GB AND 3+ OOM kills: bypass strikes → reboot + +Problem: Docker daemon hung + → system_watchdog.sh Tier 1: attempt /etc/rc.d/rc.docker restart + → If still hung: immediate reboot (no strikes needed — daemon can't be managed) + +Problem: HOST1 loses power + → failover.sh on HOST2 detects at next 2-minute check + → Tier 1 starts immediately: DDNS, Emby, VaultWarden, Live TV + → Household barely notices. Live TV may buffer for < 2 minutes. +``` + +**→ Container healing: [README-Docker_Essentials.md](Docker_Essentials/README-Docker_Essentials.md)** +**→ System stability: [README-Unraid_Essentials.md](unRAID_Essentials/README-Unraid_Essentials.md)** + +--- + +## ━━━ SETUP — WHERE TO START ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +If you're setting this up from scratch on two servers: + +``` +1. Follow README-Rsync_Setup.md completely — end-to-end setup guide + Tailscale → SSH keys → git clone → master.conf → User Scripts schedule + +2. Set your container names correctly before anything else + Shared services: no suffix (Emby, NginxProxyManager, etc.) + Personal services: suffixed (Emby-Gmer4Lfe, Organizer-Gmer4Lfe) + +3. Set your share names identically on both servers + /mnt/user/Movies on both, /mnt/user/Tv_Shows on both, etc. + +4. Run initial sync to populate HOST2 + bash daily_sync_maintenance.sh --log + +5. Verify failover works before relying on it + bash failover_test.sh --dry-run first + bash failover_test.sh when ready — uses iptables simulation, no real downtime +``` + +--- + +## ━━━ FOLDER README REFERENCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +| README | What It Covers | +|--------|---------------| +| [README-Failover.md](Failover/README-Failover.md) | DDNS sequencing, tiered failover, handback, split brain prevention, failover_test.sh | +| [README-Docker_Essentials.md](Docker_Essentials/README-Docker_Essentials.md) | Two-tier container watchdog, memory limits, dependency ordering, skip list, daily/weekly restarts | +| [README-Unraid_Essentials.md](unRAID_Essentials/README-Unraid_Essentials.md) | Three-tier system watchdog, WebGUI recovery, inotify tuning, PHP-FPM, log hygiene | +| [README-Orchestrators.md](Orchestrators/README-Orchestrators.md) | What runs when, execution order, daily/weekly windows, adding jobs | +| [README-Media.md](Media/README-Media.md) | Permissions model, junk cleanup, arr orphan removal, safety layers, testing procedure | +| [README-Transcoding.md](Transcodes/README-Transcoding.md) | Ramdisk design, symlink architecture, Docker mount requirement, SSD fallback | +| [README-Monitors.md](Monitors/README-Monitors.md) | Cert monitoring, SMART health, bandwidth tracking, health digest profiles | +| [README-Partnership.md](Partnership/README-Partnership.md) | Auth stack sharing, onboard/offboard/transfer lifecycle, deferred offboard | +| [README-Rsync_Setup.md](Rsync/README-Rsync_Setup.md) | Complete setup guide — Tailscale, SSH keys, git clone, master.conf, User Scripts | +| [README-Tools.md](Tools/README-Tools.md) | failover_state_reset, skip list manager, emby_database_repair, container export | +| [README-Unraid_Essentials.md](unRAID_Essentials/README-Unraid_Essentials.md) | `git_pull_execute.sh`, `server_reboot.sh`, `mover_stop.sh`, `user_scripts_stop.sh` | + +--- + +## ━━━ A FEW THINGS THAT WILL SURPRISE YOU ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Things that are different from what you might expect: ```bash -TRANSCODE_MANAGER_MODE="advanced" +# ───────────────────────────────────────────────────────────────────────────── +# 1. The User Scripts plugin has very few entries. +# Everything runs through orchestrators. Individual scripts are never +# scheduled directly. array_start.sh is the only "At Startup of Array" entry. -TRANSCODE_FORCE_RAMDISK=( - "LiveTv" # always ramdisk — buffering is latency sensitive -) -TRANSCODE_FORCE_SSD=( - "Audio" # music/downloads — no benefit from ramdisk -) -# Everything else follows smart threshold behavior +# 2. Scripts are silent when healthy. +# system_watchdog.sh runs 288 times per day. It produces zero output on clean +# cycles. If you see output, something needs attention. If you see nothing, +# the system is healthy. This is by design. + +# 3. master.conf is the only file you edit. +# Master_host*.conf for per-server values. Everything else is code — don't +# add configuration into scripts directly. Everything configurable has a var. + +# 4. --dry-run works on everything. +# Every single script supports --dry-run. Pre-flight checks still run. +# Review what would happen before anything changes. Always available. + +# 5. detect_hosts() does the heavy lifting. +# Scripts never compare hostnames directly. detect_hosts() aliases the correct +# HOST*_ vars based on the running server. The same script runs correctly on +# HOST1 and HOST2 — no branching on server identity inside scripts. + +# 6. The Emby Docker mount is unusual and critical. +# --mount type=bind,...,bind-propagation=shared in Extra Parameters — not a +# standard path mapping. Without shared propagation symlink flips are silently +# ignored. README-Transcoding.md explains why in detail. + +# 7. arr cleanup scripts can delete files. +# lidarr_cleanup.sh, sonarr_cleanup.sh, radarr_cleanup.sh permanently delete +# orphaned files. Safety layers prevent catastrophic runs but always test with +# --dry-run --log first on a new system. README-Media.md has the full procedure. +# ───────────────────────────────────────────────────────────────────────────── ``` -The groundwork is already in place — session display with media type parsing is working. Advanced mode requires Emby to expose media type at the point ffmpeg resolves the symlink, which means the routing decision needs to happen before the session starts. This is the problem to solve. - -### Plugin Dashboard - -The Monitors folder is already the data backend for a future unRAID plugin dashboard. Every monitor script writes structured state that a plugin could read and display: - -``` -Failover state → live status indicator -Container watchdog → strike counts, skip list -SMART health → per-drive status -ZFS pools → health + ARC utilization -Transcode sessions → live session display -Bandwidth history → transfer trend charts -Certificate expiry → days remaining per domain -``` - -The scripts exist. The data exists. The plugin is the frontend. - -### Failover — Tiered by Content Type - -Currently failover starts containers based on time elapsed. A future enhancement would start containers based on what the primary server was doing before it went down: - -``` -Primary running Live TV sessions → start Live TV stack on secondary immediately -Primary idle → standard tier delays apply -Primary in heavy transcoding → start Emby immediately, defer others -``` - -This requires the failover state file to track active session types, which requires integration with the Emby API at failover trigger time. - -### Health Digest — Plugin Integration - -Currently the digest sends a notification. In v2 it populates a persistent dashboard that shows a rolling week of system health at a glance — without requiring a notification for every event. - -### Container Health Checks — Standardised Library - -The docker_watchdog already has HTTP check support per container. A v2 enhancement is a standardised health check library — pre-built check commands for every container in the stack that can be applied via Extra Parameters in the unRAID template without manual research per container. - --- -## Origin +## ━━━ PROJECT NAME ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -This ecosystem grew from a single failover script — written as a first real bash project, refined through months of production use, redesigned multiple times as the stack grew more complex. Each script in the collection started as a one-off solution to a specific problem. Over time the patterns that worked were extracted into `common.sh`, the configuration was centralised into `Master.conf`, and the whole collection was standardised into what it is now. - -The failover script that started it all is still the soul of the ecosystem — the DDNS sequencing, the two-ping state machine, the handback order. Everything else built on top of that foundation. - -Two servers, one codebase, self-healing infrastructure. Step away. Come back to a happy system. \ No newline at end of file +``` +Working name: Arbiter +Repository: git@192.168.50.2:FailedProxy/Unraid_Scripts.git +Install path: /mnt/user/appdata/unraid_scripts +``` \ No newline at end of file diff --git a/Rsync/README-Rsync_Setup.md b/Rsync/README-Rsync_Setup.md index a49ce72..46ea6bb 100644 --- a/Rsync/README-Rsync_Setup.md +++ b/Rsync/README-Rsync_Setup.md @@ -1,628 +1,993 @@ -# Rsync Setup Guide +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# 🔄 RSYNC SETUP GUIDE +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -> For the unRAID Script Ecosystem — `Master.conf` · `common.sh` · `rsync.sh` · `daily_sync_maintenance.sh` · `weekly_sync_maintenance.sh` +**Complete setup guide for the two-server rsync ecosystem.** By the end of this guide +both servers will have SSH keys configured, Tailscale connected, the git repository +cloned, and all scheduled operations running automatically. + +> **This is a setup guide, not a script reference.** For how rsync.sh works internally, +> profiles, safety checks, and operational details — those belong in the orchestrator +> and rsync script documentation. This guide is about standing the ecosystem up from +> scratch and verifying it works. --- -## Overview +## ━━━ WHAT YOU'RE BUILDING ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -This guide walks through setting up the rsync ecosystem on both your primary and secondary unRAID 7.x servers. By the end you will have: +``` +HOST1 (unRAID-Gmer4Lfe) HOST2 (unRAID-Jayred365) +────────────────────── ────────────────────── +Source of truth: Source of truth: + Movies, Tv_Shows, Music Anime_Shows, Anime_Movies + Critical-Data (auth stack) + Emby userdata -- SSH keys configured for server-to-server communication -- Tailscale running on both servers for secure networking -- A Gitea repository cloned to both servers -- All scripts scheduled via the User Scripts plugin -- Automated daily sync of media shares driven by orchestrators -- Automated weekly clean sync of critical appdata (Emby + auth stack) -- Optional personal encrypted shares synced for offsite backup +Pushes to HOST2 daily: ──────→ Receives: + All HOST1 shares Mirror of HOST1 shares + Personal encrypted shares Personal (encrypted blocks) + +Receives from HOST2 daily: ←────── Pushes: + Anime_Shows, Anime_Movies All HOST2 shares + +Weekly clean sync (both sides stopped): + Emby — full clean mirror ←──────→ Emby + Critical-Data ──────→ Auth stack (HOST1 → HOST2) + +Every 30 minutes — dirty sync: + Emby watch states ──────→ HOST2 stays current on playback +``` --- -## Prerequisites +## ━━━ PREREQUISITES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Both servers need the following before starting: - -- unRAID 7.x -- [Community Applications plugin](https://forums.unraid.net/topic/38582-plug-in-community-applications/) installed -- [User Scripts plugin](https://forums.unraid.net/topic/48286-plugin-user-scripts/) installed via Community Applications -- [Tailscale plugin](https://forums.unraid.net/topic/136889-tailscale-plugin/) installed via Community Applications -- Access to a Gitea instance (self-hosted recommended — Gitea runs as a Docker container on HOST1) -- Terminal access to both servers (unRAID UI → Tools → Terminal, or SSH) - ---- - -## Step 1 — Tailscale Setup - -Tailscale provides the secure network tunnel between your two servers. Scripts resolve the remote server's IP via Tailscale at runtime — no hardcoded IPs needed. - -### On Both Servers - -1. Open **Apps** in the unRAID UI -2. Search for **Tailscale** and install the plugin -3. Go to **Settings → Tailscale** -4. Click **Connect** and authenticate with your Tailscale account -5. Verify both servers appear in your [Tailscale admin console](https://login.tailscale.com/admin/machines) - -### Verify Connectivity - -Run this on HOST1 to confirm it can reach HOST2: +Both servers need these before starting: ```bash +# ───────────────────────────────────────────────────────────────────────────── +# Required on both servers: + unRAID 7.x + Community Applications plugin — search "Community Applications" in unRAID plugins + User Scripts plugin — install via Community Applications + Tailscale plugin — install via Community Applications + Terminal access — unRAID UI → Tools → Terminal, or SSH + +# Optional but recommended: + Gitea (Docker container on HOST1) — self-hosted git for the script repository + Working Emby installation — for transcode management and failover +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +## ━━━ STEP 1 — TAILSCALE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Tailscale provides the encrypted mesh network between servers. Scripts resolve the +remote server's IP via Tailscale at runtime — no hardcoded IPs, no VPN configuration, +no open ports. All server-to-server communication goes through Tailscale. + +--- + +### ── Install on Both Servers ──────────────────────────────────────────────── + +```bash +# On each server: +# ───────────────────────────────────────────────────────────────────────────── +# 1. Open Apps in the unRAID UI +# 2. Search for "Tailscale" — install the plugin +# 3. Settings → Tailscale → Connect +# 4. Authenticate with your Tailscale account (browser opens on your machine) +# 5. Verify both servers appear: https://login.tailscale.com/admin/machines +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Verify Connectivity ───────────────────────────────────────────────────── + +```bash +# From HOST1 — should return HOST2's 100.x.x.x Tailscale IP: tailscale ip -4 unRAID-Jayred365 + +# From HOST2 — should return HOST1's 100.x.x.x Tailscale IP: +tailscale ip -4 unRAID-Gmer4Lfe + +# Test actual connectivity: +tailscale ping unRAID-Jayred365 # run from HOST1 ``` -You should get back a `100.x.x.x` IP. If not, check both machines are authenticated in the Tailscale admin console. - -> **Important:** The hostnames in `Master.conf` (`HOST1` and `HOST2`) must match the Tailscale machine names exactly — case sensitive. +> **Critical:** The hostnames in `master.conf` (`HOST1` and `HOST2`) must match the +> Tailscale machine names **exactly** — case sensitive. The ecosystem resolves all +> remote IPs via `tailscale ip -4 HOSTNAME` at runtime. A name mismatch means every +> script that touches the remote will fail at the IP resolution step. --- -## Step 2 — Enable SSH on unRAID +## ━━━ STEP 2 — ENABLE SSH ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -unRAID 7.x has SSH disabled by default. Enable it on both servers so scripts can connect between them. - -1. Go to **Settings → Management Access** -2. Under **Secure Shell** set **SSH** to `Enabled` -3. Set **SSH port** to `22` -4. Click **Apply** - -> SSH is only exposed on your local network and Tailscale interface. Scripts connect via Tailscale IP — traffic is encrypted end-to-end. - ---- - -## Step 3 — Generate SSH Keys - -The scripts use SSH keys for two purposes: -- **Server-to-server rsync and failover** — each server authenticates to the other -- **Gitea access** — both servers pull from the git repository - -### 3a — Server-to-Server Keys - -**On HOST1 (unRAID-Gmer4Lfe):** +unRAID 7.x has SSH disabled by default. Enable it on both servers. ```bash +# On each server: +# ───────────────────────────────────────────────────────────────────────────── +# Settings → Management Access → Secure Shell +# SSH: Enabled +# SSH port: 22 +# Apply +# ───────────────────────────────────────────────────────────────────────────── +``` + +> SSH is only exposed on your local network and Tailscale interface. Scripts connect +> via Tailscale IP — all traffic is encrypted end-to-end. No ports are opened to +> the public internet. + +--- + +## ━━━ STEP 3 — SSH KEYS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Two sets of keys needed: server-to-server for rsync and failover, and Gitea access +for script repository pull. Generate all keys before configuring anything else. + +--- + +### ── 3a — Server-to-Server Keys ───────────────────────────────────────────── + +```bash +# On HOST1 — generate HOST1's key pair: ssh-keygen -t ed25519 -f /root/.ssh/Gmer4Lfe-rsync-key -C "gmer4lfe-rsync" -N "" -``` -**On HOST2 (unRAID-Jayred365):** - -```bash +# On HOST2 — generate HOST2's key pair: ssh-keygen -t ed25519 -f /root/.ssh/Jayred365-rsync-key -C "jayred365-rsync" -N "" ``` -### 3b — Authorise Keys on Each Server +--- -HOST1's public key must be authorised on HOST2, and vice versa. - -**Copy HOST1 key → HOST2:** +### ── 3b — Authorise Keys Bidirectionally ───────────────────────────────────── ```bash -# On HOST1 — print the public key +# HOST1's public key must be authorised on HOST2 (so HOST1 can SSH into HOST2): +# ───────────────────────────────────────────────────────────────────────────── + +# On HOST1 — print the public key: cat /root/.ssh/Gmer4Lfe-rsync-key.pub -# On HOST2 — paste and authorise +# On HOST2 — create authorized_keys and paste HOST1's public key: mkdir -p /root/.ssh -echo "PASTE_PUBLIC_KEY_HERE" >> /root/.ssh/authorized_keys +echo "PASTE_HOST1_PUBLIC_KEY_HERE" >> /root/.ssh/authorized_keys chmod 600 /root/.ssh/authorized_keys -``` -**Copy HOST2 key → HOST1:** +# HOST2's public key must be authorised on HOST1 (so HOST2 can SSH into HOST1): +# ───────────────────────────────────────────────────────────────────────────── -```bash -# On HOST2 — print the public key +# On HOST2 — print the public key: cat /root/.ssh/Jayred365-rsync-key.pub -# On HOST1 — paste and authorise -echo "PASTE_PUBLIC_KEY_HERE" >> /root/.ssh/authorized_keys +# On HOST1 — append HOST2's public key: +echo "PASTE_HOST2_PUBLIC_KEY_HERE" >> /root/.ssh/authorized_keys ``` -### 3c — Test the Connection - -From HOST1, verify it can SSH to HOST2 without a password prompt: - -```bash -ssh -i /root/.ssh/Gmer4Lfe-rsync-key root@$(tailscale ip -4 unRAID-Jayred365) "echo connected" -``` - -You should see `connected`. If prompted for a password the key was not authorised correctly — recheck Step 3b. - -### 3d — Gitea SSH Key - -Generate a separate key for Gitea access on each server: - -```bash -ssh-keygen -t ed25519 -f /root/.ssh/unraid_gitea -C "unraid-gitea" -N "" -``` - -Add the public key to your Gitea account: - -```bash -cat /root/.ssh/unraid_gitea.pub -``` - -Copy the output and add it in Gitea under **Settings → SSH / GPG Keys → Add Key**. - --- -## Step 4 — Clone the Git Repository - -Both servers clone from the same Gitea repository. Updates pushed to the repo propagate to both servers on the next daily git pull. - -### On Both Servers +### ── 3c — Test Both Directions ──────────────────────────────────────────────── ```bash -# Create the target directory +# From HOST1 — should print "connected" without a password prompt: +ssh -i /root/.ssh/Gmer4Lfe-rsync-key \ + root@$(tailscale ip -4 unRAID-Jayred365) \ + "echo connected" + +# From HOST2 — should print "connected" without a password prompt: +ssh -i /root/.ssh/Jayred365-rsync-key \ + root@$(tailscale ip -4 unRAID-Gmer4Lfe) \ + "echo connected" +``` + +``` +If prompted for a password: the key was not authorised correctly. + → Recheck Step 3b — the public key content must be on one line + → Check permissions: chmod 600 /root/.ssh/authorized_keys + → Check the key file referenced in the SSH command matches what was generated +``` + +--- + +### ── 3d — Gitea SSH Key ─────────────────────────────────────────────────────── + +```bash +# On BOTH servers — generate a key for Gitea access: +ssh-keygen -t ed25519 -f /root/.ssh/unraid_gitea -C "unraid-gitea" -N "" + +# Print the public key to add to Gitea: +cat /root/.ssh/unraid_gitea.pub + +# In Gitea: Settings → SSH / GPG Keys → Add Key → paste the output above +# Do this for both servers if they have separate Gitea accounts, or once if shared +``` + +--- + +## ━━━ STEP 4 — CLONE THE REPOSITORY ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Both servers clone from the same Gitea repository. Updates pushed to the repo +propagate to both servers automatically via `git_pull_execute.sh` at the start of +each daily maintenance window. + +--- + +### ── On Both Servers ───────────────────────────────────────────────────────── + +```bash +# Create the target directory: mkdir -p /mnt/user/appdata/unraid_scripts -# Clone the repository +# Clone the repository: GIT_SSH_COMMAND="ssh -i /root/.ssh/unraid_gitea" \ git clone git@YOUR_GITEA_HOST:FailedProxy/Unraid_Scripts.git \ /mnt/user/appdata/unraid_scripts + +# Replace YOUR_GITEA_HOST with your Gitea server address and port +# Example: git@192.168.50.2:221 ``` -Replace `YOUR_GITEA_HOST` with your Gitea server address and port. +--- -### Verify the Structure +### ── Verify the Structure ───────────────────────────────────────────────────── ```bash -ls /mnt/user/appdata/unraid_scripts +ls /mnt/user/appdata/unraid_scripts/ ``` -You should see: - ``` -Master.conf -common.sh -Orchestrators/ -Rsync/ -Failover/ -Docker_Essentials/ -unRAID_Essentials/ -Media/ -Transcodes/ -Monitors/ -Tools/ +Expected output: + master.conf ← all user configuration — the only file you edit + master_host1.conf ← HOST1-specific configuration + master_host2.conf ← HOST2-specific configuration + common.sh ← shared library — functions used by all scripts + load_config.sh ← config loader + Orchestrators/ + Rsync/ + Failover/ + Docker_Essentials/ + unRAID_Essentials/ + Media/ + Transcodes/ + Monitors/ + Tools/ + Partnership/ ``` -### Make Scripts Executable +--- + +### ── Make Scripts Executable ───────────────────────────────────────────────── ```bash +# Execute permission on all scripts — required once after clone: +find /mnt/user/appdata/unraid_scripts -name "*.sh" -exec chmod +x {} \; +``` + +> `array_start.sh` auto-fixes permissions on scripts that lost the execute bit — +> but this initial chmod ensures the first run works before that safeguard is active. + +--- + +## ━━━ STEP 5 — CONFIGURE MASTER.CONF ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +All user configuration lives in `master.conf`. Every value with a comment +`# REQUIRED` must be set before the first run. Everything else has working defaults. + +```bash +nano /mnt/user/appdata/unraid_scripts/master.conf +``` + +--- + +### ── Host Identity ───────────────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# These must match Tailscale machine names exactly — case sensitive. +# The ecosystem uses these to resolve remote IPs at runtime. +# + HOST1="unRAID-Gmer4Lfe" # REQUIRED — must match tailscale machine name + HOST2="unRAID-Jayred365" # REQUIRED — same + +# SSH key paths — each server's key for authenticating to the other: + HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" # HOST1 uses this to SSH to HOST2 + HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" # HOST2 uses this to SSH to HOST1 +``` + +--- + +### ── Emby API Keys ───────────────────────────────────────────────────────────── + +```bash +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# Used by: emby_session_report.sh, sunday_morning_coffee_report.sh, +# sonarr/radarr cleanup (notify_emby_scan after deletion) +# +# Get from: Emby Dashboard → Settings → API Keys → + New API Key +# + HOST1_EMBY_URL="http://192.168.50.2:8096" + HOST1_EMBY_API_KEY="your-host1-emby-api-key" # REQUIRED for Emby features + HOST1_EMBY_CONTAINER="Emby" + +# master_host2.conf + HOST2_EMBY_URL="http://localhost:8096" + HOST2_EMBY_API_KEY="your-host2-emby-api-key" + HOST2_EMBY_CONTAINER="Emby" +``` + +--- + +### ── Git Repository ──────────────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Used by git_pull_execute.sh — pulls latest scripts at start of each daily window. +# + GITEA_CONTAINER="Gitea" # exact Docker container name + GITEA_REPO_PATH="FailedProxy/Unraid_Scripts.git" + TARGET_DIR="/mnt/user/appdata/unraid_scripts" + GITEA_SSH_KEY="/root/.ssh/unraid_gitea" + SSH_PORT=221 # your Gitea SSH port +``` + +--- + +### ── Daily Sync Shares ──────────────────────────────────────────────────────── + +```bash +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# Shares HOST1 is source of truth for — pushed to HOST2 every night at 1am. +# HOST2 treats these as read-only mirrors. Never put the same share in both lists. +# +HOST1_DAILY_SYNC_SHARES=( + "/mnt/user/Movies" # HOST1 manages this — Radarr runs here + "/mnt/user/Tv_Shows" # HOST1 manages this — Sonarr runs here + "/mnt/user/Music" # HOST1 manages this — Lidarr runs here + "/mnt/user/Kids_Movies" + "/mnt/user/Kids_Tv_Shows" + "/mnt/user/Sports" + "/mnt/user/stand-up_comedy" +) + +# master_host2.conf +HOST2_DAILY_SYNC_SHARES=( + "/mnt/user/Anime_Shows" # HOST2 manages this — his Sonarr runs here + "/mnt/user/Anime_Movies" # HOST2 manages this — his Radarr runs here +) +``` + +--- + +### ── Weekly Sync Shares ─────────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Synced during the Sunday 2:30am window — containers stopped both sides. +# Do NOT add these to a separate cron schedule — they run via weekly_sync_maintenance.sh. +# +WEEKLY_SYNC_SHARES=( + "/mnt/user/Media_Server/Emby" # full clean Emby mirror + "/mnt/user/appdata-Failover/Critical-Data" # auth stack clean state +) +``` + +--- + +## ━━━ STEP 6 — MASTER_HOST*.CONF ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Per-host configuration lives in `master_host1.conf` and `master_host2.conf`. +`detect_hosts()` in `common.sh` reads which server is running and aliases the correct +`HOST*_` prefixed variables to their unprefixed names. Scripts only ever reference the +unprefixed name — they work identically on both servers. + +```bash +# master_host1.conf is only sourced on HOST1 +# master_host2.conf is only sourced on HOST2 +# Changes go in the right file for the right server + +nano /mnt/user/appdata/unraid_scripts/master_host1.conf # on HOST1 +nano /mnt/user/appdata/unraid_scripts/master_host2.conf # on HOST2 +``` + +See each conf file's comments — every variable is documented with its purpose +and the reasoning behind the value. + +--- + +## ━━━ STEP 7 — RSYNC PROFILES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Profiles control per-share behaviour — which containers to stop, rsync flags, +bandwidth limits, what to exclude. Profile is matched by directory basename +(lowercased). Override with `--profile=name`. + +--- + +### ── How Profile Matching Works ────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# rsync.sh /mnt/user/appdata-Failover/Arrs_Stack +# basename: Arrs_Stack +# lowercased: arrs_stack +# matched profile: [arrs_stack] +# +# rsync.sh /mnt/user/Movies +# basename: Movies +# lowercased: movies +# no matching profile → global defaults apply (no containers stopped) +# +# rsync.sh /mnt/user/appdata-Failover/Critical-Data --profile=critical-failover +# explicit override → uses [critical-failover] profile regardless of path +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Current Profiles ───────────────────────────────────────────────────────── + +```bash +# master.conf — profile definitions +# ───────────────────────────────────────────────────────────────────────────── +# Each profile defines which containers to stop, rsync flags, excludes, etc. +# Containers in PROFILE_CRITICAL_CONTAINER_NAMES are stopped on BOTH servers. +# PROFILE_DELAYED_CONTAINERS restart after PROFILE_CONTAINER_DELAY seconds. + +# ── arrs_stack ───────────────────────────────────────────────────────────── +# Arr databases — stopped for clean SQLite snapshot + PROFILES["arrs_stack_CRITICAL_CONTAINER_NAMES"]=( + "Sonarr" "Radarr" "Lidarr" "Prowlarr" "Bazarr" "Pinchflat" + ) + +# ── critical-data ────────────────────────────────────────────────────────── +# Auth stack — stopped for clean database snapshot, delayed restart + PROFILES["critical-data_CRITICAL_CONTAINER_NAMES"]=( + "Mariadb-Authelia" "Redis-Authelia" + "NginxProxyManager" "Lldap-Gmer4Lfe" + ) + PROFILES["critical-data_DELAYED_CONTAINERS"]=( + "Authelia" "Authelia-Secondary" # auth services restart after delay + ) + PROFILES["critical-data_CONTAINER_DELAY"]=30 # seconds before delayed containers start + +# ── important-data ───────────────────────────────────────────────────────── +# NextCloud + Postgres — stopped for clean snapshot + PROFILES["important-data_CRITICAL_CONTAINER_NAMES"]=( + "Postgres-NextCloud" + ) + PROFILES["important-data_DELAYED_CONTAINERS"]=("NextCloud") + +# ── emby ─────────────────────────────────────────────────────────────────── +# Weekly full clean sync — both Emby instances stopped + PROFILES["emby_CRITICAL_CONTAINER_NAMES"]=("Emby") + PROFILES["emby_EXCLUDE_DIRS"]=( + "transcodes/" "logs/" "crash*" "cache/" + ) + +# ── emby-failover ────────────────────────────────────────────────────────── +# Every 30 minutes, Emby STAYS RUNNING — dirty sync of critical state only +# WAL and SHM excluded — safe to copy while Emby is writing + PROFILES["emby-failover_CRITICAL_CONTAINER_NAMES"]=() # empty — nothing stops + PROFILES["emby-failover_EXCLUDE_DIRS"]=( + "*.wal" "*.shm" # WAL files — unsafe mid-write + "transcodes/" "logs/" "crash*" "cache/" # volatile data — skip + ) + PROFILES["emby-failover_REMOTE_RESTART_CONTAINERS"]=("Emby") + # Emby on HOST2 restarts after sync to pick up config changes +``` + +--- + +### ── Two Emby Profiles — Why Both Exist ────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# emby-failover — every 30 minutes, Emby stays running: +# What syncs: users.db, library.db, authentication.db, config/ +# What skips: *.wal *.shm transcodes/ logs/ cache/ +# Why: WAL files are being written while Emby runs — copying them +# would produce a corrupt database on HOST2 +# Result: HOST2 is always within 30 minutes of HOST1 on watch state +# and user activity. Failover is seamless — nobody notices. +# +# emby — Sunday 2:30am, both Emby instances stopped: +# What syncs: everything except transcodes, logs, cache, crash files +# What includes: metadata, plugins, full database state, all config +# Why: WAL is checkpointed on clean shutdown — safe to copy everything +# Full consistent mirror including metadata and plugin state +# Result: HOST2 has a gold-standard Emby state once per week +# Image cache warm for 6 days — only reset Sunday when users sleep +# +# The two profiles work together: +# emby-failover: keeps HOST2 current on what matters for immediate failover +# emby: gives HOST2 full fidelity once per week +# Neither alone is sufficient — both are needed. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +## ━━━ STEP 8 — PERSONAL ENCRYPTED SHARES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Personal shares are synced to the remote server for offsite backup. ZFS encrypts at +the dataset level — the remote server receives encrypted blocks and cannot read the +content without your passphrase or keyfile. + +--- + +### ── Create an Encrypted ZFS Dataset ──────────────────────────────────────── + +```bash +# In the unRAID UI: +# ───────────────────────────────────────────────────────────────────────────── +# Main → click your ZFS pool name → + Dataset +# Name: Gmer4Lfe-Personal +# Encryption: Enabled +# Passphrase: [your passphrase] +# ⚠️ Write your passphrase down — if lost, data is completely unrecoverable +# +# Settings → Shares → Add Share +# Share path: point to the new encrypted dataset +# Use cache: Only — keeps data on ZFS pool, not array +# ───────────────────────────────────────────────────────────────────────────── + +# Verify encryption is active before syncing: +zfs get encryption poolname/Gmer4Lfe-Personal +# Should show: encryption aes-256-gcm +``` + +--- + +### ── Auto-Unlock on Boot (Optional) ───────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Keyfile approach — passphrase stored in a file, loaded at boot. +# More convenient but the keyfile is a secret that must be protected. +# Never sync the keyfile to the remote server. +# +# Create keyfile — on HOST1 only: +dd if=/dev/urandom bs=32 count=1 | base64 > /root/.zfs-keys/personal.key +chmod 600 /root/.zfs-keys/personal.key + +# Set dataset to use keyfile instead of passphrase: +zfs change-key \ + -o keylocation=file:///root/.zfs-keys/personal.key \ + -o keyformat=raw \ + poolname/Gmer4Lfe-Personal + +# Add to ramdisk_setup.sh or array_start.sh custom scripts: +zfs load-key poolname/Gmer4Lfe-Personal +zfs mount poolname/Gmer4Lfe-Personal +# ───────────────────────────────────────────────────────────────────────────── +# Manual unlock alternative (most secure — passphrase only in your head): +zfs load-key poolname/Gmer4Lfe-Personal # prompts for passphrase +zfs mount poolname/Gmer4Lfe-Personal +``` + +--- + +### ── Add to master_host1.conf ──────────────────────────────────────────────── + +```bash +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# Personal shares append to the daily sync after DAILY_SYNC_SHARES. +# Remote server receives encrypted blocks — cannot read content without your key. +# +HOST1_PERSONAL_SHARES=( + "/mnt/user/Gmer4Lfe-Personal" +) +``` + +--- + +## ━━━ STEP 9 — USER SCRIPTS SETUP ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +The ecosystem is designed so the User Scripts plugin has only a small number of entries — +each one an orchestrator. Individual scripts are never scheduled directly. + +--- + +### ── At Startup of Array ───────────────────────────────────────────────────── + +```bash +# Create one script entry named "array start": +# ───────────────────────────────────────────────────────────────────────────── +#!/bin/bash +bash /mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh +# ───────────────────────────────────────────────────────────────────────────── +# Schedule: At Startup of Array +# Run as: Background Task +# +# This is the ONLY "At Startup of Array" entry needed. +# It launches everything in ARRAY_START_SCRIPTS from master.conf: +# inotify_tuning.sh — raise inotify limits before containers start +# docker_syslog_filter.sh — suppress veth log noise +# php_fpm_max_children.sh — WebGUI tuning +# ramdisk_setup.sh — create ramdisk before Emby starts +# docker_network_connect.sh — connect containers to extra networks +# system_watchdog.sh — continuous system health monitor +# docker_watchdog.sh — continuous container health monitor +# failover.sh — continuous mutual failover +``` + +--- + +### ── Cron Schedule ──────────────────────────────────────────────────────────── + +```bash +# Create one script entry per cron schedule below. +# All entries: Run as Background Task +# ───────────────────────────────────────────────────────────────────────────── + +# Every 3 minutes — transcode cleanup + manager: +*/3 * * * * +bash /mnt/user/appdata/unraid_scripts/Orchestrators/transcode_management.sh + +# Every 30 minutes — Emby dirty sync (watch states, library delta): +*/30 * * * * +bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ + /mnt/user/Media_Server/Emby --profile=emby-failover + +# Every 6 hours — failed import + stalled download recovery: +0 */6 * * * +bash /mnt/user/appdata/unraid_scripts/Orchestrators/arrs_failed_stalled_recovery.sh + +# 1am daily — full maintenance window: +# git pull → rsync all shares → permissions → cleaners → arr cleanup → docker restart +0 1 * * * +bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh + +# 2:30am Sunday — weekly maintenance window: +# stop containers → pull updates → clean sync → start containers → weekly restarts +30 2 * * 0 +bash /mnt/user/appdata/unraid_scripts/Orchestrators/weekly_sync_maintenance.sh + +# 8am daily — health digest (DIGEST_PROFILE in master.conf controls when it notifies): +0 8 * * * +bash /mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh + +# Every 6 hours — inotify + php-fpm utilisation snapshot: +0 */6 * * * +bash /mnt/user/appdata/unraid_scripts/Monitors/system_tuning_monitor.sh + +# Sunday morning — weekly reports: +0 6 * * 0 bash .../Monitors/zfs_memory_snapshot.sh +0 7 * * 0 bash .../Monitors/smart_health.sh +0 9 * * 0 bash .../Monitors/cert_monitor.sh +0 10 * * 0 bash .../Monitors/backup_verify.sh +0 11 * * 0 bash .../Monitors/emby_session_report.sh +0 11 * * 0 bash .../Monitors/bandwidth_monitor.sh --report +# ───────────────────────────────────────────────────────────────────────────── +``` + +> **Set all entries to "Background Task"** — output streams correctly to the User +> Scripts log rather than buffering in the browser tab. Non-background tasks can +> appear to hang on long-running scripts. + +--- + +## ━━━ STEP 10 — VERIFY THE SETUP ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Before relying on scheduled jobs, test manually from the terminal on HOST1. +Test with `--dry-run` first — no changes made, but the full pre-flight and +configuration resolution runs. + +--- + +### ── Test a Single Profile Sync ───────────────────────────────────────────── + +```bash +# Dry run with verbose output — shows every decision the script makes: +bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ + /mnt/user/appdata-Failover/Arrs_Stack --dry-run --log +``` + +``` +Expected output (healthy): + ━━━ ⚙️ Setup ━━━ + Host: HOST1 (unRAID-Gmer4Lfe) → HOST2 (unRAID-Jayred365) + Remote IP: 100.x.x.x + Profile: arrs_stack + + ━━━ 🛡️ Pre-flight Checks ━━━ + ✅ Remote reachable + ✅ version parity — both on unRAID X.Y.Z + ✅ Remote Docker daemon responding + ✅ Remote rootfs: 12% (threshold: 75%) + ✅ Remote share exists and not empty + ✅ All pre-flight checks passed +``` + +--- + +### ── Test the Daily Orchestrator ───────────────────────────────────────────── + +```bash +# Dry run of the full daily window — shows every job that would run: +bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh --dry-run +``` + +``` +If any pre-flight check fails, the script aborts with a clear error message +before touching anything. Fix the reported issue and re-run --dry-run. + +Common pre-flight failures and their causes: + "Remote not reachable" → Tailscale not connected on HOST2 + "Version mismatch" → different unRAID versions — update before syncing + "Remote rootfs above X%" → HOST2's root filesystem nearly full + "Remote share missing" → share doesn't exist on HOST2 yet (see Step 11) + "Docker daemon not responding" → HOST2's Docker service not started +``` + +--- + +### ── Check the Configuration Resolved Correctly ────────────────────────────── + +```bash +# --status shows how master.conf resolved for this server and profile: +bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ + /mnt/user/appdata-Failover/Arrs_Stack --status +``` + +--- + +## ━━━ STEP 11 — INITIAL HOST2 SYNC ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +If HOST2 is being set up from scratch with empty shares: + +--- + +### ── Create Share Structure on HOST2 ──────────────────────────────────────── + +```bash +# On HOST2 — start the array and create shares via the unRAID UI. +# Or use the share recreation tool to create disk directories from HOST1's cfg files: +bash /mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh +``` + +--- + +### ── Initial Push From HOST1 ───────────────────────────────────────────────── + +```bash +# On HOST1 — push all shares to HOST2 for the first time: +# Use --log for verbose output on first run +bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh --log +``` + +``` +First run may take several hours for large libraries — this is normal. +The scheduled nightly sync will be incremental after the initial push. +Progress shows per-share throughout. +``` + +--- + +## ━━━ NAMING CONSISTENCY — THIS IS REQUIRED ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +The ecosystem uses one codebase on both servers. This only works if containers and +shares have identical names on both servers. This is not configurable — it is a +design requirement. + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Container names must match exactly on both servers: + "Emby" ← both HOST1 and HOST2 + "NginxProxyManager" ← both HOST1 and HOST2 + "Mariadb-Authelia" ← both HOST1 and HOST2 + +# Share paths must match exactly on both servers: + /mnt/user/Movies ← both HOST1 and HOST2 (HOST2 has a mirror) + /mnt/user/Tv_Shows ← both HOST1 and HOST2 +# ───────────────────────────────────────────────────────────────────────────── +# If a container has a different name on one server: the script skips it +# without error. It silently does the wrong thing. You only notice when +# the container is not stopped during a sync that requires it to stop. +# +# If a share has a different path: rsync.sh aborts with "remote share missing". +# Easier to catch — but still requires renaming the share to fix. +# +# Keep names consistent and one codebase covers both servers automatically. +# Diverge and every script that touches containers or shares needs custom logic. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +## ━━━ REPOSITORY STRUCTURE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +``` +Unraid_Scripts/ +├── master.conf ← All shared configuration — edit this file +├── master_host1.conf ← HOST1-specific configuration +├── master_host2.conf ← HOST2-specific configuration +├── common.sh ← Shared library — functions used by all scripts +├── load_config.sh ← Config loader — sources all conf files +│ +├── Orchestrators/ +│ ├── array_start.sh ← Single "At Startup of Array" entry point +│ ├── daily_sync_maintenance.sh ← 1am daily window orchestrator +│ ├── weekly_sync_maintenance.sh ← Sunday 2:30am window orchestrator +│ ├── critical_sync_maintenance.sh ← Every 15 minutes — critical sync + partnership +│ ├── media_management.sh ← Permissions + cleaners + arr cleanup +│ ├── transcode_management.sh ← Transcode cleanup then manager +│ └── arrs_failed_stalled_recovery.sh ← Failed import + stalled download recovery +│ +├── Rsync/ +│ └── rsync.sh ← Core rsync script — called per share +│ +├── Failover/ +│ ├── failover.sh ← Mutual container failover — continuous loop +│ ├── failover_test.sh ← Controlled iptables failover simulation +│ └── failover_state_reset.sh ← Reset failover state manually +│ +├── Docker_Essentials/ +│ ├── docker_watchdog.sh ← Two-tier container monitor — continuous loop +│ ├── docker_daily_restart.sh ← Nightly container restarts +│ ├── docker_weekly_restart.sh ← Weekly container restarts +│ ├── docker_network_connect.sh ← Ensure networks + connections at array start +│ └── watchdog_skip_list_manager.sh ← Skip list inspection and recovery +│ +├── unRAID_Essentials/ +│ ├── system_watchdog.sh ← Three-tier system health monitor — continuous +│ ├── ramdisk_setup.sh ← Creates ramdisk + symlink at array start +│ ├── inotify_tuning.sh ← Raise inotify limits at array start +│ ├── docker_syslog_filter.sh ← Suppress veth log noise +│ ├── php_fpm_max_children.sh ← WebGUI performance tuning +│ ├── server_reboot.sh ← Graceful reboot with pre-flight warnings +│ ├── mover_stop.sh ← Stop mover cleanly with wall warning +│ ├── clear_logs.sh ← Size-threshold log cleanup +│ ├── webgui_restart.sh ← nginx → php-fpm → emhttp escalation +│ └── git_pull_execute.sh ← Pull latest scripts from Gitea +│ +├── Media/ +│ ├── media_shares_permissions.sh ← Apply permissions to media shares +│ ├── media_cleaner.sh ← Remove junk files from media shares +│ ├── lidarr_cleanup.sh ← Remove orphaned music files (HOST1 only) +│ ├── sonarr_cleanup.sh ← Remove orphaned TV files (host-aware) +│ └── radarr_cleanup.sh ← Remove orphaned movie files (host-aware) +│ +├── Transcodes/ +│ ├── transcode_manager.sh ← Ramdisk/SSD symlink management +│ └── transcode_cleanup.sh ← Remove stale segment files +│ +├── Monitors/ +│ ├── cert_monitor.sh ← SSL cert expiry via live TLS connection +│ ├── backup_verify.sh ← rsync mirror MD5 checksum verification +│ ├── smart_health.sh ← Drive SMART attribute monitoring +│ ├── zfs_memory_snapshot.sh ← ZFS health + ARC + memory report +│ ├── bandwidth_monitor.sh ← rsync transfer logging + weekly report +│ ├── weekly_health_digest.sh ← Full ecosystem health aggregation +│ ├── emby_session_report.sh ← Emby streaming usage statistics +│ ├── system_tuning_monitor.sh ← inotify + php-fpm utilisation tracking +│ └── continuous_scripts_status.sh ← Live dashboard for background processes +│ +├── Partnership/ +│ └── partnership_manage.sh ← Two-server relationship lifecycle manager +│ +└── Tools/ + ├── recreate_shares.sh ← Create share directories from cfg files + ├── bulk_permissions_repair.sh ← One-shot permission repair + ├── rsync_stop.sh ← Stop active rsync jobs cleanly + ├── user_scripts_stop.sh ← Stop running user script processes + ├── server_reboot.sh ← Graceful scheduled reboot + ├── zfs_pool_scrub.sh ← Trigger ZFS pool scrub + └── container_data_export.sh ← Export container configuration +``` + +--- + +## ━━━ TROUBLESHOOTING ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +--- + +### 🔴 SSH Connection Refused / Timeout + +```bash +# Verify SSH is enabled on the remote: +# Settings → Management Access → Secure Shell → Enabled + +# Verify Tailscale is connected: +tailscale ip -4 unRAID-Jayred365 # should return 100.x.x.x + +# Test SSH manually with the key: +ssh -i /root/.ssh/Gmer4Lfe-rsync-key root@$(tailscale ip -4 unRAID-Jayred365) "hostname" +# Expected: unRAID-Jayred365 +# If password prompted: key not authorised — recheck Step 3b +``` + +--- + +### 🔴 Pre-flight Aborts on Remote Rootfs + +```bash +# Remote rootfs above ROOTFS_WARN threshold +# Check current usage on remote: +ssh -i /root/.ssh/Gmer4Lfe-rsync-key root@[HOST2-ip] "df /" + +# Common cause: array not started, drives not mounted +# Verify array is started on HOST2 before running syncs +``` + +--- + +### 🔴 Remote Share Missing + +```bash +# Share exists locally but not on remote +# Verify the share exists on HOST2: +ssh -i /root/.ssh/Gmer4Lfe-rsync-key root@[HOST2-ip] "ls /mnt/user/" + +# If missing — create the share on HOST2 first, then run initial sync (Step 11) +# Or run recreate_shares.sh on HOST2 to create directories from HOST1's cfg files +``` + +--- + +### 🔴 Containers Not Stopping / Starting + +```bash +# Verify container names in master_host*.conf match Docker exactly — case sensitive +# Check what Docker actually calls the container: +docker ps --format "{{.Names}}" + +# Test Docker commands to remote manually: +ssh -i /root/.ssh/Gmer4Lfe-rsync-key root@[HOST2-ip] "docker ps --format '{{.Names}}'" +``` + +--- + +### 🔴 Profile Not Matching + +```bash +# Profile key = directory basename lowercased +# /mnt/user/appdata-Failover/Arrs_Stack → basename: Arrs_Stack → key: arrs_stack + +# Override explicitly if basename doesn't match a profile name: +bash rsync.sh /mnt/user/appdata-Failover/My_Stuff --profile=arrs_stack + +# Verify what profile resolved for a path: +bash rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --status +``` + +--- + +### 🔴 Script Not Found + +```bash +# Verify repo was cloned to the correct location: +ls /mnt/user/appdata/unraid_scripts/master.conf + +# Make scripts executable: find /mnt/user/appdata/unraid_scripts -name "*.sh" -exec chmod +x {} \; ``` --- -## Step 5 — Configure Master.conf +## ━━━ AVAILABLE FLAGS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -All user configuration lives in `Master.conf`. Open it and fill in your values: +All scripts support these flags. Use `--dry-run` before any live operation. ```bash -nano /mnt/user/appdata/unraid_scripts/Master.conf -``` - -### Host Configuration - -```bash -HOST1="unRAID-Gmer4Lfe" # must match Tailscale machine name exactly -HOST2="unRAID-Jayred365" - -HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" -HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" - -HOST1_EMBY_CONTAINER="Emby" -HOST1_EMBY_URL="http://localhost:8096" -HOST1_EMBY_API_KEY="your-host1-emby-api-key" # Emby Dashboard → API Keys → + New Key - -HOST2_EMBY_CONTAINER="Emby-Jayred365" -HOST2_EMBY_URL="http://localhost:8096" -HOST2_EMBY_API_KEY="your-host2-emby-api-key" -``` - -### Git / Repo - -```bash -GITEA_CONTAINER="Gitea" # exact Docker container name -GITEA_REPO_PATH="FailedProxy/Unraid_Scripts.git" -GITEA_DOMAIN="" # optional public domain fallback -TARGET_DIR="/mnt/user/appdata/unraid_scripts" -GITEA_SSH_KEY="/root/.ssh/unraid_gitea" -SSH_PORT=221 -``` - -### Orchestrators — Daily Sync Shares - -Define which shares each server owns. Each server only pushes the shares it is source of truth for — the other server mirrors and treats them as read-only. - -```bash -HOST1_DAILY_SYNC_SHARES=( - /mnt/user/Movies - /mnt/user/Tv_Shows - /mnt/user/Music - # add all HOST1-managed shares here -) - -HOST2_DAILY_SYNC_SHARES=( - /mnt/user/Anime_Shows - /mnt/user/Anime_Movies - # add all HOST2-managed shares here -) -``` - -> Never put the same share in both lists. One server is always the truth holder for each share. - -### Orchestrators — Weekly Sync Jobs - -Shares synced during the Sunday maintenance window with containers stopped both sides: - -```bash -WEEKLY_SYNC_JOBS=( - "/mnt/user/Media_Server/Emby" # full clean Emby mirror - "/mnt/user/appdata-Failover/Critical-Data" # auth stack -) -``` - ---- - -## Step 6 — Rsync Profiles - -Profiles control per-share rsync behaviour for appdata syncs. The profile key is matched automatically by the basename of the directory passed to `rsync.sh` (lowercased). Override with `--profile=name`. - -One array drives both local and remote container stops. Same container names on both servers — consistent naming is a requirement of this ecosystem. - -### Current Profiles - -| Profile | Purpose | Containers Stopped | -|---|---|---| -| `arrs_stack` | Arr databases | Sonarr, Radarr, Lidarr, Prowlarr, Bazarr, Pinchflat | -| `critical-data` | Auth stack | Mariadb, Redis, LLDAP, NPM, Authelia (delayed start) | -| `important-data` | NextCloud + Postgres | Postgres, NextCloud (delayed start) | -| `gmer4lfe` | Server-specific appdata | Organizr, UptimeKuma, VaultWarden | -| `emby` | Weekly clean sync | Emby both sides — WAL checkpointed | -| `emby-failover` | Frequent dirty sync | None — Emby stays running | - -### Two Emby Profiles - -``` -emby-failover — every 30-60min, Emby stays running: - WAL and SHM excluded — safe while Emby is active - Critical failover data only: users.db, library.db, authentication.db, config/ - Fast, small dataset — users continue watching without interruption on failover - Also used for failover writeback on handback - -emby — weekly Sunday 2:30am, both Emby instances stopped: - WAL checkpointed on shutdown — full consistent mirror - Full mirror: metadata, plugins, config all included - Minimal excludes: logs, transcodes, cache, crash files only - Cache stays warm on HOST2 all week — only reset on Sunday - emby-failover covers the critical state between weekly syncs -``` - ---- - -## Step 7 — Personal Encrypted Shares - -Personal shares can be synced to the remote server for offsite backup. ZFS encrypts at the dataset level — the remote server receives encrypted blocks and cannot read the content without your passphrase or keyfile. - -### ZFS Encryption Setup (unRAID 7) - -**Step 1 — Create an encrypted dataset:** - -1. In the unRAID UI go to **Main** → click your ZFS pool name -2. Click **+ Dataset** to create a new dataset -3. Name it — e.g. `Gmer4Lfe-Personal` -4. Enable **Encryption** → set your passphrase - > ⚠️ Write your passphrase down — if lost, data is unrecoverable - -**Step 2 — Create the share:** - -1. Go to **Settings → Shares → Add Share** -2. Set the share path to your new encrypted dataset -3. Set **Use cache:** `Only` — keeps data on ZFS pool, not array - -**Step 3 — Verify encryption is active before syncing:** - -```bash -zfs get encryption poolname/Gmer4Lfe-Personal -# Should show: encryption aes-256-gcm -``` - -**Step 4 — Auto-unlock on boot (keyfile approach — optional):** - -```bash -# Create keyfile — on HOST1 only, never sync this file -dd if=/dev/urandom bs=32 count=1 | base64 > /root/.zfs-keys/personal.key -chmod 600 /root/.zfs-keys/personal.key - -# Set dataset to use keyfile -zfs change-key -o keylocation=file:///root/.zfs-keys/personal.key \ - -o keyformat=raw poolname/Gmer4Lfe-Personal - -# Add to array start (via array_start.sh or User Scripts): -zfs load-key poolname/Gmer4Lfe-Personal -zfs mount poolname/Gmer4Lfe-Personal -``` - -**Manual unlock alternative (most secure):** - -```bash -zfs load-key poolname/Gmer4Lfe-Personal # prompts for passphrase -zfs mount poolname/Gmer4Lfe-Personal -``` - -**Step 5 — Add to Master.conf:** - -```bash -HOST1_PERSONAL_SHARES=( - /mnt/user/Gmer4Lfe-Personal -) -``` - -Personal shares sync automatically with the daily media share sync in `daily_sync_maintenance.sh`. The remote server receives encrypted blocks — content is unreadable without your key. - ---- - -## Step 8 — Set Up User Scripts - -The ecosystem uses a single orchestrator entry for array startup plus a small number of scheduled scripts. - -### At Startup of Array - -Create one script entry named `array start`: - -```bash -#!/bin/bash -bash /mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh -``` - -Set schedule to: **At Startup of Array** - -This single entry launches everything configured in `ARRAY_START_SCRIPTS` in `Master.conf`: -- `ramdisk_setup.sh` — creates ramdisk before Emby starts -- `docker_syslog_filter.sh` — suppresses veth log noise -- `php_fpm_max_children.sh` — WebGUI tuning -- `docker_network_connect.sh` — connects containers to extra networks -- `system_watchdog.sh` — continuous system health monitor -- `docker_watchdog.sh` — continuous container health monitor -- `failover.sh` — continuous mutual failover - -### Cron Schedules - -| Script | Schedule | Purpose | -|---|---|---| -| `transcode_management.sh` | `*/3 * * * *` | Transcode cleanup + manager | -| `arrs_failed_stalled_recovery.sh` | `0 */6 * * *` | Blocklist + re-search failed imports | -| `rsync.sh ... --profile=emby-failover` | `*/30 * * * *` | Emby dirty sync | -| `daily_sync_maintenance.sh` | `0 1 * * *` | Full daily maintenance window | -| `weekly_sync_maintenance.sh` | `30 2 * * 0` | Weekly sync + updates + restarts | -| `weekly_health_digest.sh` | `0 8 * * 6` | Saturday morning health digest | - -### Emby Failover Dirty Sync - -Create a separate script entry named `rsync emby failover`: - -```bash -#!/bin/bash -bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ - /mnt/user/Media_Server/Emby --profile=emby-failover -``` - -Set schedule to: `*/30 * * * *` - -### Appdata Profile Syncs - -Create one entry per appdata profile you want on a schedule: - -```bash -#!/bin/bash -bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ - /mnt/user/appdata-Failover/Arrs_Stack -``` - -| Profile | Recommended Schedule | -|---|---| -| `Arrs_Stack` | Every 12-24 hours | -| `Important-Data` | Every 6-12 hours | -| `Gmer4Lfe` | Daily or weekly | -| `emby` | Via `weekly_sync_maintenance.sh` only — do NOT schedule separately | -| `Critical-Data` | Via `weekly_sync_maintenance.sh` only — do NOT schedule separately | - -> Set all scripts to run as **Background Task** — output streams correctly rather than buffering in the browser. - ---- - -## Step 9 — Verify the Setup - -Before letting scheduled jobs run, test manually from the terminal on HOST1: - -```bash -# Test a single appdata profile sync -bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ - /mnt/user/appdata-Failover/Arrs_Stack --dry-run --log - -# Test the daily sync orchestrator -bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh --dry-run -``` - -A healthy run will show: - -``` -━━━ ⚙️ Setup ━━━ -ℹ️ Host: unRAID-Gmer4Lfe → unRAID-Jayred365 -ℹ️ Remote IP: 100.x.x.x - -━━━ 🛡️ Pre-flight Checks ━━━ -✅ Remote reachable -✅ Remote rootfs: 12% (threshold: 75%) -✅ All pre-flight checks passed -``` - -If any pre-flight check fails the script aborts with a clear error before touching anything. - ---- - -## Step 10 — Secondary Server Initial Sync - -If setting up HOST2 from scratch with empty shares: - -1. Complete Steps 1–8 on HOST2 -2. Start the array and create your shares in the unRAID UI -3. Run the share recreation tool to create disk directories from cfg files: - -```bash -bash /mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh -``` - -4. Run the initial push from HOST1 — this populates HOST2's empty shares: - -```bash -bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh --log -``` - -5. Once complete, scheduled runs take over automatically. - ---- - -## Naming Consistency — Required - -The ecosystem is built on the assumption that containers and shares have identical names on both servers. This is not optional — it is what makes one codebase work on both servers without modification. - -``` -Container names must match exactly: - Emby ← HOST1 and HOST2 - NginxProxyManager ← HOST1 and HOST2 - Mariadb-Authelia ← HOST1 and HOST2 - -Share paths must match exactly: - /mnt/user/Movies ← HOST1 and HOST2 - /mnt/user/Tv_Shows ← HOST1 and HOST2 -``` - -If a container or share has a different name on one server the script skips it gracefully — but it will not do what you expect. Diverge from consistent naming and every script that touches containers or shares needs custom logic for each server. Keep naming consistent and one codebase covers both servers automatically. - ---- - -## Troubleshooting - -### SSH connection refused -- Verify SSH is enabled (Step 2) -- Confirm the correct key is referenced in `Master.conf` -- Test Tailscale: `tailscale ip -4 HOSTNAME` - -### Pre-flight aborts on rootfs -- Remote rootfs above `ROOTFS_WARN` threshold -- Check remote array is started and drives are mounted -- Run `df /` on the remote to see current usage - -### Pre-flight aborts on empty share -- Share exists but has no content — drives may not be mounted -- Run `recreate_shares.sh` if setting up fresh - -### Containers not stopping/starting -- Verify container names in `Master.conf` match Docker exactly — case sensitive -- Test manually: `ssh -i /root/.ssh/KEY root@REMOTE_IP "docker ps"` - -### Script not found -- Verify repo was cloned to `/mnt/user/appdata/unraid_scripts/` -- Make scripts executable: `find /mnt/user/appdata/unraid_scripts -name "*.sh" -exec chmod +x {} \;` - -### Profile not matching -- Profile key is matched by directory basename lowercased -- `/mnt/user/appdata-Failover/Arrs_Stack` → basename `Arrs_Stack` → key `arrs_stack` -- Override with `--profile=name` if basename doesn't match - ---- - -## Available Flags - -All scripts support: - -| Flag | Description | -|---|---| -| `--dry-run` | Run without making any changes | -| `--log` | Enable verbose logging output | -| `--no-log` | Disable logging | -| `--status` | Print resolved configuration and exit | - -```bash -# Preview what would be synced -bash rsync.sh /mnt/user/Movies --dry-run --log - -# Check resolved profile settings -bash rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --status - -# Test daily orchestrator without changes -bash daily_sync_maintenance.sh --dry-run -``` - ---- - -## Repository Structure - -``` -Unraid_Scripts/ -├── Master.conf # All user configuration — edit this file only -├── common.sh # Shared library — functions used by all scripts -│ -├── Orchestrators/ -│ ├── array_start.sh # Single array-start entry point -│ ├── daily_sync_maintenance.sh # Daily maintenance window orchestrator -│ ├── weekly_sync_maintenance.sh # Weekly maintenance window orchestrator -│ ├── media_management.sh # Permissions + cleaners + arr cleanup -│ └── transcode_management.sh # Transcode cleanup + manager -│ -├── Rsync/ -│ └── rsync.sh # Core rsync script — called per share -│ -├── Failover/ -│ ├── failover.sh # Mutual container failover — continuous loop -│ ├── failover_test.sh # Controlled failover simulation -│ └── failover_state_reset.sh # Reset failover state manually -│ -├── Docker_Essentials/ -│ ├── docker_watchdog.sh # Two-tier container monitor — continuous loop -│ ├── docker_daily_restart.sh # Daily container restarts -│ ├── docker_weekly_restart.sh # Weekly container restarts -│ └── docker_network_connect.sh # Connect containers to extra networks -│ -├── unRAID_Essentials/ -│ ├── system_watchdog.sh # System health monitor — continuous loop -│ ├── ramdisk_setup.sh # Creates ramdisk + symlink at array start -│ ├── docker_syslog_filter.sh # Suppress veth log noise -│ ├── php_fpm_max_children.sh # WebGUI performance tuning -│ ├── server_reboot.sh # Graceful scheduled reboot -│ ├── mover_stop.sh # Stop mover cleanly -│ ├── clear_logs.sh # Weekly log cleanup -│ ├── webgui_restart.sh # nginx + emhttp restart escalation -│ └── git_pull_execute.sh # Pull latest scripts from Gitea -│ -├── Media/ -│ ├── media_shares_permissions.sh # Apply permissions to media shares -│ ├── media_cleaner.sh # Remove junk files from media shares -│ ├── lidarr_cleanup.sh # Remove orphaned music files -│ ├── sonarr_cleanup.sh # Remove orphaned TV files -│ ├── radarr_cleanup.sh # Remove orphaned movie files -│ └── arrs_failed_stalled_recovery.sh # Blocklist + re-search failed imports -│ -├── Transcodes/ -│ ├── transcode_manager.sh # Symlink direction management -│ ├── transcode_cleanup.sh # Remove old inactive transcode files -│ └── ramdisk_setup.sh # (also in unRAID_Essentials — symlinked) -│ -├── Monitors/ -│ ├── cert_monitor.sh # SSL certificate expiry monitoring -│ ├── backup_verify.sh # Checksum verification against remote -│ ├── smart_health.sh # Drive SMART attribute monitoring -│ ├── zfs_memory_snapshot.sh # Weekly ZFS health + memory report -│ ├── bandwidth_monitor.sh # Rsync transfer logging + weekly summary -│ ├── weekly_health_digest.sh # Aggregated health digest email -│ ├── emby_session_report.sh # Weekly Emby usage statistics -│ └── emby_database_repair.sh # Emby SQLite database repair -│ -└── Tools/ - ├── recreate_shares.sh # Recreate share directories from cfg files - ├── bulk_permissions_repair.sh # One-shot permission repair - ├── watchdog_skip_list_manager.sh # Manage docker watchdog skip list - ├── rsync_stop.sh # Stop active rsync jobs cleanly - ├── user_scripts_stop.sh # Stop running user scripts - └── container_data_export.sh # Export container configuration +# ───────────────────────────────────────────────────────────────────────────── + --dry-run run without making any changes — pre-flight still runs ✅ + --log verbose output — show every decision made + --status show resolved configuration and exit — no rsync, no sync + --no-log suppress verbose output (some scripts) + +# Examples: +bash rsync.sh /mnt/user/Movies --dry-run --log # preview a media sync +bash rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --status # check profile resolution +bash daily_sync_maintenance.sh --dry-run # preview full daily window +bash docker_watchdog.sh --status # check watchdog state +# ───────────────────────────────────────────────────────────────────────────── ``` \ No newline at end of file diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index 3894c4a..5a27efc 100644 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -1,30 +1,71 @@ #!/bin/bash -# ---------------------------------------------------------------------------------------------- -# --------------------------------- Rsync Core Script ------------------------------------------ -# ---------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= Rsync Core Script ========================================== +# ============================================================================================== # Core rsync script — called per share or per appdata profile. +# Called by orchestrators (daily/weekly/critical sync) and directly for manual syncs. +# +# ── PROFILE SYSTEM ──────────────────────────────────────────────────────────────────────────── # Profile is inferred from the directory basename (lowercased). -# If no profile match is found all settings fall through to global defaults in Master.conf. +# Override with --profile=name for explicit profile selection. +# If no profile match found → all settings fall back to global defaults in master.conf. # -# After each successful sync, logs the transfer to bandwidth_monitor.sh for weekly reporting. -# Log entry: date | time | profile | duration | status +# Profiles define: +# PROFILE_RSYNC_OPTS — rsync flags (does NOT inherit DEFAULT_RSYNC_OPTS) +# PROFILE_BW_LIMIT — bandwidth limit in KB/s +# PROFILE_RETRY_COUNT — retry attempts before giving up +# PROFILE_SLEEP — seconds between retry attempts +# PROFILE_CRITICAL_CONTAINER_NAMES — containers stopped both sides before sync +# PROFILE_DELAYED_CONTAINERS — containers needing delay before starting after sync +# PROFILE_CONTAINER_DELAY — seconds before starting delayed containers +# PROFILE_EXCLUDE_DIRS — paths excluded from transfer +# PROFILE_REMOTE_RESTART_CONTAINERS — containers restarted on remote after dirty sync +# (critical-failover, emby-failover profiles) +# Was running → restart. Was stopped → leave stopped. # -# Usage: -# rsync.sh /mnt/user/Movies — media share, uses global defaults -# rsync.sh /mnt/user/appdata-Failover/Arrs_Stack — matched to [arrs_stack] profile +# ── RSYNC ENABLE/DISABLE ────────────────────────────────────────────────────────────────────── +# Two-tier toggle system — checked at entry: +# Tier 1: RSYNC_ENABLED=false → all rsync stops +# Tier 2: Per-orchestrator flag (DAILY_RSYNC_ENABLED etc.) — checked by caller +# Direct calls to rsync.sh only check Tier 1 +# +# ── BANDWIDTH LOGGING ───────────────────────────────────────────────────────────────────────── +# After each sync logs to bandwidth_monitor.sh --log-transfer: +# profile | duration_seconds | status | bytes_transferred +# Bytes captured from rsync --stats output — version-proof parsing. +# bandwidth_monitor.sh flags syncs exceeding BANDWIDTH_WARN_GB. +# +# ── DIRTY SYNC REMOTE RESTART ───────────────────────────────────────────────────────────────── +# Profiles using dirty sync (critical-failover, emby-failover) define +# PROFILE_REMOTE_RESTART_CONTAINERS — containers restarted on remote after sync completes. +# This ensures the remote picks up config changes synced during the dirty window. +# Was running → restart. Was stopped → leave stopped. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# check_rsync_enabled() — Tier 1 gate before any operation +# check_unraid_version_parity — refuses if servers on incompatible unRAID versions +# check_remote_docker_daemon — verifies remote daemon before container operations +# check_local_disk_temps() — temp check before transfer (exit 1=skip, 2=abort all) +# check_connectivity() — verifies remote reachable +# check_remote_rootfs() — aborts if remote rootfs nearly full +# check_remote_share() — aborts if target directory missing or empty +# check_remote_disks() — verifies all backing disks online on remote +# acquire_rsync_lock() — per-profile lock + global concurrent limit +# validate_unraid_cmd — notify validated before use +# Silent by default — only failures produce visible output +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# rsync.sh /mnt/user/Movies — media share, global defaults +# rsync.sh /mnt/user/appdata-Failover/Arrs_Stack — matched to [arrs_stack] profile # rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --dry-run --log -# ---------------- User Variables, Please adjust in Master.conf as needed ---------------------- -# ---------------------------------------------------------------------------------------------- +# rsync.sh /mnt/user/appdata-Failover/Critical-Data --profile=critical-failover +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" -# ----------------------------------------------------------------------------------------------- -# Separate the positional directory argument from flag/key=value args. -# Optional --profile=name overrides the basename profile inference. -# ----------------------------------------------------------------------------------------------- +# ── Separate positional directory arg from flags ─────────────────────────────────────────────── DIRECTORY="" PROFILE_OVERRIDE="" RAW_ARGS=() @@ -32,185 +73,272 @@ RAW_ARGS=() for ARG in "$@"; do case "$ARG" in --profile=*) PROFILE_OVERRIDE="${ARG#--profile=}" ;; - --*|*=*) RAW_ARGS+=("$ARG") ;; - *) [[ -z "$DIRECTORY" ]] && DIRECTORY="$ARG" || RAW_ARGS+=("$ARG") ;; + --*|*=*) RAW_ARGS+=("$ARG") ;; + *) [[ -z "$DIRECTORY" ]] && DIRECTORY="$ARG" || RAW_ARGS+=("$ARG") ;; esac done parse_args "${RAW_ARGS[@]}" -[[ -z "$DIRECTORY" ]] && error "No directory specified. Usage: rsync.sh [--dry-run] [--log] [--profile=name]" && exit 1 +[[ -z "$DIRECTORY" ]] && { + error "No directory specified" + error "Usage: rsync.sh [--dry-run] [--log] [--profile=name]" + exit 1 +} -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" detect_hosts -# Tier 1 global gate — check before doing anything -# Tier 2 (per-orchestrator) is handled by the calling orchestrator -# Direct calls to rsync.sh only check Tier 1 +# Tier 1 global gate — Tier 2 (per-orchestrator) checked by caller if ! check_rsync_enabled; then warn "RSYNC_ENABLED=false — exiting cleanly" exit 0 fi + resolve_remote_ip -# ----------------------------------------------------------------------------------------------- -# Profile inference — basename of directory lowercased -# Optional --profile=name overrides basename inference -# ----------------------------------------------------------------------------------------------- -echo "" +# ── Profile inference ───────────────────────────────────────────────────────────────────────── if [[ -n "$PROFILE_OVERRIDE" ]]; then PROFILE_NAME="$PROFILE_OVERRIDE" - info "$ICON_GEAR Profile override: $PROFILE_NAME" + log "Profile override: $PROFILE_NAME" else PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]') - info "$ICON_GEAR Loading profile: $PROFILE_NAME" + log "Profile inferred: $PROFILE_NAME" fi # Acquire per-profile lock and check global concurrent limit acquire_rsync_lock "$PROFILE_NAME" -# Scalar overrides +# ── Load profile settings ───────────────────────────────────────────────────────────────────── BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT} RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT} SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP} CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY} -# Array overrides read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]:-}" read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]:-}" read -r -a EXCLUDE_DIRS <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-}" +read -r -a REMOTE_RESTART_CONTAINERS <<< "${PROFILE_REMOTE_RESTART_CONTAINERS[$PROFILE_NAME]:-}" -# Local and remote use the same container list — same naming scheme on both servers +# Local containers use same names as remote (mirrored naming scheme) LOCAL_CRITICAL_CONTAINER_NAMES=("${CRITICAL_CONTAINER_NAMES[@]}") [[ "$SHOW_STATUS" == true ]] && show_status && exit 0 +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SHIELD Pre-flight Checks ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Pre-flight Checks ━━━ +# ============================================================================================== echo "" -echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━" +echo "━━━ $ICON_SHIELD Pre-flight ━━━" -# Disk temp check — before touching remote or moving any data -# Returns: 0=OK 1=warn(skip this profile) 2=crit(abort all remaining) +# Disk temp — before touching remote or moving data +# Exit 1 = skip this profile | Exit 2 = abort all remaining profiles check_local_disk_temps TEMP_RESULT=$? if [[ "$TEMP_RESULT" -eq 2 ]]; then - error "Drive temps CRITICAL — aborting sync for all remaining profiles" - exit 2 # caller (daily_sync_maintenance.sh) sees exit 2 → stops all syncs + error "Drive temps CRITICAL — aborting all remaining syncs" + exit 2 elif [[ "$TEMP_RESULT" -eq 1 ]]; then - warn "Drive temps too high — skipping profile [$PROFILE_NAME]" - exit 1 # caller sees exit 1 → skips this profile, continues to next + warn "Drive temps high — skipping profile [$PROFILE_NAME]" + exit 1 else - success "Drive temps OK — $TEMP_CHECK_RESULT" + log "Drive temps OK — $TEMP_CHECK_RESULT" fi +# Version parity — refuse if servers on incompatible unRAID versions +check_unraid_version_parity || exit 1 + check_connectivity check_remote_rootfs check_remote_share "$DIRECTORY" check_remote_disks "$DIRECTORY" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_STOP $ICON_CONTAINERS Containers ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_STOP $ICON_CONTAINERS Containers ━━━" +# Remote Docker daemon — check before attempting container operations +if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]] || [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then + check_remote_docker_daemon || { + warn "Remote Docker daemon unresponsive — skipping container operations" + warn "Proceeding with rsync only — containers will not be stopped or restarted" + CRITICAL_CONTAINER_NAMES=() + LOCAL_CRITICAL_CONTAINER_NAMES=() + REMOTE_RESTART_CONTAINERS=() + } +fi -# Stop local containers first — flush local databases before pushing -stop_local_containers +# ============================================================================================== +# ━━━ Stop Containers ━━━ +# ============================================================================================== +if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then + echo "" + echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━" + # Local first — flush local databases before pushing + stop_local_containers + # Remote next — prevent writes while receiving + stop_containers +fi -# Stop remote containers — prevent writes while receiving -stop_containers - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SYNC Transfer ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Transfer ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_SYNC Transfer ━━━" -echo "$ICON_RUN Source: $DIRECTORY" -echo "$ICON_NET Remote: $REMOTE_SERVER:$DIRECTORY" -echo "$ICON_GEAR Profile: $PROFILE_NAME" +echo "$ICON_RUN Source: $DIRECTORY" +echo "$ICON_NET Remote: $REMOTE_SERVER:$DIRECTORY" +echo "$ICON_GEAR Profile: $PROFILE_NAME" +echo "$ICON_HOST Identity: $MY_ID → $REMOTE_ID" echo "" get_rsync_opts # Append profile excludes -for ex in "${EXCLUDE_DIRS[@]}"; do +for ex in "${EXCLUDE_DIRS[@]:-}"; do [[ -n "$ex" ]] && RSYNC_OPTS+=(--exclude="$ex") done -[[ "$DRY_RUN" == true ]] && RSYNC_OPTS+=("--dry-run") && warn "DRY RUN — no changes will be made" +# Add --stats to capture bytes transferred for bandwidth logging +RSYNC_OPTS+=(--stats) + +[[ "$DRY_RUN" == true ]] && RSYNC_OPTS+=("--dry-run") START=$(date +%s) RSYNC_SUCCESS=false +BYTES_TRANSFERRED=0 +ATTEMPT=0 -for i in $(seq 1 "$RETRY_COUNT"); do - info "$ICON_RETRY Attempt $i of $RETRY_COUNT..." +for (( ATTEMPT=1; ATTEMPT<=RETRY_COUNT; ATTEMPT++ )); do + log "$ICON_RETRY Attempt $ATTEMPT of $RETRY_COUNT..." - if rsync "${RSYNC_OPTS[@]}" \ + RSYNC_OUTPUT=$(rsync "${RSYNC_OPTS[@]}" \ -e "ssh -i $SSH_KEY -T -o Compression=no -o IPQoS=throughput" \ - "$DIRECTORY" "root@${REMOTE_SERVER}:$(dirname "$DIRECTORY")/"; then - echo "$ICON_DONE Rsync complete" + "$DIRECTORY" "root@${REMOTE_SERVER}:$(dirname "$DIRECTORY")/" 2>&1) + + RSYNC_EXIT=$? + + if [[ "$RSYNC_EXIT" -eq 0 ]]; then + # Parse bytes transferred from --stats output + BYTES_TRANSFERRED=$(echo "$RSYNC_OUTPUT" | \ + awk '/Total transferred file size:/{gsub(/,/,"",$NF); gsub(/[^0-9]/,"",$NF); print $NF+0}') + BYTES_TRANSFERRED="${BYTES_TRANSFERRED:-0}" + + log "$ICON_DONE Rsync complete — $BYTES_TRANSFERRED bytes transferred" RSYNC_SUCCESS=true break else - warn "$ICON_RETRY Rsync failed (attempt $i/$RETRY_COUNT)" - [[ "$i" -lt "$RETRY_COUNT" ]] && info "Retrying in ${SLEEP}s..." && sleep "$SLEEP" + warn "$ICON_RETRY Rsync failed (attempt $ATTEMPT/$RETRY_COUNT)" + log "Exit code: $RSYNC_EXIT" + if [[ "$ATTEMPT" -lt "$RETRY_COUNT" ]]; then + log "Retrying in ${SLEEP}s..." + sleep "$SLEEP" + fi fi done -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_START $ICON_CONTAINERS Containers ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_START $ICON_CONTAINERS Containers ━━━" +# ============================================================================================== +# ━━━ Start Containers ━━━ +# ============================================================================================== +if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then + echo "" + echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━" + # Remote first — can be coming up while local restarts + start_containers + # Local next + start_local_containers +fi -# Start remote containers first — they can be coming up while local restarts -start_containers +# ============================================================================================== +# ━━━ Remote Restart (dirty sync profiles) ━━━ +# ============================================================================================== +# For dirty sync profiles (critical-failover, emby-failover) — restart containers on remote +# that were running before sync so they pick up config changes from the dirty sync window. +# Was running → restart. Was stopped → leave stopped. +if [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then + echo "" + echo "━━━ $ICON_START $ICON_CONTAINERS Remote Restart (post dirty sync) ━━━" + log "Restarting configured containers on $REMOTE_SERVER_NAME..." -# Start local containers -start_local_containers + for container in "${REMOTE_RESTART_CONTAINERS[@]}"; do + [[ -z "$container" ]] && continue + + # Check if container was running before sync (still tracked via RUNNING_CONTAINERS) + WAS_RUNNING=false + for prev in "${RUNNING_CONTAINERS[@]:-}"; do + [[ "$prev" == "$container" ]] && WAS_RUNNING=true && break + done + + if [[ "$WAS_RUNNING" == false ]]; then + # Not in stop list — check current remote state + REMOTE_STATUS=$(timeout 15 ssh -i "$SSH_KEY" \ + -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ + "docker inspect -f '{{.State.Running}}' $container 2>/dev/null" 2>/dev/null) + [[ "$REMOTE_STATUS" != "true" ]] && \ + log "$container not running on $REMOTE_SERVER_NAME — skipping remote restart" && \ + continue + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would restart $container on $REMOTE_SERVER_NAME" + continue + fi + + timeout 15 ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ + "docker restart $container" >/dev/null 2>&1 && \ + log "$ICON_STARTED $container restarted on $REMOTE_SERVER_NAME ✅" || \ + warn "Failed to restart $container on $REMOTE_SERVER_NAME" + done +fi END=$(date +%s) -DURATION=$((END - START)) +DURATION=$(( END - START )) -# ----------------------------------------------------------------------------------------------- -# Log transfer to bandwidth monitor — only on successful non-dry-run syncs -# Reliable format: date|time|profile|duration|status -# Does not parse rsync output — version-proof and always works -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Bandwidth Logging ━━━ +# ============================================================================================== +# Logs to bandwidth_monitor.sh — new format includes bytes transferred and warn flag. +# Only logs on actual runs (not dry-run) and only when bandwidth_monitor.sh exists. BANDWIDTH_MONITOR="$SCRIPT_DIR/../Monitors/bandwidth_monitor.sh" if [[ "$DRY_RUN" == false ]] && [[ -f "$BANDWIDTH_MONITOR" ]]; then STATUS="success" [[ "$RSYNC_SUCCESS" == false ]] && STATUS="failed" - bash "$BANDWIDTH_MONITOR" --log-transfer "$PROFILE_NAME" "$DURATION" "$STATUS" - log "$ICON_BANDWIDTH Transfer logged to bandwidth monitor" + bash "$BANDWIDTH_MONITOR" --log-transfer \ + "$PROFILE_NAME" "$DURATION" "$STATUS" "$BYTES_TRANSFERRED" + log "$ICON_BANDWIDTH Transfer logged to bandwidth monitor ($BYTES_TRANSFERRED bytes)" fi -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== # ━━━ Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== echo "" -echo "━━━━━ $ICON_SUMMARY SUMMARY ━━━━━" -echo "$ICON_RUN Directory: $DIRECTORY" -echo "$ICON_GEAR Profile: $PROFILE_NAME" -echo "$ICON_DISK Disk check: $([[ "$SKIP_DISK_CHECK" == "true" ]] && echo "skipped (ZFS pool)" || echo "passed")" -echo "$ICON_TIME Duration: $(format_duration $DURATION)" +echo "━━━━━ $ICON_SUMMARY RSYNC SUMMARY ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_RUN Directory: $DIRECTORY" +echo "$ICON_GEAR Profile: $PROFILE_NAME" +echo "$ICON_TIME Duration: $(format_duration $DURATION)" +[[ "$BYTES_TRANSFERRED" -gt 0 ]] && \ + echo "$ICON_BANDWIDTH Transferred: $(awk "BEGIN {printf \"%.2fGB\", $BYTES_TRANSFERRED / 1073741824}")" -if [[ "$RSYNC_SUCCESS" == true ]]; then - echo "$ICON_DONE Status: $ICON_SUCCESS DONE" - notify "Rsync complete — $DIRECTORY ($PROFILE_NAME) in $(format_duration $DURATION)" "Rsync" "normal" +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — no changes made" +elif [[ "$RSYNC_SUCCESS" == true ]]; then + log "$ICON_DONE Status: $ICON_SUCCESS DONE" else - echo "$ICON_ERROR Status: $ICON_ERROR FAILED after $RETRY_COUNT attempts" - notify "Rsync failed — $DIRECTORY ($PROFILE_NAME) after $RETRY_COUNT attempts" "Rsync" "warning" + echo "$ICON_ERROR Status: FAILED after $RETRY_COUNT attempts" + notify "Rsync FAILED — $DIRECTORY ($PROFILE_NAME) after $RETRY_COUNT attempts on $(hostname)" \ + "Rsync" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" -[[ "$RSYNC_SUCCESS" == false ]] && exit 1 +[[ "$RSYNC_SUCCESS" == false ]] && [[ "$DRY_RUN" == false ]] && exit 1 exit 0 \ No newline at end of file diff --git a/Tools/README-Tools.md b/Tools/README-Tools.md index 5e01367..0c6009e 100644 --- a/Tools/README-Tools.md +++ b/Tools/README-Tools.md @@ -1,262 +1,834 @@ -# Tools +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# 🔧 TOOLS +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Utility scripts for specific operational situations — recovery, repair, migration, and one-time tasks that don't fit the scheduled maintenance model of the other folders. +**Situational utilities — run when something needs fixing, not on a schedule.** +Recovery, repair, migration, cleanup, and one-time tasks that don't fit the scheduled +maintenance model. These scripts sit ready for the moment you actually need them. ``` -unRAID_Essentials/ — regular system maintenance, scheduled -Docker_Essentials/ — regular container management, scheduled -Monitors/ — regular health reporting, scheduled -Tools/ — situational utilities, run when needed +unRAID_Essentials/ ← regular system maintenance — scheduled +Docker_Essentials/ ← regular container management — scheduled +Monitors/ ← regular health reporting — scheduled +Orchestrators/ ← regular maintenance windows — scheduled +Tools/ ← situational utilities — run when needed +``` + +> **None of these scripts run on a schedule.** A script belongs here when it solves +> a specific operational situation rather than ongoing maintenance — something you run +> in response to a problem, a planned migration, or a one-time task. Having a dedicated +> folder keeps the other folders clean and makes it obvious what runs routinely vs what +> runs situationally. + +--- + +## ━━━ THE PROBLEM THAT BUILT THIS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Every tool here exists because a specific situation arose that required bash commands +to resolve — and that situation is guaranteed to arise again. + +--- + +### 🔴 Failover State Stuck After Testing + +Run `failover_test.sh`, something goes wrong mid-test, script exits uncleanly. State +file shows `FAILOVER`. `failover.sh` resumes and reads FAILOVER — starts containers +it shouldn't start, makes decisions based on a state that doesn't reflect reality. +Or the test completed but handback didn't finish — state is partially reset. + +Manual recovery: edit the state file by hand? Know the exact format? Know which +fields to reset? At 2am after a failed test, none of that is obvious. + +The tool: `failover_state_reset.sh` — one command, shows you the current state before +asking for confirmation, resets cleanly to NORMAL, explains exactly what it changed. + +--- + +### 🔴 Container Stuck on Watchdog Skip List After Fixing the Problem + +Authelia hit the restart loop limit — three restarts in an hour — went on the skip list. +You fixed the underlying database issue. But the watchdog still isn't monitoring it +because it's on the skip list and you don't know where that file lives or what format +it's in. You restart Authelia manually, it runs fine, but the watchdog has no idea +it recovered and still thinks it's broken. + +The tool: `watchdog_skip_list_manager.sh` — shows the skip list, shows which containers +are stopped vs running, clears specific containers with a single command, clears restart +history so the loop protection window starts fresh. + +--- + +### 🔴 Emby Crashing With No Clear Cause After a Power Cut + +Server lost power with Emby running. Emby comes back up, runs for 20 minutes, crashes. +Comes back up, crashes again. Logs show database errors. Which database? library.db? +users.db? authentication.db? They're all SQLite, they all need a different recovery +approach, and the errors aren't always obvious from the log output alone. + +The tool: `emby_database_repair.sh` — stops Emby, runs `PRAGMA integrity_check` on +every database, reports per database what's clean and what's corrupted with specific +guidance on what to do about each one. + +--- + +### 🔴 Files Owned by Root After an Admin Copy + +`scp` a file from another machine directly into a media share. File arrives as +`root:root`. Radarr tries to move it and fails — permission denied. The daily +permissions script won't run for another 20 hours. Running the full +`media_shares_permissions.sh` on the whole share takes 30 minutes on a large library +just to fix one directory. + +The tool: `bulk_permissions_repair.sh` — takes a specific path or list of paths, +applies correct ownership and permissions in seconds, done. + +--- + +### 🔴 No Way to Back Up a Container Before a Risky Update + +Container has a major version update. The changelog says "database migration — no +rollback." You want a point-in-time backup before you proceed. But the container's +appdata is scattered across dozens of files and `cp -r` while it's running produces +an inconsistent backup. + +The tool: `container_data_export.sh` — stops the container cleanly, archives the +entire appdata directory to a timestamped `.tar.gz`, verifies the archive integrity, +restarts the container. The backup is valid and complete before anything else happens. +If the update goes wrong you have a clean restore point. + +--- + +### 🔴 Fresh HOST2 Has Shares Configured But Directories Missing + +Fresh install on HOST2. Restored `/boot/config/shares/` cfg files from backup. Array +starts. Shares show in the UI. But the actual `/mnt/diskN/sharename` directories don't +exist on the individual disks — unRAID created the share definitions but not the +directories. rsync.sh tries to write, finds the path doesn't exist, aborts. + +The tool: `recreate_shares.sh` — reads every `.cfg` file from `/boot/config/shares/`, +parses the `shareInclude` list, creates the correct directory on each included disk. +Run once after fresh setup, directories exist, rsync works. + +--- + +## ━━━ SCRIPTS AT A GLANCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +| Script | What It Fixes | When to Run | +|--------|--------------|-------------| +| `failover_state_reset.sh` | State file stuck in FAILOVER after test or failed handback | After failover testing or manual intervention | +| `watchdog_skip_list_manager.sh` | Container stuck on watchdog skip list | After fixing a container that hit restart loop limit | +| `bulk_permissions_repair.sh` | Files owned by wrong user after admin copy or bad container | When arr operations fail due to permissions | +| `container_data_export.sh` | Need a clean backup before a risky container update | Before major updates, migrations, or removals | +| `emby_database_repair.sh` | Emby crashing with database errors after power loss | When Emby logs show corruption or repeated crashes | +| `zfs_pool_scrub.sh` | Verify ZFS pool integrity — catch silent corruption | Monthly, or after any disk/power event | +| `recreate_shares.sh` | Share directories missing on fresh install or rebuild | After fresh unRAID install or disk replacement | +| `rsync_stop.sh` | rsync stuck or needs emergency stop | When rsync is running and must be stopped cleanly | +| `user_scripts_stop.sh` | User Scripts running mid-cycle and need stopping | Before planned reboots, emergency stop | +| `server_reboot.sh` | Graceful reboot with pre-flight warnings and clean shutdown | Planned maintenance reboots | + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔀 failover_state_reset.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Resets the failover state file to NORMAL and clears all tier flags. State file only — +does NOT start or stop any containers. + +> Full documentation in `README-Failover.md` — `failover_state_reset.sh` section. +> This entry is a quick reference. + +--- + +### ── When to Use ────────────────────────────────────────────────────────────── + +``` +After failover_test.sh didn't complete cleanly + → state left in FAILOVER but containers are actually back to normal + +After a failed handback + → state shows FAILOVER but remote is back up and containers are split + +After killing failover.sh directly (not via User Scripts Abort) + → state is unknown, cycle was interrupted mid-operation + +After a dev/debug session + → state left in a non-NORMAL state from testing ``` --- -## What Belongs Here - -A script belongs in Tools when it: - -- Solves a specific operational problem rather than ongoing maintenance -- Is run manually in response to a situation rather than on a schedule -- Is used rarely — recovery scenarios, repairs, migrations, initial setup -- Would be dangerous or meaningless to run routinely -- Doesn't fit cleanly into any of the other folders - -Tools scripts are not scheduled. They sit here ready for when you need them. - ---- - -## Scripts - -### `failover_state_reset.sh` - -Resets the failover state file to NORMAL manually. +### ── Verify Before Resetting ──────────────────────────────────────────────── ```bash -/mnt/user/appdata/unraid_scripts/Tools/failover_state_reset.sh --status -/mnt/user/appdata/unraid_scripts/Tools/failover_state_reset.sh --dry-run -/mnt/user/appdata/unraid_scripts/Tools/failover_state_reset.sh +# ───────────────────────────────────────────────────────────────────────────── +# Verify reality matches what you're about to declare as NORMAL: +# +# Right containers on right server? +# DDNS pointing correctly? nslookup Gmer4Lfe.com +# failover.sh not running? pgrep -f "failover.sh" +# Both servers Tailscale connected? tailscale status +# +# Resetting during an actual failover causes failover.sh to think everything +# is normal and stop covering the remote — services go offline until the next +# detection cycle catches it again. +# ───────────────────────────────────────────────────────────────────────────── ``` -**When you need this:** - -After failover testing, a failed handback, or manual intervention that left the state file inconsistent. The `failover.sh` state machine reads this file on every cycle — if it shows `FAILOVER` when the system is actually in `NORMAL` operation, the script will make incorrect decisions. - -**What it does:** Rewrites the state file with `state=NORMAL` and clears all tier flags. Does NOT start or stop any containers — state file only. - -**⚠️ Verify first:** Only run after manually confirming both servers are in their correct states — right containers running on the right server, DDNS pointing correctly. The reset doesn't check any of this — it just trusts you. - -**Confirmation required:** Type `YES` to proceed — prevents accidental runs. - --- -### `watchdog_skip_list_manager.sh` +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +failover_state_reset.sh --status # show current state file contents — always first +failover_state_reset.sh --dry-run # show what would be written, no write +failover_state_reset.sh # interactive reset — prompts "YES" to confirm +failover_state_reset.sh --force # non-interactive — for scripts, no terminal +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🐳 watchdog_skip_list_manager.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ View and manage the persistent container skip list used by `docker_watchdog.sh`. -```bash -# View current skip list and restart history -/mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --status - -# Clear a specific container -/mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --clear Authelia - -# Clear everything -/mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --clear-all -``` - -**When you need this:** - -When a container hits the restart loop limit and gets added to the skip list — it stops being monitored until manually cleared. The `--status` view shows which containers are on the list and whether they're currently running, so you can see at a glance what needs attention. - -**After clearing a container:** -1. Fix whatever caused the failure -2. Start it manually: `docker start ContainerName` -3. The watchdog monitors it normally on the next cycle - -**Files managed:** -``` -/boot/config/system_watchdog_failed.db — skip list -/boot/config/container_restart_history.db — restart loop tracking -``` - -Both are cleared per-container or together. The restart history is also cleared when clearing a specific container — gives it a fresh slate for the loop protection window. +> Full documentation in `README-Docker_Essentials.md` — `watchdog_skip_list_manager.sh` +> section including the full recovery workflow. This entry is a quick reference. --- -### `bulk_permissions_repair.sh` +### ── When to Use ────────────────────────────────────────────────────────────── -Applies correct permissions to a single share or specific path. Faster than running the full `media_shares_permissions.sh` which processes every share. - -```bash -# Single share -/mnt/user/appdata/unraid_scripts/Tools/bulk_permissions_repair.sh /mnt/user/Movies - -# Multiple shares -/mnt/user/appdata/unraid_scripts/Tools/bulk_permissions_repair.sh \ - /mnt/user/Movies /mnt/user/Tv_Shows - -# Dry run first -/mnt/user/appdata/unraid_scripts/Tools/bulk_permissions_repair.sh \ - /mnt/user/Movies --dry-run ``` +docker_watchdog.sh restarts the same container N times within the rolling window +→ container added to skip list on /boot/config/ +→ critical notification sent +→ watchdog stops touching it entirely -**When you need this:** - -- A failed transfer left files owned by wrong user -- A container wrote files as root instead of `nobody:users` -- Manual file operations bypassed normal permission handling -- A new share needs permissions applied before the next nightly run - -Uses `PERMISSIONS_MODE` and `PERMISSIONS_OWNER` from `Master.conf` — same values as the full permissions script. Applies `chown` before `chmod` to ensure correct ownership before mode change. +You fix the underlying problem (database, config, dependencies). +You need to clear the container from the skip list so monitoring resumes. +``` --- -### `container_data_export.sh` - -Exports a container's appdata directory to a compressed tar archive. Stops the container before archiving for a clean consistent backup, restarts after. +### ── Recovery Workflow ──────────────────────────────────────────────────────── ```bash -/mnt/user/appdata/unraid_scripts/Tools/container_data_export.sh \ +# 1. Understand the situation — always start here: +watchdog_skip_list_manager.sh --status +# Shows: skip list contents, which are running vs stopped, restart history + +# 2. Fix the underlying problem first +# Check logs: docker logs ContainerName --tail 100 +# Check disk: df -h /mnt/user +# Check db: docker exec ContainerName sqlite3 /path/to.db ".tables" + +# 3. Clear from skip list + restart history: +watchdog_skip_list_manager.sh --clear ContainerName + +# 4. Start the container manually — confirm your fix worked: +docker start ContainerName + +# 5. Watchdog resumes normal monitoring on next cycle — no further action needed +``` + +--- + +### ── Files Managed ──────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── + /boot/config/system_watchdog_failed.db # persistent skip list + /boot/config/container_restart_history.db # restart loop tracking + +# Both live on /boot/config — survive reboots intentionally. +# A container that was skip-listed before a reboot is still broken after it. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +watchdog_skip_list_manager.sh # show status (default) +watchdog_skip_list_manager.sh --status # explicit status +watchdog_skip_list_manager.sh --clear ContainerName # clear specific + restart history +watchdog_skip_list_manager.sh --clear ContainerName --force # no confirmation prompt +watchdog_skip_list_manager.sh --clear-all # clear everything +watchdog_skip_list_manager.sh --clear-all --force # non-interactive +watchdog_skip_list_manager.sh --dry-run # preview any clear action +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔐 bulk_permissions_repair.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Applies correct ownership and permissions to specific paths. Faster than running +`media_shares_permissions.sh` which processes every configured share — use this when +you know exactly what needs fixing and don't want to wait for a full library walk. + +--- + +### ── When to Use ────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Use instead of the full permissions script when: +# +# Admin copy left root:root files — scp, cp, direct file transfer +# New share needs permissions now — can't wait for nightly run +# Container wrote as root — before PUID/PGID was fixed +# Specific directory has wrong perms — targeted fix, not a full library walk +# +# The full media_shares_permissions.sh is the right tool for: +# Regular nightly maintenance (already scheduled in daily_sync_maintenance.sh) +# After confirming a container's PUID/PGID is now correct +# Initial permissions setup on a new server +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── What It Applies ────────────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Same values as media_shares_permissions.sh — consistent permissions everywhere + PERMISSIONS_DIR_MODE="755" # directories — enter, list, no world-write + PERMISSIONS_FILE_MODE="664" # files — owner+group rw, others read-only + PERMISSIONS_OWNER="nobody:users" # matches PUID=99 PGID=100 in containers +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Single path: +bulk_permissions_repair.sh /mnt/user/Movies + +# Multiple paths — all corrected in one run: +bulk_permissions_repair.sh /mnt/user/Movies /mnt/user/Tv_Shows /mnt/user/Music + +# Dry run first — shows count of files that would be corrected: +bulk_permissions_repair.sh /mnt/user/Movies --dry-run + +# Verbose — show each corrected file: +bulk_permissions_repair.sh /mnt/user/Movies --log +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 📦 container_data_export.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Exports a container's appdata directory to a compressed tar archive. Stops the +container first for a clean consistent backup, verifies the archive after creation, +then restarts the container. + +--- + +### ── When to Use ────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Before major container updates — especially those with "no rollback" database migrations +# Before pool migrations — clean backup before moving appdata to a new pool +# Before removing a container from the stack — archive its data before deletion +# Manual point-in-time backup before risky config changes +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Sequence ───────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# 1. Space check +# Estimates required space from appdata size × 1.1 +# Aborts if output directory doesn't have enough free space +# Container is NOT stopped until the space check passes +# +# 2. Stop container cleanly +# docker stop ContainerName — graceful shutdown +# +# 3. Create archive +# tar -czf ContainerName_YYYY-MM-DD_HH-MM.tar.gz /path/to/appdata +# +# 4. Verify archive integrity +# tar -tzf archive.tar.gz — confirms archive is valid and complete +# If verification fails → restart container anyway, report error +# +# 5. Restart container +# docker start ContainerName — always restarted, even if archiving failed +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +# Syntax: container_data_export.sh ContainerName AppDataPath OutputDir + +# Emby backup example: +container_data_export.sh \ Emby \ /mnt/media-servers/Media_Server/Emby \ /mnt/user/Backups/ -# Dry run — verify space and paths without stopping anything -/mnt/user/appdata/unraid_scripts/Tools/container_data_export.sh \ +# Dry run — verify space and paths without stopping anything: +container_data_export.sh \ Emby \ /mnt/media-servers/Media_Server/Emby \ /mnt/user/Backups/ \ --dry-run + +# Output filename format: ContainerName_YYYY-MM-DD_HH-MM.tar.gz ``` -**Output filename:** `ContainerName_YYYY-MM-DD_HH-MM.tar.gz` +--- -**When you need this:** +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🎬 emby_database_repair.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -- Before a major container update you're not sure about -- Before migrating appdata to a different pool -- Before removing a container from the stack — archive its data first -- As a manual point-in-time backup before making significant config changes - -**Space check:** Script estimates required space from appdata size × 1.1 and aborts if the output directory doesn't have enough free space. The container is not stopped until the space check passes. - -**Recovery:** If archiving fails, the container is restarted before the script exits — it tries to leave things clean regardless of outcome. +Stops Emby, runs SQLite `PRAGMA integrity_check` on every Emby database, and restarts. +Reports per-database with specific guidance on what to do if corruption is found. --- -### `emby_database_repair.sh` +### ── When to Use ────────────────────────────────────────────────────────────── -Stops Emby, runs SQLite integrity checks on all Emby databases, and restarts. - -```bash -# Check and report (restarts Emby after) -/mnt/user/appdata/unraid_scripts/Tools/emby_database_repair.sh - -# Dry run — detect config path and show what would be checked -/mnt/user/appdata/unraid_scripts/Tools/emby_database_repair.sh --dry-run +``` +Emby logs show database errors → run this first +Emby crashing repeatedly with no clear cause → likely database corruption +Playback history or user data behaving strangely → users.db or library.db issue +After a hard shutdown or power loss with Emby running → check for WAL corruption ``` -**When you need this:** - -- Emby reports database errors in logs -- Unexpected Emby crashes with no clear cause -- Playback history or user data behaving strangely -- After a hard shutdown or power loss with Emby running - -**Databases checked:** - -| Database | Contains | If corrupted | -|----------|----------|--------------| -| `library.db` | Media library metadata | Delete — Emby rebuilds from media files | -| `users.db` | User accounts and settings | Delete resets all user accounts | -| `authentication.db` | API keys and sessions | Delete — keys regenerated on restart | -| `activity.db` | Activity log | Delete safely — log only | - -**Important:** This script checks and reports. It does NOT automatically delete or repair corrupted databases — that requires judgment about which database is corrupted and whether you have a backup. The summary provides specific guidance per database type. - -**Config path detection:** Automatically detects the Emby config path from Docker volume mounts — no configuration needed beyond `TRANSCODE_EMBY_CONTAINER` in `Master.conf`. - --- -### `zfs_pool_scrub.sh` - -Triggers ZFS scrub on all pools (or a specific pool), waits for completion, and reports results. +### ── Databases Checked ──────────────────────────────────────────────────────── ```bash -# Scrub all pools (skips ZFS_REPORT_IGNORE_POOLS) -/mnt/user/appdata/unraid_scripts/Tools/zfs_pool_scrub.sh - -# Scrub a specific pool -/mnt/user/appdata/unraid_scripts/Tools/zfs_pool_scrub.sh gaming - -# Check current scrub status without starting a new one -/mnt/user/appdata/unraid_scripts/Tools/zfs_pool_scrub.sh --status - -# Dry run — show which pools would be scrubbed -/mnt/user/appdata/unraid_scripts/Tools/zfs_pool_scrub.sh --dry-run +# ───────────────────────────────────────────────────────────────────────────── +# Each database has different recovery implications: +# +# library.db — media library metadata: titles, seasons, episodes, artwork +# CORRUPT → safe to delete — Emby fully rebuilds from media files on next start +# Rebuild takes time but loses nothing permanent +# +# users.db — user accounts, watch history, playback positions, settings +# CORRUPT → deleting resets ALL user accounts and watch history +# Check if you have a recent backup (weekly_sync_maintenance.sh) +# before deleting +# +# authentication.db — API keys, session tokens +# CORRUPT → safe to delete — API keys regenerated on restart +# Any connected clients will need to re-authenticate +# +# activity.db — activity/access log +# CORRUPT → safe to delete — it's a log, losing it is acceptable +# +# ───────────────────────────────────────────────────────────────────────────── +# This script checks and reports ONLY. It does NOT automatically delete or repair +# corrupted databases. Recovery requires judgment — and potentially a backup restore. +# The summary provides specific guidance per database type. +# ───────────────────────────────────────────────────────────────────────────── ``` -**When you need this:** - -ZFS scrub reads every block on every pool and verifies checksums — it catches silent data corruption that would otherwise only surface when you try to read the corrupted data. Running monthly is recommended. - -**Safe to run while in use.** Scrub does not interrupt normal I/O — it runs in the background at low priority. The script polls every 60 seconds until all scrubs complete, then reports errors found. - -**Pool filtering:** Pools in `ZFS_REPORT_IGNORE_POOLS` are skipped during all-pool scrubs. To scrub an ignored pool explicitly, specify it by name. - -**Notifications:** -- Clean completion — normal notification with pool count and duration -- Errors found — warning notification listing affected pools - --- -## Scheduled Summary - -None. Tools are not scheduled — they run when needed. - ---- - -## Adding a New Tool - -When you encounter a situation that required manual bash commands to resolve — write a tool. You'll face it again. - -The pattern for a Tools script: +### ── Usage ─────────────────────────────────────────────────────────────────── ```bash +# ───────────────────────────────────────────────────────────────────────────── +# Normal run — stops Emby, checks all databases, restarts: +emby_database_repair.sh + +# Dry run — detect config path and show what would be checked, no Emby stop: +emby_database_repair.sh --dry-run + +# Verbose — show SQLite output for each database: +emby_database_repair.sh --log + +# Status — show Emby config path and database file locations: +emby_database_repair.sh --status +# ───────────────────────────────────────────────────────────────────────────── +# Config path is detected automatically from Docker volume mounts. +# No configuration needed — just run it. +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🗄️ zfs_pool_scrub.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Triggers ZFS scrub on all pools (or a specific named pool) and waits for completion. +Notifies when done with a summary of any errors found. + +--- + +### ── What ZFS Scrub Does ────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# ZFS stores a checksum with every block of data. Scrub reads every block on +# every pool and verifies the checksum matches the stored hash. +# +# Why this matters: +# Silent data corruption can sit on disk for months without triggering any +# error — until you try to read that specific file. By then: +# - It may already be mirrored to HOST2 in its corrupted state +# - The original source may no longer exist +# - The corruption may have spread if it was a drive issue +# +# ZFS can self-repair during scrub if redundancy exists — RAIDZ or mirrors. +# It cannot repair if you have a single-disk pool (JBOD). +# But it will tell you corruption exists before you find out the hard way. +# +# Recommended: run monthly or after any disk replacement / power event. +# Safe to run while the system is in use — scrub runs at low I/O priority. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Pool Filtering ─────────────────────────────────────────────────────────── + +```bash +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# Single-disk JBOD members can be excluded from all-pool scrubs. +# To scrub a pool that's in the ignore list: specify it by name explicitly. +# +HOST1_ZFS_REPORT_IGNORE_POOLS=( + "disk10" # JBOD member — no redundancy, scrub still useful but excluded from default + "disk9" + "disk8" +) +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +# Scrub all pools except those in ZFS_REPORT_IGNORE_POOLS: +zfs_pool_scrub.sh + +# Scrub a specific pool by name — ignores the ignore list: +zfs_pool_scrub.sh gaming + +# Check current scrub status without starting a new one: +zfs_pool_scrub.sh --status + +# Dry run — show which pools would be scrubbed: +zfs_pool_scrub.sh --dry-run +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 📁 recreate_shares.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Creates share directories on the correct disks after a fresh install or disk rebuild. +Reads `.cfg` files from `/boot/config/shares/` and creates the corresponding +`/mnt/diskN/sharename` directory on each disk listed in the `shareInclude` setting. + +--- + +### ── When to Use ────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# After a fresh unRAID install where /boot/config/shares/*.cfg were restored: +# The share definitions exist → UI shows shares → directories are missing +# rsync.sh tries to write to /mnt/user/Movies → path doesn't exist → aborts +# +# After a disk replacement or rebuild where share folders were lost: +# Replacement disk is blank → no share directories on the new disk +# unRAID won't create them automatically +# +# Run once on HOST2 after fresh setup, before the first rsync from HOST1. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── What It Does ───────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# For each .cfg file in /boot/config/shares/: +# 1. Read the share name (e.g. Movies) +# 2. Read the shareInclude list (e.g. disk1,disk2,disk5) +# 3. Create /mnt/disk1/Movies, /mnt/disk2/Movies, /mnt/disk5/Movies +# 4. Set correct ownership: nobody:users +# +# Does not create content — just the directories. +# rsync.sh can then write into them normally. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +recreate_shares.sh # create all missing share directories +recreate_shares.sh --dry-run # show what would be created without creating +recreate_shares.sh --log # verbose — show each directory created +recreate_shares.sh --status # show share configs and current directory state +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔄 rsync_stop.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Stops rsync intelligently on both local and remote servers. Auto-detects if an +orchestrator is running and chooses the safest stop strategy automatically. + +--- + +### ── Two Modes ──────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Smart (default — auto-detected): +# Orchestrator detected → kill rsync subprocess only +# Orchestrator sees rsync died → moves to next share or exits cleanly +# No orphaned lock files, orchestrator exits naturally +# +# --full-stop: +# Kill orchestrator first, then rsync +# Orchestrator will NOT continue to next share +# Use when: you need everything stopped immediately +# +# Why smart is usually correct: +# Killing the orchestrator directly (daily_sync_maintenance.sh) leaves it +# mid-execution. Containers may be stopped but not restarted. Lock files +# may not be released. The smart approach lets the orchestrator clean up +# after itself — fewer side effects. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── When to Use ────────────────────────────────────────────────────────────── + +``` +rsync running during a window where it shouldn't be → smart stop +rsync stuck with no progress → smart stop +Need to start a manual sync that conflicts → smart stop first +Everything must stop NOW (emergency) → --full-stop +Called by partnership_manage.sh --offboard → --rsync-only flag +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +rsync_stop.sh # smart stop — auto-detect orchestrator +rsync_stop.sh --full-stop # kill orchestrator + rsync +rsync_stop.sh --rsync-only # stop rsync, skip container recovery +rsync_stop.sh --dry-run # preview without stopping anything +rsync_stop.sh --status # show what's currently running +rsync_stop.sh --full-stop --dry-run # preview full stop +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🛑 user_scripts_stop.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Stops all running User Script processes spawned by the unRAID User Scripts plugin. +Identifies processes by their `/tmp/user.scripts` path signature, shows script names +not just PIDs, uses SIGTERM → SIGKILL sequence with verification. + +--- + +### ── When to Use ────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# A script is stuck and won't respond to the Abort button in the User Scripts UI +# → the UI button sends a signal that the script may have trapped or ignored +# → user_scripts_stop.sh finds the process by path signature, not by UI state +# +# Before a planned reboot to ensure scripts exit cleanly +# → server_reboot.sh calls this automatically as part of its shutdown sequence +# +# Emergency stop of all background ecosystem scripts +# → stops system_watchdog, docker_watchdog, failover, and any running maintenance +# → use when you need to take manual control immediately +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Self-Exclusion ──────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# If this script is run via the User Scripts plugin it would find its own PID. +# Self-exclusion prevents the script from killing itself mid-execution. +# Own PID and parent PID are excluded before any killing begins. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +user_scripts_stop.sh # stop all — SIGTERM → verify → SIGKILL if needed +user_scripts_stop.sh --dry-run # show which scripts would be stopped, by name +user_scripts_stop.sh --status # show currently running scripts with PIDs and runtime +user_scripts_stop.sh --log # verbose — show each signal and verification step +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔁 server_reboot.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Graceful reboot with pre-flight warnings, wall message, unRAID notification, clean +shutdown sequence, and VM graceful shutdown before stopping services. + +--- + +### ── When to Use ────────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Planned maintenance reboots — gives users notice and shuts down cleanly +# After kernel or firmware updates that require a reboot +# As an alternative to the unRAID UI reboot — more visibility into state +# +# NOT needed for: system_watchdog.sh triggered reboots (those use /sbin/reboot +# directly after their own shutdown sequence) +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Shutdown Sequence ──────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# 1. Pre-flight warnings (warn not block): +# rsync running → "partial files possible, consider rsync_stop.sh" +# mover running → "files may be left mid-move, consider mover_stop.sh" +# Emby sessions active → "N streams will be interrupted" +# +# 2. Wall message → terminal users +# 3. unRAID notification → dashboard +# 4. Wait REBOOT_SLEEP seconds (default 30) — users can save work +# 5. virsh shutdown each running VM → wait REBOOT_VM_WAIT seconds +# 6. Stop libvirt (VM Manager) +# 7. Stop Docker service +# 8. sync — flush filesystem buffers +# 9. /sbin/reboot +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +server_reboot.sh # reboot with 30s warning +server_reboot.sh --dry-run # walk through sequence without rebooting +server_reboot.sh --status # show running processes that would be affected +server_reboot.sh --reason="maintenance" # include reason in wall + notification +server_reboot.sh --log # verbose output per shutdown step +``` + +--- + +## ━━━ ADDING A NEW TOOL ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# When you encounter a situation that required manual bash commands to resolve — +# write a tool. You'll face it again. The cost of writing the tool is 30 minutes. +# The cost of reconstructing the commands at 2am is much higher. +# +# Checklist for a new Tools script: +# +# ✓ Header explains the specific situation that requires this tool +# ✓ "When to Use" section — exactly the symptoms that trigger this +# ✓ Root check — most tools need root +# ✓ --dry-run support — always +# ✓ --status support — show current state before acting +# ✓ Confirmation for destructive operations (read -p "Type YES:") +# ✓ Notify on completion — success and failure +# ✓ Leave system in clean state on any exit — trap for cleanup +# ───────────────────────────────────────────────────────────────────────────── + +# Minimal skeleton: #!/bin/bash -# Short description of what situation this solves. -# When to run it. -# Any warnings about destructive operations. - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" - +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# Always: -# 1. Check for root -# 2. Support --dry-run -# 3. Confirm before destructive operations (read -p "Type YES:") -# 4. Notify on completion -``` +if [[ "$EUID" -ne 0 ]]; then error "Must be run as root"; exit 1; fi +validate_unraid_cmd "/usr/local/emhttp/plugins/dynamix/scripts/notify" "" "" "notify" +acquire_lock +detect_hosts -Good candidates for future tools: -``` -array_migration.sh — move appdata from one pool to another - with container stop/start and path updates +# Show status if requested +if [[ "$SHOW_STATUS" == true ]]; then + echo "Current state: ..." + exit 0 +fi -emby_metadata_refresh.sh — trigger full library refresh via API - useful after storage changes +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" -tailscale_rekey.sh — rotate Tailscale keys on both servers - with SSH key update on both ends +# Confirm before destructive operations +read -r -p "Type YES to proceed: " CONFIRM +[[ "$CONFIRM" != "YES" ]] && { warn "Aborted."; exit 0; } + +# Do the work +# ... + +notify "Tool completed on $(hostname) ($MY_ID)" "Tool Name" "normal" ``` --- -## Philosophy +## ━━━ PHILOSOPHY ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Tools exist because not every problem has a scheduled solution. Some things only need to happen once. Some things only happen after something goes wrong. Having a dedicated folder keeps the other folders clean and makes it obvious what runs routinely vs what runs situationally. +``` +Write the tool when you solve the problem. +Store it here. +Find it at 2am when you need it again. -Write the tool when you solve the problem. Store it here. Find it at 2am when you need it again. \ No newline at end of file +Tools exist because not every problem has a scheduled solution. +Some things only need to happen once. +Some things only happen after something goes wrong. +Having a dedicated folder keeps the other folders clean — +everything in Orchestrators, Docker_Essentials, and Monitors +has a reason to run regularly. + +Everything here has a reason to exist and wait. +``` \ No newline at end of file diff --git a/Tools/bulk_permissions_repair.sh b/Tools/bulk_permissions_repair.sh index c7efbb1..c78f65c 100644 --- a/Tools/bulk_permissions_repair.sh +++ b/Tools/bulk_permissions_repair.sh @@ -1,64 +1,93 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Bulk Permissions Repair ------------------------------------ -# ----------------------------------------------------------------------------------------------- -# Applies correct permissions to a single share or specific path. -# Faster than running media_shares_permissions.sh which processes all shares. -# Use when a specific share has wrong ownership or permissions after: -# - A failed transfer that left files owned by wrong user -# - A container writing files as root instead of nobody:users -# - Manual file operations that bypassed normal permission handling +# ============================================================================================== +# ============================= Bulk Permissions Repair ======================================== +# ============================================================================================== +# Applies correct ownership and permissions to one or more specific paths. +# Faster than running media_shares_permissions.sh which processes all configured shares. +# +# ── WHEN TO USE ─────────────────────────────────────────────────────────────────────────────── +# Use for targeted repair after: +# - A failed transfer that left files owned by wrong user (root:root from rsync) +# - A container writing as root instead of nobody:users — before PUID/PGID was fixed +# - Manual file copies that bypassed normal permission handling # - A new share that needs permissions applied before the next nightly run +# - A large rsync that imported thousands of files before media_shares_permissions.sh ran # -# Usage: +# ── PERMISSIONS MODEL ───────────────────────────────────────────────────────────────────────── +# Directories: PERMISSIONS_DIR_MODE (default 755) +# Owner (nobody) — rwx enter, list, create files +# Group (users) — r-x enter and list +# Others — r-x Samba guests can browse +# +# Files: PERMISSIONS_FILE_MODE (default 664) +# Owner (nobody) — rw read + write +# Group (users) — rw arrs can import and rename +# Others — r Samba guests can read +# No execute bit — media files are never executable +# +# ── DIAGNOSTIC — HIGH WRONG OWNER COUNT ─────────────────────────────────────────────────────── +# This script counts files with wrong ownership before applying the fix. +# A high count on a share that was recently written → a container has wrong PUID/PGID. +# Fix: add PUID=99 PGID=100 to the container's Docker template. +# Common culprits: SABnzbd, qBittorrent, slskd. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# Root check — required for chown +# Path existence check — skips missing paths with error +# Separate passes — directories and files chmod'd separately for correctness +# validate_unraid_cmd — notify validated before use +# Silent on success — only failures produce visible output +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # bulk_permissions_repair.sh /mnt/user/Movies -# bulk_permissions_repair.sh /mnt/user/Movies --dry-run # bulk_permissions_repair.sh /mnt/user/Movies /mnt/user/Tv_Shows -# -# Uses PERMISSIONS_MODE and PERMISSIONS_OWNER from Master.conf. -# Supports --dry-run to show what would be changed without applying. -# ----------------------------------------------------------------------------------------------- +# bulk_permissions_repair.sh /mnt/user/Movies --dry-run +# bulk_permissions_repair.sh /mnt/user/Movies --log +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" - +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root" + error "Must be run as root — chown requires root" exit 1 fi -success "Running as root" +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + +# detect_hosts() sets MY_ID — used in summary +detect_hosts if [[ ${#PARSED_ARGS[@]} -eq 0 ]]; then error "No paths specified" - error "Usage: bulk_permissions_repair.sh /path/to/share [/another/path]" - error " bulk_permissions_repair.sh /mnt/user/Movies --dry-run" + error "Usage: bulk_permissions_repair.sh /path/to/share [/another/path] [--dry-run]" exit 1 fi -info "Mode: $PERMISSIONS_MODE" -info "Owner: $PERMISSIONS_OWNER" +log "Dir mode: ${PERMISSIONS_DIR_MODE:-755}" +log "File mode: ${PERMISSIONS_FILE_MODE:-664}" +log "Owner: $PERMISSIONS_OWNER" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permissions will be changed" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_PERMS Apply Permissions ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Apply Permissions ━━━ +# ============================================================================================== echo "" -echo "━━━ $ICON_PERMS Permissions Repair ━━━" +echo "━━━ $ICON_PERMS Permissions Repair — $MY_ID ━━━" START=$(date +%s) PASS=() FAIL=() +TOTAL_WRONG_OWNER=0 for share_path in "${PARSED_ARGS[@]}"; do [[ -z "$share_path" ]] && continue @@ -72,62 +101,94 @@ for share_path in "${PARSED_ARGS[@]}"; do continue fi - # Count files for progress context + # Count files for context — warn level so user knows what they're in for on large shares FILE_COUNT=$(find "$share_path" -type f 2>/dev/null | wc -l) - DIR_COUNT=$(find "$share_path" -type d 2>/dev/null | wc -l) + DIR_COUNT=$(find "$share_path" -type d 2>/dev/null | wc -l) SIZE=$(du -sh "$share_path" 2>/dev/null | cut -f1) + warn "$share_path — $FILE_COUNT files, $DIR_COUNT dirs ($SIZE)" - info "$share_path — $FILE_COUNT files, $DIR_COUNT dirs ($SIZE)" + # Count files with wrong ownership before fixing — diagnostic + WRONG_OWNER=$(find "$share_path" \( ! -user nobody -o ! -group users \) \ + 2>/dev/null | wc -l) + if [[ "$WRONG_OWNER" -gt 0 ]]; then + warn "$WRONG_OWNER file(s) with wrong ownership — fixing..." + if [[ "$WRONG_OWNER" -gt 500 ]]; then + warn "High wrong-owner count — check container PUID/PGID settings (should be PUID=99 PGID=100)" + warn "Common culprits: SABnzbd, qBittorrent, slskd" + fi + TOTAL_WRONG_OWNER=$(( TOTAL_WRONG_OWNER + WRONG_OWNER )) + else + log "Ownership already correct — applying mode only" + fi if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would apply: chmod -R $PERMISSIONS_MODE $share_path" - warn "DRY RUN — would apply: chown -R $PERMISSIONS_OWNER $share_path" + warn "DRY RUN — would apply:" + warn " chown -R $PERMISSIONS_OWNER $share_path" + warn " find -type d → chmod ${PERMISSIONS_DIR_MODE:-755}" + warn " find -type f → chmod ${PERMISSIONS_FILE_MODE:-664}" PASS+=("$(basename "$share_path")") continue fi - # Apply ownership first — chmod after so files are owned correctly before mode change - info "Applying ownership: $PERMISSIONS_OWNER..." - chown -R "$PERMISSIONS_OWNER" "$share_path" 2>/dev/null - CHOWN_EXIT=$? + CHOWN_OK=true + CHMOD_DIR_OK=true + CHMOD_FILE_OK=true - info "Applying permissions: $PERMISSIONS_MODE..." - chmod -R "$PERMISSIONS_MODE" "$share_path" 2>/dev/null - CHMOD_EXIT=$? + # Apply ownership first + log "Applying ownership: $PERMISSIONS_OWNER..." + chown -R "$PERMISSIONS_OWNER" "$share_path" 2>/dev/null || CHOWN_OK=false - if [[ "$CHOWN_EXIT" -eq 0 && "$CHMOD_EXIT" -eq 0 ]]; then - success "$ICON_UNLOCKED $(basename "$share_path") — permissions applied" + # Apply directory permissions — separate pass (dirs need execute bit) + log "Applying directory permissions: ${PERMISSIONS_DIR_MODE:-755}..." + find "$share_path" -type d \ + -exec chmod "${PERMISSIONS_DIR_MODE:-755}" {} + 2>/dev/null || CHMOD_DIR_OK=false + + # Apply file permissions — no execute bit on media files + log "Applying file permissions: ${PERMISSIONS_FILE_MODE:-664}..." + find "$share_path" -type f \ + -exec chmod "${PERMISSIONS_FILE_MODE:-664}" {} + 2>/dev/null || CHMOD_FILE_OK=false + + if [[ "$CHOWN_OK" == true && "$CHMOD_DIR_OK" == true && "$CHMOD_FILE_OK" == true ]]; then + log "$ICON_UNLOCKED $(basename "$share_path") — permissions applied ✅" PASS+=("$(basename "$share_path")") else - error "$(basename "$share_path") — permission repair failed" + error "$(basename "$share_path") — repair failed" + error " chown: $CHOWN_OK chmod dirs: $CHMOD_DIR_OK chmod files: $CHMOD_FILE_OK" FAIL+=("$(basename "$share_path")") fi done END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY PERMISSIONS REPAIR SUMMARY ━━━━━" -echo "$ICON_PERMS Mode: $PERMISSIONS_MODE" -echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_PERMS Dir mode: ${PERMISSIONS_DIR_MODE:-755}" +echo "$ICON_PERMS File mode: ${PERMISSIONS_FILE_MODE:-664}" +echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" -echo " $ICON_SUCCESS Pass: ${#PASS[@]} $ICON_ERROR Fail: ${#FAIL[@]}" -echo "" -[[ ${#PASS[@]} -gt 0 ]] && for p in "${PASS[@]}"; do echo " $ICON_UNLOCKED $p"; done -[[ ${#FAIL[@]} -gt 0 ]] && for f in "${FAIL[@]}"; do echo " $ICON_ERROR $f"; done +[[ ${#PASS[@]} -gt 0 ]] && log "Pass: ${PASS[*]}" +[[ ${#FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Fail: ${FAIL[*]}" + +if [[ "$TOTAL_WRONG_OWNER" -gt 0 ]]; then + warn "Total wrong-owner files fixed: $TOTAL_WRONG_OWNER" +fi echo "" if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no changes made" + warn "DRY RUN — no changes made" elif [[ ${#FAIL[@]} -gt 0 ]]; then echo "$ICON_ERROR Status: SOME REPAIRS FAILED" - notify "Permissions repair failed on $(hostname) — failed shares: ${FAIL[*]}" "Permissions Repair" "warning" + notify "Permissions repair failed on $(hostname) — ${FAIL[*]}" \ + "Permissions Repair" "warning" else - echo "$ICON_DONE Status: $ICON_SUCCESS DONE" - notify "Permissions repair complete on $(hostname) — ${#PASS[@]} share(s) repaired" "Permissions Repair" "normal" + log "$ICON_DONE Status: done — ${#PASS[@]} path(s) repaired" fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ ${#FAIL[@]} -gt 0 ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/Tools/container_data_export.sh b/Tools/container_data_export.sh index 7acfcc8..192b3d0 100644 --- a/Tools/container_data_export.sh +++ b/Tools/container_data_export.sh @@ -1,50 +1,79 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Container Data Export -------------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ============================= Container Data Export ========================================== +# ============================================================================================== # Exports a container's appdata directory to a compressed tar archive. # Stops the container before archiving and restarts it after — ensures clean consistent backup. +# Verifies the archive after creation — confirms backup is valid before restarting container. # -# Usage: +# ── WHEN TO USE ─────────────────────────────────────────────────────────────────────────────── +# - Before major container updates (roll back if update goes wrong) +# - Before pool migrations or disk replacements +# - When archiving a container being removed from the stack +# - Before destructive operations on appdata (database migrations etc.) +# - One-off backup of a specific container without running full backup +# +# ── OUTPUT FILE NAMING ──────────────────────────────────────────────────────────────────────── +# ContainerName_YYYY-MM-DD_HH-MM.tar.gz +# Timestamp in filename — run multiple times safely, no overwrite ✅ +# +# ── SPACE CHECK ─────────────────────────────────────────────────────────────────────────────── +# Estimates required space as appdata size × 1.1 (10% buffer). +# Compressed archive will typically be much smaller — this is a conservative floor. +# gzip compression ratio depends heavily on content — database files compress well, +# media files do not. If output is on a media share estimate may be pessimistic. +# +# ── ARCHIVE VERIFICATION ────────────────────────────────────────────────────────────────────── +# After creation the archive is tested with tar --test-file before restarting the container. +# If verification fails the container is still restarted (data unchanged) and an error logged. +# A corrupt archive is not a usable backup — do not assume the archive is good without this. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# DOCKER_TIMEOUT — docker calls protected against hung daemon +# Container restart rule — was running → restart | was stopped → leave stopped ✅ +# Archive cleanup — partial archive removed on tar failure +# Archive verification — tar --test-file after creation +# Container restart on — any failure path still restarts container if it was running +# validate_unraid_cmd — notify validated before use +# Silent on success — only problems produce visible output +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # container_data_export.sh ContainerName /path/to/appdata /path/to/output/dir # container_data_export.sh Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/ -# -# Output file naming: -# ContainerName_YYYY-MM-DD_HH-MM.tar.gz -# -# Use before major container updates, pool migrations, or when archiving -# a container you are removing from the stack. -# -# Supports --dry-run to show what would be archived without making changes. -# ----------------------------------------------------------------------------------------------- +# container_data_export.sh Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/ --dry-run +# container_data_export.sh Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/ --log +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# Args -# ----------------------------------------------------------------------------------------------- +DOCKER_TIMEOUT=30 # longer timeout — stop can take time on large containers + +# ── Positional args ─────────────────────────────────────────────────────────────────────────── CONTAINER_NAME="${PARSED_ARGS[0]:-}" APPDATA_PATH="${PARSED_ARGS[1]:-}" OUTPUT_DIR="${PARSED_ARGS[2]:-}" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" - +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi -success "Running as root" +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" +# detect_hosts() sets MY_ID — used in summary +detect_hosts + +# Arg validation if [[ -z "$CONTAINER_NAME" || -z "$APPDATA_PATH" || -z "$OUTPUT_DIR" ]]; then error "Usage: container_data_export.sh " error "Example: container_data_export.sh Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/" @@ -58,55 +87,94 @@ fi if [[ ! -d "$OUTPUT_DIR" ]]; then error "Output directory not found: $OUTPUT_DIR" + error "Create it first: mkdir -p \"$OUTPUT_DIR\"" exit 1 fi -# Check free space — rough estimate: appdata size × 1.1 +# Space check — conservative: appdata × 1.1 APPDATA_SIZE_KB=$(du -sk "$APPDATA_PATH" 2>/dev/null | cut -f1) -OUTPUT_FREE_KB=$(df "$OUTPUT_DIR" --output=avail | tail -1 | tr -d ' ') +OUTPUT_FREE_KB=$(df "$OUTPUT_DIR" --output=avail 2>/dev/null | tail -1 | tr -d ' ') REQUIRED_KB=$(( APPDATA_SIZE_KB * 11 / 10 )) - APPDATA_SIZE_H=$(du -sh "$APPDATA_PATH" 2>/dev/null | cut -f1) -OUTPUT_FREE_H=$(df -h "$OUTPUT_DIR" --output=avail | tail -1 | tr -d ' ') - -info "Container: $CONTAINER_NAME" -info "Appdata: $APPDATA_PATH ($APPDATA_SIZE_H)" -info "Output dir: $OUTPUT_DIR ($OUTPUT_FREE_H free)" +OUTPUT_FREE_H=$(df -h "$OUTPUT_DIR" --output=avail 2>/dev/null | tail -1 | tr -d ' ') if [[ "$OUTPUT_FREE_KB" -lt "$REQUIRED_KB" ]]; then - error "Insufficient space in $OUTPUT_DIR — need ~${APPDATA_SIZE_H}, have ${OUTPUT_FREE_H}" + error "Insufficient space in $OUTPUT_DIR" + error "Estimated need: ~${APPDATA_SIZE_H} (×1.1 conservative) — available: ${OUTPUT_FREE_H}" exit 1 fi -success "Space check passed" +log "Container: $CONTAINER_NAME" +log "Appdata: $APPDATA_PATH ($APPDATA_SIZE_H)" +log "Output: $OUTPUT_DIR ($OUTPUT_FREE_H free)" +log "Space check passed" + [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_STOP Stop Container ━━━ -# ----------------------------------------------------------------------------------------------- +# ── Ensure container is restarted on any exit if it was running ──────────────────────────────── +CONTAINER_WAS_RUNNING=false +ARCHIVE_PATH="" + +cleanup_on_exit() { + local exit_code=$? + # Remove partial archive on failure + if [[ "$exit_code" -ne 0 && -n "$ARCHIVE_PATH" && -f "$ARCHIVE_PATH" ]]; then + warn "Removing partial archive: $ARCHIVE_PATH" + rm -f "$ARCHIVE_PATH" 2>/dev/null + fi + # Always restart container if it was running + if [[ "$CONTAINER_WAS_RUNNING" == true && "$DRY_RUN" == false ]]; then + local status + status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \ + "$CONTAINER_NAME" 2>/dev/null) + if [[ "$status" != "true" ]]; then + warn "Restarting $CONTAINER_NAME (cleanup)..." + timeout "$DOCKER_TIMEOUT" docker start "$CONTAINER_NAME" >/dev/null 2>&1 || \ + error "Failed to restart $CONTAINER_NAME — start it manually" + fi + fi +} +trap cleanup_on_exit EXIT + +# ============================================================================================== +# ━━━ Stop Container ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_STOP Stop Container ━━━" -CONTAINER_WAS_RUNNING=false -STATUS=$(docker inspect -f '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null) +STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \ + "$CONTAINER_NAME" 2>/dev/null) -if [[ "$STATUS" == "true" ]]; then - CONTAINER_WAS_RUNNING=true - info "$ICON_STOP Stopping $CONTAINER_NAME for clean export..." - if [[ "$DRY_RUN" == false ]]; then - docker stop "$CONTAINER_NAME" >/dev/null 2>&1 && \ - success "$ICON_STOPPED $CONTAINER_NAME stopped" || \ - { error "Failed to stop $CONTAINER_NAME"; exit 1; } - else - warn "DRY RUN — would stop $CONTAINER_NAME" - fi -else - info "$CONTAINER_NAME is not running — archiving as-is" -fi +case "$STATUS" in + true) + CONTAINER_WAS_RUNNING=true + log "Stopping $CONTAINER_NAME for clean export..." + if [[ "$DRY_RUN" == false ]]; then + if timeout "$DOCKER_TIMEOUT" docker stop "$CONTAINER_NAME" >/dev/null 2>&1; then + log "$CONTAINER_NAME stopped ✅" + else + error "Failed to stop $CONTAINER_NAME — aborting export" + exit 1 + fi + else + warn "DRY RUN — would stop $CONTAINER_NAME" + fi + ;; + false) + log "$CONTAINER_NAME is not running — archiving as-is (was stopped state respected)" + ;; + "") + error "$CONTAINER_NAME not found — check container name" + exit 1 + ;; + *) + warn "$CONTAINER_NAME status: $STATUS — proceeding with caution" + ;; +esac -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SYNC Archive ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Archive ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_SYNC Archive ━━━" @@ -114,60 +182,107 @@ TIMESTAMP=$(date '+%Y-%m-%d_%H-%M') ARCHIVE_NAME="${CONTAINER_NAME}_${TIMESTAMP}.tar.gz" ARCHIVE_PATH="${OUTPUT_DIR}/${ARCHIVE_NAME}" -info "Creating: $ARCHIVE_PATH" +warn "Creating: $ARCHIVE_PATH" +warn "Source: $APPDATA_PATH ($APPDATA_SIZE_H)" + START=$(date +%s) +ARCHIVE_VERIFIED=false if [[ "$DRY_RUN" == false ]]; then - tar -czf "$ARCHIVE_PATH" -C "$(dirname "$APPDATA_PATH")" "$(basename "$APPDATA_PATH")" 2>/dev/null - TAR_EXIT=$? + if tar -czf "$ARCHIVE_PATH" \ + -C "$(dirname "$APPDATA_PATH")" \ + "$(basename "$APPDATA_PATH")" 2>/dev/null; then - if [[ "$TAR_EXIT" -ne 0 ]]; then - error "Archive failed (exit code $TAR_EXIT)" - # Restart container before exiting - [[ "$CONTAINER_WAS_RUNNING" == true ]] && docker start "$CONTAINER_NAME" >/dev/null 2>&1 + ARCHIVE_SIZE=$(du -sh "$ARCHIVE_PATH" 2>/dev/null | cut -f1) + warn "Archive created: $ARCHIVE_NAME ($ARCHIVE_SIZE)" + + # Verify archive integrity before declaring success + log "Verifying archive..." + if tar --test-label -f "$ARCHIVE_PATH" 2>/dev/null || \ + tar -tzf "$ARCHIVE_PATH" >/dev/null 2>&1; then + log "Archive verified ✅" + ARCHIVE_VERIFIED=true + else + error "Archive verification FAILED — archive may be corrupt" + error "Container will be restarted but DO NOT rely on this backup" + notify "Container export archive corrupt — $CONTAINER_NAME backup may be unusable" \ + "Container Export" "warning" + fi + else + error "tar failed — archive creation unsuccessful" exit 1 fi - - ARCHIVE_SIZE=$(du -sh "$ARCHIVE_PATH" 2>/dev/null | cut -f1) - success "Archive created: $ARCHIVE_NAME ($ARCHIVE_SIZE)" else warn "DRY RUN — would create: $ARCHIVE_PATH" + ARCHIVE_VERIFIED=true fi END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_START Restart Container ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Restart Container ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_START Restart Container ━━━" +RESTART_OK=false + if [[ "$CONTAINER_WAS_RUNNING" == true ]]; then - info "$ICON_START Restarting $CONTAINER_NAME..." + log "Restarting $CONTAINER_NAME..." if [[ "$DRY_RUN" == false ]]; then - docker start "$CONTAINER_NAME" >/dev/null 2>&1 && \ - success "$ICON_STARTED $CONTAINER_NAME restarted" || \ + if timeout "$DOCKER_TIMEOUT" docker start "$CONTAINER_NAME" >/dev/null 2>&1; then + # Brief settle then verify + sleep 3 + POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ + '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null) + if [[ "$POST_STATUS" == "true" ]]; then + log "$CONTAINER_NAME restarted and running ✅" + RESTART_OK=true + else + error "$CONTAINER_NAME started but crashed immediately — check container logs" + notify "$CONTAINER_NAME failed to stay running after export on $(hostname)" \ + "Container Export" "warning" + fi + else error "Failed to restart $CONTAINER_NAME — start it manually" + notify "$CONTAINER_NAME failed to restart after export on $(hostname)" \ + "Container Export" "warning" + fi else warn "DRY RUN — would restart $CONTAINER_NAME" + RESTART_OK=true fi else - info "$CONTAINER_NAME was not running — not restarting" + log "$CONTAINER_NAME was not running — leaving stopped (state respected) ✅" + RESTART_OK=true fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# Clear trap — clean exit, cleanup_on_exit no longer needed +trap - EXIT + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY CONTAINER EXPORT SUMMARY ━━━━━" -echo "$ICON_CONTAINERS Container: $CONTAINER_NAME" -echo "$ICON_DISK Appdata: $APPDATA_PATH ($APPDATA_SIZE_H)" -echo "$ICON_SYNC Archive: ${ARCHIVE_NAME:-DRY RUN} ${ARCHIVE_SIZE:+($ARCHIVE_SIZE)}" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_CONTAINERS Container: $CONTAINER_NAME" +echo "$ICON_DISK Appdata: $APPDATA_PATH ($APPDATA_SIZE_H)" +echo "$ICON_SYNC Archive: ${ARCHIVE_NAME:-DRY RUN} ${ARCHIVE_SIZE:+($ARCHIVE_SIZE)}" +echo "$ICON_SHIELD Verified: $([[ "$ARCHIVE_VERIFIED" == true ]] && echo "✅" || echo "❌ FAILED")" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" +echo "" + if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no changes made" + warn "DRY RUN — no changes made" +elif [[ "$ARCHIVE_VERIFIED" == true && "$RESTART_OK" == true ]]; then + log "$ICON_DONE Status: done — $ARCHIVE_NAME" +elif [[ "$ARCHIVE_VERIFIED" == false ]]; then + echo "$ICON_ERROR Status: archive verification FAILED — check backup before relying on it" else - echo "$ICON_DONE Status: $ICON_SUCCESS DONE" - notify "Container export complete — $CONTAINER_NAME archived to $ARCHIVE_NAME" "Container Export" "normal" + warn "Status: complete with warnings — check restart status above" fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ "$ARCHIVE_VERIFIED" == false ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/Tools/emby_database_repair.sh b/Tools/emby_database_repair.sh index 7228ce4..b9ad60b 100644 --- a/Tools/emby_database_repair.sh +++ b/Tools/emby_database_repair.sh @@ -1,122 +1,199 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Emby Database Repair --------------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ============================= Emby Database Repair =========================================== +# ============================================================================================== # Stops Emby, runs SQLite integrity checks on all Emby databases, and restarts. # Use when Emby reports database corruption, unexpected crashes, or playback state issues. # -# Checks performed: -# integrity_check — full SQLite integrity verification per database file -# quick_check — faster check for common corruption patterns +# ── CHECKS PERFORMED ────────────────────────────────────────────────────────────────────────── +# PRAGMA integrity_check — full SQLite integrity verification per database +# Skips missing databases gracefully — not all files exist on all setups # -# If corruption is found: -# Reports which database files are corrupted -# Does NOT automatically repair — corruption repair requires manual steps -# Provides guidance on next steps per database type +# ── DATABASES CHECKED ───────────────────────────────────────────────────────────────────────── +# library.db — media library metadata (largest, most critical) +# library.db-wal — write-ahead log (if exists — uncommitted transactions) +# librarydb.db — legacy library database +# users.db — user accounts and settings +# authentication.db — API keys and sessions +# activity.db — activity log (least critical, safe to delete) # -# Emby database files checked: -# library.db — media library metadata -# library.db-wal — write-ahead log (if exists) -# librarydb.db — legacy library database -# users.db — user accounts and settings -# authentication.db — API keys and sessions -# activity.db — activity log +# ── IF CORRUPTION FOUND ─────────────────────────────────────────────────────────────────────── +# Reports which databases are corrupted. Does NOT automatically repair. +# Corruption repair requires manual steps — see guidance in summary output. +# Always take a backup before deleting any database file. # -# HOST1 repairs its own Emby (HOST1_EMBY_CONTAINER). -# HOST2 repairs its own Emby (HOST2_EMBY_CONTAINER). -# detect_hosts() selects the correct container at runtime. -# Container name defined in Host Configuration in Master.conf. -# Supports --dry-run to show what would be checked without stopping Emby. -# ----------------------------------------------------------------------------------------------- +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID and aliases HOST*_EMBY_CONTAINER → EMBY_CONTAINER. +# Each server checks its own Emby instance automatically. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# EXIT trap — Emby always restarted even if script crashes mid-check +# DOCKER_TIMEOUT — all docker calls protected against hung daemon +# jq validation — verifies jq available before config path detection +# validate_unraid_cmd — sqlite3 and notify validated before use +# Container verify — checks Emby stayed running after restart +# Silent healthy — only corruption produces visible output +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# emby_database_repair.sh — stop Emby, check all databases, restart +# emby_database_repair.sh --dry-run — show what would be checked, no Emby stop +# emby_database_repair.sh --log — verbose output per database +# emby_database_repair.sh --status — show config and exit +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" +DOCKER_TIMEOUT=30 # Emby can take time to stop cleanly +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi -success "Running as root" +# Validate required tools +validate_unraid_cmd \ + "$(command -v sqlite3 2>/dev/null || echo /usr/bin/sqlite3)" \ + "--version" "." \ + "sqlite3" || { error "sqlite3 not found — install sqlite package"; exit 1; } -acquire_lock +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" -# Select correct Emby container based on which server is running this script -detect_hosts - -if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - EMBY_CONTAINER="$HOST1_EMBY_CONTAINER" -else - EMBY_CONTAINER="$HOST2_EMBY_CONTAINER" -fi - -info "Emby container: $LOCAL_SERVER_NAME → $EMBY_CONTAINER" - -if ! command -v sqlite3 >/dev/null 2>&1; then - error "sqlite3 not found — install sqlite package" +if ! command -v jq >/dev/null 2>&1; then + error "jq not found — required to detect Emby config path from Docker mounts" exit 1 fi -success "sqlite3 available" -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — Emby will not be stopped" +acquire_lock -# Detect Emby config path from Docker mount -EMBY_CONFIG_HOST=$(docker inspect "$EMBY_CONTAINER" 2>/dev/null | \ +# detect_hosts() sets MY_ID and aliases HOST*_EMBY_CONTAINER → EMBY_CONTAINER +detect_hosts + +if [[ -z "${EMBY_CONTAINER:-}" ]]; then + error "EMBY_CONTAINER not set for $MY_ID — check HOST*_EMBY_CONTAINER in master_host*.conf" + exit 1 +fi + +log "Identity: $MY_ID ($LOCAL_SERVER_NAME)" +log "Emby container: $EMBY_CONTAINER" + +# Detect Emby config path from Docker container mounts +EMBY_CONFIG_HOST=$(timeout "$DOCKER_TIMEOUT" docker inspect "$EMBY_CONTAINER" 2>/dev/null | \ jq -r '.[] | .Mounts[] | select(.Destination == "/config") | .Source' 2>/dev/null) if [[ -z "$EMBY_CONFIG_HOST" ]]; then error "Could not detect Emby config path from Docker mounts" - error "Make sure $EMBY_CONTAINER is the correct container name in Master.conf" + error "Is $EMBY_CONTAINER the correct container name? Check HOST*_EMBY_CONTAINER in master_host*.conf" exit 1 fi -success "Emby config path: $EMBY_CONFIG_HOST" +log "Emby config: $EMBY_CONFIG_HOST" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_STOP Stop Emby ━━━ -# ----------------------------------------------------------------------------------------------- +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — Emby will not be stopped, no checks run" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_EMBY Container: $EMBY_CONTAINER" + echo "$ICON_EMBY Config: $EMBY_CONFIG_HOST" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "" + echo "━━━ Database Files ━━━" + for db_rel in "data/library.db" "data/library.db-wal" "data/librarydb.db" \ + "data/users.db" "data/authentication.db" "data/activity.db"; do + db_path="${EMBY_CONFIG_HOST}/${db_rel}" + db_name=$(basename "$db_rel") + if [[ -f "$db_path" ]]; then + db_size=$(du -sh "$db_path" 2>/dev/null | cut -f1) + echo " $ICON_SUCCESS $db_name ($db_size)" + else + echo " $ICON_SKIP $db_name — not found" + fi + done + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ── EXIT trap — Emby always restarted if it was running ─────────────────────────────────────── +EMBY_WAS_RUNNING=false + +cleanup_on_exit() { + local exit_code=$? + if [[ "$EMBY_WAS_RUNNING" == true && "$DRY_RUN" == false ]]; then + local status + status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \ + "$EMBY_CONTAINER" 2>/dev/null) + if [[ "$status" != "true" ]]; then + warn "Restarting $EMBY_CONTAINER (cleanup)..." + timeout "$DOCKER_TIMEOUT" docker start "$EMBY_CONTAINER" >/dev/null 2>&1 || \ + error "Failed to restart $EMBY_CONTAINER — start it manually" + fi + fi +} + +trap cleanup_on_exit EXIT + +# ============================================================================================== +# ━━━ Stop Emby ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_STOP Stop Emby ━━━" -EMBY_WAS_RUNNING=false -STATUS=$(docker inspect -f '{{.State.Running}}' "$EMBY_CONTAINER" 2>/dev/null) +STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \ + "$EMBY_CONTAINER" 2>/dev/null) -if [[ "$STATUS" == "true" ]]; then - EMBY_WAS_RUNNING=true - warn "Stopping $EMBY_CONTAINER — active sessions will be interrupted" - if [[ "$DRY_RUN" == false ]]; then - docker stop "$EMBY_CONTAINER" >/dev/null 2>&1 && \ - success "$ICON_STOPPED $EMBY_CONTAINER stopped" || \ - { error "Failed to stop $EMBY_CONTAINER"; exit 1; } - sleep 3 # brief wait for file handles to release - else - warn "DRY RUN — would stop $EMBY_CONTAINER" - fi -else - info "$EMBY_CONTAINER is not running — proceeding with checks" -fi +case "$STATUS" in + true) + EMBY_WAS_RUNNING=true + warn "Stopping $EMBY_CONTAINER — active sessions will be interrupted" + if [[ "$DRY_RUN" == false ]]; then + if timeout "$DOCKER_TIMEOUT" docker stop "$EMBY_CONTAINER" >/dev/null 2>&1; then + log "$EMBY_CONTAINER stopped ✅" + sleep 3 # let file handles release + else + error "Failed to stop $EMBY_CONTAINER — aborting" + exit 1 + fi + else + warn "DRY RUN — would stop $EMBY_CONTAINER" + fi + ;; + false) + log "$EMBY_CONTAINER is not running — proceeding with checks" + ;; + "") + error "$EMBY_CONTAINER not found — check container name" + exit 1 + ;; + *) + warn "$EMBY_CONTAINER status: $STATUS — proceeding with caution" + ;; +esac -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_HEALTH Database Integrity Check ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Database Integrity Check ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_HEALTH Database Integrity Check ━━━" START=$(date +%s) -# Emby database files to check DB_FILES=( "data/library.db" + "data/library.db-wal" "data/librarydb.db" "data/users.db" "data/authentication.db" @@ -138,25 +215,38 @@ for db_rel in "${DB_FILES[@]}"; do fi DB_SIZE=$(du -sh "$db_path" 2>/dev/null | cut -f1) - info "$ICON_HEALTH Checking $db_name ($DB_SIZE)..." + log "Checking $db_name ($DB_SIZE)..." + + # WAL file — different check (not a full SQLite database) + if [[ "$db_name" == "*.wal" || "$db_name" == "library.db-wal" ]]; then + if [[ -s "$db_path" ]]; then + warn "$db_name exists and is non-empty (${DB_SIZE})" + warn "Uncommitted WAL data — will be merged when Emby next starts cleanly" + PASS_DBS+=("$db_name (WAL — see warning)") + else + log "$db_name exists but is empty — no pending transactions ✅" + PASS_DBS+=("$db_name") + fi + continue + fi if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would check: $db_path" continue fi - # Run integrity check + # Full integrity check RESULT=$(sqlite3 "$db_path" "PRAGMA integrity_check;" 2>/dev/null) EXIT_CODE=$? if [[ "$EXIT_CODE" -ne 0 ]]; then - error "$db_name — sqlite3 could not open database (may be locked or corrupt)" + error "$db_name — sqlite3 could not open database (locked or corrupt)" FAIL_DBS+=("$db_name") elif [[ "$RESULT" == "ok" ]]; then - success "$db_name — integrity check passed" + log "$db_name — integrity check passed ✅" PASS_DBS+=("$db_name") else - error "$db_name — integrity check FAILED" + error "$db_name — CORRUPTION DETECTED" echo "$RESULT" | head -10 | while IFS= read -r line; do error " $line" done @@ -166,61 +256,88 @@ done END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_START Restart Emby ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Restart Emby ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_START Restart Emby ━━━" -if [[ "$EMBY_WAS_RUNNING" == true && "$DRY_RUN" == false ]]; then - docker start "$EMBY_CONTAINER" >/dev/null 2>&1 && \ - success "$ICON_STARTED $EMBY_CONTAINER restarted" || \ - error "Failed to restart $EMBY_CONTAINER — start it manually" -elif [[ "$DRY_RUN" == true && "$EMBY_WAS_RUNNING" == true ]]; then - warn "DRY RUN — would restart $EMBY_CONTAINER" +RESTART_OK=false + +if [[ "$EMBY_WAS_RUNNING" == true ]]; then + if [[ "$DRY_RUN" == false ]]; then + log "Restarting $EMBY_CONTAINER..." + if timeout "$DOCKER_TIMEOUT" docker start "$EMBY_CONTAINER" >/dev/null 2>&1; then + sleep 5 # Emby takes longer to initialise than most containers + POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ + '{{.State.Running}}' "$EMBY_CONTAINER" 2>/dev/null) + if [[ "$POST_STATUS" == "true" ]]; then + log "$EMBY_CONTAINER restarted and running ✅" + RESTART_OK=true + else + error "$EMBY_CONTAINER started but crashed — database may be corrupt" + error "Check Docker logs: docker logs $EMBY_CONTAINER" + notify "$EMBY_CONTAINER crashed on restart — possible database corruption on $(hostname)" \ + "Emby DB Repair" "warning" + fi + else + error "Failed to restart $EMBY_CONTAINER — start it manually" + notify "$EMBY_CONTAINER failed to restart after integrity check on $(hostname)" \ + "Emby DB Repair" "warning" + fi + else + warn "DRY RUN — would restart $EMBY_CONTAINER" + RESTART_OK=true + fi else - info "$EMBY_CONTAINER was not running — not restarting" + log "$EMBY_CONTAINER was not running — leaving stopped (state respected) ✅" + RESTART_OK=true fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# Clear EXIT trap — clean exit +trap - EXIT + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY EMBY DATABASE REPAIR SUMMARY ━━━━━" -echo "$ICON_HEALTH Container: $EMBY_CONTAINER" -echo "$ICON_HEALTH Config path: $EMBY_CONFIG_HOST" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_EMBY Container: $EMBY_CONTAINER" +echo "$ICON_EMBY Config: $EMBY_CONFIG_HOST" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" -echo " $ICON_SUCCESS Passed: ${#PASS_DBS[@]} $ICON_ERROR Failed: ${#FAIL_DBS[@]} $ICON_INFO Missing: ${#MISSING_DBS[@]}" +echo " $ICON_SUCCESS Passed: ${#PASS_DBS[@]}" +[[ ${#FAIL_DBS[@]} -gt 0 ]] && echo " $ICON_ERROR Failed: ${#FAIL_DBS[@]}" +[[ ${#MISSING_DBS[@]} -gt 0 ]] && log "Skipped: ${#MISSING_DBS[@]} (not found)" echo "" -if [[ ${#PASS_DBS[@]} -gt 0 ]]; then - for db in "${PASS_DBS[@]}"; do echo " $ICON_SUCCESS $db"; done -fi -if [[ ${#FAIL_DBS[@]} -gt 0 ]]; then - for db in "${FAIL_DBS[@]}"; do echo " $ICON_ERROR $db"; done -fi +[[ ${#PASS_DBS[@]} -gt 0 ]] && for db in "${PASS_DBS[@]}"; do log " $ICON_SUCCESS $db"; done +[[ ${#FAIL_DBS[@]} -gt 0 ]] && for db in "${FAIL_DBS[@]}"; do echo " $ICON_ERROR $db"; done echo "" if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no checks performed" + warn "DRY RUN — no checks performed" elif [[ ${#FAIL_DBS[@]} -gt 0 ]]; then - echo "$ICON_ERROR Status: CORRUPTION FOUND" + echo "$ICON_ERROR Status: CORRUPTION FOUND — manual intervention needed" echo "" - echo "$ICON_INFO Next steps for corrupted databases:" - echo " library.db — Stop Emby, delete library.db, restart" - echo " Emby will rebuild from media files (slow first start)" - echo " users.db — Stop Emby, restore from backup or delete" - echo " Deleting resets all user accounts" - echo " authentication.db — Stop Emby, delete, restart" - echo " API keys and sessions will be regenerated" - echo " activity.db — Stop Emby, delete, restart — activity log only" + echo "$ICON_INFO Next steps per corrupted database:" + echo " library.db — Delete file, restart Emby — rebuilds from media (slow first start)" + echo " library.db-wal — Delete WAL file, restart Emby — safe, no permanent data loss" + echo " librarydb.db — Delete file, restart Emby — legacy, Emby recreates" + echo " users.db — Restore from backup or delete — deleting resets all user accounts" + echo " authentication.db — Delete file, restart Emby — API keys regenerated automatically" + echo " activity.db — Delete file, restart Emby — activity log only, no media data" echo "" - echo "$ICON_WARN Always take a backup before deleting any database file" - notify "Emby database corruption found on $(hostname) — failed: ${FAIL_DBS[*]} — manual intervention needed" "Emby DB Repair" "warning" + warn "⚠️ Always take a backup before deleting any database file" + warn " Run: container_data_export.sh $EMBY_CONTAINER " + notify "Emby database CORRUPTION on $(hostname) — failed: ${FAIL_DBS[*]} — manual intervention needed" \ + "Emby DB Repair" "warning" else - echo "$ICON_DONE Status: $ICON_SUCCESS ALL DATABASES HEALTHY" - notify "Emby database integrity check passed on $(hostname) — ${#PASS_DBS[@]} databases healthy" "Emby DB Repair" "normal" + log "$ICON_DONE Status: all ${#PASS_DBS[@]} databases healthy ✅" fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ ${#FAIL_DBS[@]} -gt 0 ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/Tools/failover_state_reset.sh b/Tools/failover_state_reset.sh index 37989c9..fde0d0b 100644 --- a/Tools/failover_state_reset.sh +++ b/Tools/failover_state_reset.sh @@ -1,127 +1,229 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Failover State Reset --------------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ============================= Failover State Reset =========================================== +# ============================================================================================== # Resets the failover state file to NORMAL and clears all tier flags. -# Use when the failover state file is stuck in a non-NORMAL state after testing, -# a failed handback, or manual intervention that left state inconsistent. +# Use when the failover state file is stuck in a non-NORMAL state after: +# - Failover testing that left state as FAILOVER +# - A failed handback that did not complete cleanly +# - Manual intervention that left state inconsistent +# - failover.sh was killed mid-cycle and state is unknown +# +# ── WHAT THIS DOES ──────────────────────────────────────────────────────────────────────────── +# Writes a fresh state file with: +# state=NORMAL +# failover_start=0 +# handback_strikes=0 +# tier2_started=false / tier3_started=false / tier4_started=false # # Does NOT start or stop any containers — state file only. # After reset, failover.sh will resume from NORMAL on its next cycle. # -# ⚠️ Only run this when you have manually verified both servers are in their -# correct states — right containers running on the right server, DDNS correct. -# Resetting state without verifying the actual state can cause failover.sh -# to make incorrect decisions on its next cycle. +# ── ⚠️ ONLY RUN WHEN SAFE ──────────────────────────────────────────────────────────────────── +# Verify BEFORE resetting: +# ✓ Right containers running on the right server +# ✓ DDNS pointing at the correct server +# ✓ No active failover actually in progress +# ✓ Both servers can see each other # -# Supports --dry-run to show what would be reset without changing anything. -# Supports --status to show the current state file contents. -# ----------------------------------------------------------------------------------------------- +# Resetting state while a real failover is happening causes failover.sh to stop +# covering the remote server — services go offline until next detection cycle. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# failover.sh running check — warns if failover.sh is active when reset is attempted +# acquire_lock — prevents concurrent resets +# flock on state write — prevents race with failover.sh mid-cycle read +# Confirmation required — interactive: type YES | non-interactive: --force flag +# validate_unraid_cmd — notify validated before use +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# failover_state_reset.sh — interactive reset (prompts for YES) +# failover_state_reset.sh --dry-run — show current state, show what would be written +# failover_state_reset.sh --status — show current state file contents and exit +# failover_state_reset.sh --force — non-interactive reset (no prompt, use in scripts) +# failover_state_reset.sh --force --dry-run — dry run without prompt +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" -parse_args "$@" +# ── Handle --force flag before parse_args ───────────────────────────────────────────────────── +FORCE=false +FILTERED_ARGS=() +for arg in "$@"; do + case "$arg" in + --force) FORCE=true ;; + *) FILTERED_ARGS+=("$arg") ;; + esac +done -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" +parse_args "${FILTERED_ARGS[@]}" +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi -success "Running as root" -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Current State ━━━ -# ----------------------------------------------------------------------------------------------- +acquire_lock + +# detect_hosts() sets MY_ID — used in summary and notification +detect_hosts + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" +[[ "$FORCE" == true ]] && warn "FORCE mode — confirmation prompt skipped" + +# ============================================================================================== +# ━━━ Current State ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_FAILOVER Current Failover State ━━━" +echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)" echo "" -echo "━━━ $ICON_SUMMARY Current State ━━━" if [[ ! -f "$FAILOVER_STATE_FILE" ]]; then warn "State file not found: $FAILOVER_STATE_FILE" warn "Will be created fresh on reset" + CURRENT_STATE="NOT FOUND" else - info "State file: $FAILOVER_STATE_FILE" + log "State file: $FAILOVER_STATE_FILE" echo "" while IFS='=' read -r key value; do [[ -z "$key" ]] && continue echo " $ICON_INFO $key = $value" done < "$FAILOVER_STATE_FILE" + CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) fi if [[ "$SHOW_STATUS" == true ]]; then + echo "" + # Check if failover.sh is running — informational in status mode + if pgrep -f "failover.sh" >/dev/null 2>&1; then + warn "failover.sh is currently RUNNING — any reset would race with active cycle" + else + log "failover.sh is not running" + fi exit 0 fi -# ----------------------------------------------------------------------------------------------- -# ━━━ Confirmation ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Safety Checks ━━━ +# ============================================================================================== echo "" -warn "$ICON_WARN This will reset the failover state to NORMAL" -warn "Only proceed if you have verified both servers are in their correct states" -warn " — Right containers running on the right server" -warn " — DDNS pointing at the correct server" -warn " — No active failover in progress" +echo "━━━ $ICON_SHIELD Safety Checks ━━━" + +# Check if failover.sh is actively running +FAILOVER_RUNNING=false +if pgrep -f "failover.sh" >/dev/null 2>&1; then + FAILOVER_RUNNING=true + warn "⚠️ failover.sh is currently RUNNING" + warn "Resetting state mid-cycle may cause incorrect decisions on the next iteration" + warn "Consider stopping failover.sh first (click Abort in User Scripts)" + warn "Then reset state, then restart failover.sh" + echo "" + warn "If you are sure you want to proceed anyway, confirm below" +else + log "failover.sh is not running — safe to reset ✅" +fi + +# Check current state — if already NORMAL warn user +if [[ "$CURRENT_STATE" == "NORMAL" ]]; then + warn "State is already NORMAL — reset may not be necessary" + warn "Proceeding anyway (will refresh the state file)" +fi + +# ============================================================================================== +# ━━━ Confirmation ━━━ +# ============================================================================================== +echo "" +warn "This will reset failover state to NORMAL on $MY_ID ($LOCAL_SERVER_NAME)" +warn "Verify before proceeding:" +warn " ✓ Right containers running on the right server" +warn " ✓ DDNS pointing at correct server" +warn " ✓ No real failover actually in progress" +warn " ✓ Both servers can reach each other" echo "" if [[ "$DRY_RUN" == false ]]; then - read -r -p "Type YES to confirm reset: " CONFIRM - if [[ "$CONFIRM" != "YES" ]]; then - info "Reset cancelled" - exit 0 + if [[ "$FORCE" == true ]]; then + log "FORCE flag set — skipping confirmation prompt" + elif [[ -t 0 ]]; then + # Interactive terminal — prompt for confirmation + read -r -p "Type YES to confirm reset: " CONFIRM + if [[ "$CONFIRM" != "YES" ]]; then + warn "Reset cancelled" + exit 0 + fi + else + # Non-interactive — no terminal, cannot prompt + error "Non-interactive mode — use --force flag to skip confirmation" + error "Usage: failover_state_reset.sh --force" + exit 1 fi fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_FAILOVER Reset State File ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Reset State File ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_FAILOVER Resetting State File ━━━" -if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would write:" - echo " state=NORMAL" - echo " failover_start=0" - echo " handback_strikes=0" - echo " tier2_started=false" - echo " tier3_started=false" - echo " tier4_started=false" - echo " last_reset=$(date '+%Y-%m-%d %H:%M:%S')" -else - mkdir -p "$(dirname "$FAILOVER_STATE_FILE")" - cat > "$FAILOVER_STATE_FILE" << EOF -state=NORMAL +NEW_STATE_CONTENT="state=NORMAL failover_start=0 handback_strikes=0 tier2_started=false tier3_started=false tier4_started=false last_reset=$(date '+%Y-%m-%d %H:%M:%S') -EOF - success "State file reset to NORMAL" +reset_by=$MY_ID" + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would write to $FAILOVER_STATE_FILE:" + echo "" + echo "$NEW_STATE_CONTENT" | while IFS= read -r line; do + echo " $line" + done +else + mkdir -p "$(dirname "$FAILOVER_STATE_FILE")" + + # flock prevents race with failover.sh mid-cycle read/write + ( + flock -x 200 + echo "$NEW_STATE_CONTENT" > "$FAILOVER_STATE_FILE" + ) 200>"${FAILOVER_STATE_FILE}.lock" + + warn "State file reset to NORMAL ✅" + log "Written to: $FAILOVER_STATE_FILE" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY FAILOVER STATE RESET SUMMARY ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_FAILOVER File: $FAILOVER_STATE_FILE" +echo "$ICON_TIME Reset at: $(date '+%Y-%m-%d %H:%M:%S')" +echo "" + if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no changes made" + warn "DRY RUN — no changes made" else - echo "$ICON_DONE Status: $ICON_SUCCESS State reset to NORMAL" - echo "$ICON_TIME Reset at: $(date '+%Y-%m-%d %H:%M:%S')" + warn "$ICON_DONE State reset to NORMAL" + log "failover.sh will resume from NORMAL on next cycle" + log "No containers were started or stopped" echo "" - echo "$ICON_INFO failover.sh will resume from NORMAL on next cycle" - echo "$ICON_INFO No containers were started or stopped" - notify "Failover state manually reset to NORMAL on $(hostname)" "Failover State Reset" "normal" + [[ "$FAILOVER_RUNNING" == true ]] && \ + warn "⚠️ failover.sh was running during reset — monitor next cycle carefully" + notify "Failover state manually reset to NORMAL on $(hostname) ($MY_ID)" \ + "Failover State Reset" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Tools/recreate_shares.sh b/Tools/recreate_shares.sh index 999925a..8c1fd34 100644 --- a/Tools/recreate_shares.sh +++ b/Tools/recreate_shares.sh @@ -1,134 +1,273 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Recreate Shares Script ------------------------------------- -# ----------------------------------------------------------------------------------------------- -# Reads all .cfg files from /boot/config/shares/ and creates the corresponding share -# directories on the correct disks based on shareInclude settings. -# Also drops a .recovery marker file in each share via /mnt/user/ so that an initial -# rsync push can run without --delete and self-clean on the second nightly run. +# ============================================================================================== +# ============================= Recreate Shares ================================================ +# ============================================================================================== +# Creates share directories on the correct disks after a fresh unRAID install or disk rebuild. +# Reads all .cfg files from /boot/config/shares/ and creates the corresponding directories +# on each disk listed in the shareInclude setting. # -# Run this script directly on the secondary server after array is started. -# Usage: bash recreate_shares.sh -# ----------------------------------------------------------------------------------------------- - +# ── WHEN TO USE ─────────────────────────────────────────────────────────────────────────────── +# Run directly on HOST2 after array is started following: +# - A full disk replacement or rebuild where share folders were lost +# - A fresh unRAID install where /boot/config/shares/*.cfg files were restored +# - Any situation where the share folder structure exists in config but not on disk +# +# The array must be started before running this script — /mnt/user must be mounted. +# +# ── WHAT IT DOES ────────────────────────────────────────────────────────────────────────────── +# For each share .cfg file: +# 1. Reads shareInclude= to determine which disks own this share +# 2. Creates /mnt/diskN/ShareName/ on each listed disk if it doesn't exist +# 3. Places a .recovery marker file in /mnt/user/ShareName/ via the union filesystem +# +# ── .RECOVERY MARKER FILE ───────────────────────────────────────────────────────────────────── +# The .recovery marker signals to rsync.sh that this is a fresh share with no existing data. +# rsync.sh checks for .recovery before running with --delete: +# .recovery present → rsync WITHOUT --delete (safe — new files only, nothing removed) +# .recovery absent → rsync WITH --delete (normal — mirror mode) +# +# The marker self-cleans: after the first successful rsync the source side has no .recovery +# file so the second nightly run will delete it from the mirror, restoring normal --delete +# behaviour automatically. No manual cleanup needed. ✅ +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# This script runs on the server that needs shares recreated — typically HOST2 during rebuild. +# detect_hosts() sets MY_ID for output clarity. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock — prevents duplicate runs placing duplicate markers +# Root check — mkdir on /mnt/diskN requires root +# Array mount check — exits cleanly if array not started +# Empty cfg guard — warns if no share cfg files found +# Per-disk guards — skips missing disks with warning, continues others +# validate_unraid_cmd — notify validated before use +# Silent on success — only failures produce visible output +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# recreate_shares.sh — create all shares from .cfg files +# recreate_shares.sh --dry-run — preview what would be created, no changes +# recreate_shares.sh --log — verbose output per disk +# recreate_shares.sh --status — show current share state and exit +# ============================================================================================== + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" - + +source "$SCRIPT_DIR/../load_config.sh" + +parse_args "$@" + SHARE_CFG_DIR="/boot/config/shares" MARKER_FILE=".recovery" - + CREATED=() SKIPPED=() FAILED=() - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" - -if ! mountpoint -q /mnt/user; then - error "Array is not started — /mnt/user is not mounted" - info "Start the array in the unRAID UI before running this script" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root — mkdir on /mnt/diskN requires root" exit 1 fi - -success "Array is started" - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_DISK Recreating Shares ━━━ -# ----------------------------------------------------------------------------------------------- + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + +acquire_lock + +# detect_hosts() sets MY_ID — used in summary +detect_hosts + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no directories or markers will be created" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_DISK Cfg dir: $SHARE_CFG_DIR" + echo "$ICON_DISK Marker: $MARKER_FILE" + echo "" + + if ! mountpoint -q /mnt/user; then + warn "Array: NOT STARTED — /mnt/user not mounted" + else + echo " Array: started ✅" + fi + + echo "" + echo "━━━ Share Config Files ━━━" + CFG_COUNT=0 + for cfg in "$SHARE_CFG_DIR"/*.cfg; do + [[ ! -f "$cfg" ]] && continue + (( CFG_COUNT++ )) + SHARE_NAME=$(basename "$cfg" .cfg) + INCLUDE=$(grep '^shareInclude=' "$cfg" 2>/dev/null | cut -d'"' -f2) + MARKER_EXISTS="no" + [[ -f "/mnt/user/${SHARE_NAME}/${MARKER_FILE}" ]] && MARKER_EXISTS="yes" + echo " $ICON_DISK $SHARE_NAME — disks: ${INCLUDE:-none} — recovery marker: $MARKER_EXISTS" + done + [[ "$CFG_COUNT" -eq 0 ]] && warn "No .cfg files found in $SHARE_CFG_DIR" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Pre-flight ━━━ +# ============================================================================================== +# Array must be started — /mnt/user must be mounted +if ! mountpoint -q /mnt/user; then + error "Array is not started — /mnt/user is not mounted" + warn "Start the array in the unRAID UI before running this script" + notify "Recreate shares failed on $(hostname) — array is not started" \ + "Recreate Shares" "warning" + exit 1 +fi +log "Array is started — /mnt/user is mounted ✅" + +# Check share cfg directory exists and has files +if [[ ! -d "$SHARE_CFG_DIR" ]]; then + error "Share config directory not found: $SHARE_CFG_DIR" + error "Is /boot mounted? Is this the correct server?" + exit 1 +fi + +CFG_FILES=("$SHARE_CFG_DIR"/*.cfg) +if [[ ! -f "${CFG_FILES[0]}" ]]; then + warn "No share .cfg files found in $SHARE_CFG_DIR" + warn "Nothing to recreate — are share configs present on /boot?" + exit 0 +fi + +log "Found ${#CFG_FILES[@]} share .cfg file(s) in $SHARE_CFG_DIR" + +# ============================================================================================== +# ━━━ Recreate Shares ━━━ +# ============================================================================================== echo "" -echo "━━━ $ICON_DISK Recreating Shares ━━━" +echo "━━━ $ICON_DISK Recreate Shares — $MY_ID ━━━" echo "" - -for cfg in "$SHARE_CFG_DIR"/*.cfg; do + +for cfg in "${CFG_FILES[@]}"; do + [[ ! -f "$cfg" ]] && continue SHARE_NAME=$(basename "$cfg" .cfg) - INCLUDE=$(grep '^shareInclude=' "$cfg" | cut -d'"' -f2) - + INCLUDE=$(grep '^shareInclude=' "$cfg" 2>/dev/null | cut -d'"' -f2) + + echo "━━━ $ICON_DISK $SHARE_NAME ━━━" + if [[ -z "$INCLUDE" ]]; then - warn "$SHARE_NAME — no shareInclude defined, skipping" + warn "$SHARE_NAME — no shareInclude in .cfg — skipping" SKIPPED+=("$SHARE_NAME") + echo "" continue fi - - info "$ICON_DISK Processing $SHARE_NAME (disks: $INCLUDE)..." - + + log "$SHARE_NAME — disks: $INCLUDE" + SHARE_OK=true - + DIRS_CREATED=0 + DIRS_EXISTED=0 + + # Create directory on each listed disk IFS=',' read -ra DISKS <<< "$INCLUDE" for disk in "${DISKS[@]}"; do - DISK_PATH="/mnt/${disk}/${SHARE_NAME}" - + disk="${disk// /}" # trim whitespace + [[ -z "$disk" ]] && continue + + DISK_MOUNT="/mnt/${disk}" + DISK_PATH="${DISK_MOUNT}/${SHARE_NAME}" + + # Verify disk is mounted + if ! mountpoint -q "$DISK_MOUNT" 2>/dev/null; then + warn "$disk not mounted — skipping $DISK_PATH" + continue + fi + if [[ -d "$DISK_PATH" ]]; then - echo "$ICON_RUNNING $disk/$SHARE_NAME already exists, skipping" + log "$disk/$SHARE_NAME already exists — skipping" + (( DIRS_EXISTED++ )) else - if mkdir -p "$DISK_PATH"; then - echo "$ICON_STARTED Created $DISK_PATH" + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would create: $DISK_PATH" + (( DIRS_CREATED++ )) + elif mkdir -p "$DISK_PATH"; then + log "Created: $DISK_PATH ✅" + (( DIRS_CREATED++ )) else - error "Failed to create $DISK_PATH" + error "Failed to create: $DISK_PATH" SHARE_OK=false fi fi done - + + # Place .recovery marker via /mnt/user (union filesystem) MARKER_PATH="/mnt/user/${SHARE_NAME}/${MARKER_FILE}" + if [[ "$SHARE_OK" == true ]]; then - if touch "$MARKER_PATH" 2>/dev/null; then - echo "$ICON_DONE Marker placed: $MARKER_PATH" + if [[ -f "$MARKER_PATH" ]]; then + log ".recovery marker already exists in $SHARE_NAME" + CREATED+=("$SHARE_NAME") + elif [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would place marker: $MARKER_PATH" + CREATED+=("$SHARE_NAME") + elif touch "$MARKER_PATH" 2>/dev/null; then + log "Marker placed: $MARKER_PATH ✅" CREATED+=("$SHARE_NAME") else - warn "Could not place marker in $SHARE_NAME — share may not be visible yet" + warn "$SHARE_NAME — could not place .recovery marker" + warn "Share directory may not be visible via /mnt/user yet" + warn "Try: touch /mnt/user/${SHARE_NAME}/.recovery manually after verifying share" SKIPPED+=("$SHARE_NAME") fi else FAILED+=("$SHARE_NAME") fi - + + [[ "$DIRS_CREATED" -gt 0 ]] && warn "$SHARE_NAME — created $DIRS_CREATED dir(s) on disk" + [[ "$DIRS_EXISTED" -gt 0 ]] && log "$SHARE_NAME — $DIRS_EXISTED dir(s) already existed" echo "" done - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "━━━━━ $ICON_SUMMARY RECREATE SHARES SUMMARY ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "" - -if [[ ${#CREATED[@]} -gt 0 ]]; then - echo " $ICON_DONE Created & marked:" - for s in "${CREATED[@]}"; do - echo " $ICON_STARTED $s" - done - echo "" -fi - -if [[ ${#SKIPPED[@]} -gt 0 ]]; then - echo " $ICON_WARN Skipped:" - for s in "${SKIPPED[@]}"; do - echo " $ICON_NOT_RUNNING $s" - done - echo "" -fi - -if [[ ${#FAILED[@]} -gt 0 ]]; then - echo " $ICON_ERROR Failed:" - for s in "${FAILED[@]}"; do - echo " $ICON_ERROR $s" - done - echo "" -fi - -echo " $ICON_SUCCESS Created: ${#CREATED[@]}" -echo " $ICON_NOT_RUNNING Skipped: ${#SKIPPED[@]}" -echo " $ICON_ERROR Failed: ${#FAILED[@]}" + +[[ ${#CREATED[@]} -gt 0 ]] && warn "Created + marked: ${CREATED[*]}" +[[ ${#SKIPPED[@]} -gt 0 ]] && warn "Skipped: ${SKIPPED[*]}" +[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" + echo "" -echo "Next steps:" -echo " 1. $ICON_HEALTH Verify shares are visible in unRAID UI" -echo " 2. $ICON_GEAR Remove --delete from Master.conf rsync opts" -echo " 3. $ICON_RUN Run initial push — marker files self-clean on second nightly run" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -[[ ${#FAILED[@]} -gt 0 ]] && exit 1 -exit 0 \ No newline at end of file +echo " Created: ${#CREATED[@]}" +echo " Skipped: ${#SKIPPED[@]}" +echo " Failed: ${#FAILED[@]}" + +if [[ "$DRY_RUN" == false && ${#CREATED[@]} -gt 0 ]]; then + echo "" + echo "━━━ Next Steps ━━━" + echo " 1. $ICON_HEALTH Verify shares are visible in unRAID UI" + echo " 2. $ICON_SYNC Run initial rsync push from HOST1 → HOST2" + echo " rsync.sh will detect .recovery markers and skip --delete" + echo " Normal --delete mode restores automatically on second nightly run" + echo " 3. $ICON_GEAR No manual config changes needed — markers self-clean ✅" +fi + +echo "" +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — no changes made" +elif [[ ${#FAILED[@]} -gt 0 ]]; then + echo "$ICON_ERROR Status: completed with failures" + notify "Recreate shares failed on $(hostname) ($MY_ID) — failed: ${FAILED[*]}" \ + "Recreate Shares" "warning" + exit 1 +else + log "$ICON_DONE Status: done — ${#CREATED[@]} created, ${#SKIPPED[@]} skipped" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Tools/watchdog_skip_list_manager.sh b/Tools/watchdog_skip_list_manager.sh index 9b8c6b8..05011c3 100644 --- a/Tools/watchdog_skip_list_manager.sh +++ b/Tools/watchdog_skip_list_manager.sh @@ -1,175 +1,259 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Watchdog Skip List Manager --------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# =========================== Watchdog Skip List Manager ======================================= +# ============================================================================================== # View and manage the persistent container skip list used by docker_watchdog.sh. -# Containers are added to the skip list when they exceed the restart loop limit. -# They stay there until manually cleared or until found running again automatically. # -# Usage: -# watchdog_skip_list_manager.sh --status — show current skip list and restart history -# watchdog_skip_list_manager.sh --clear-all — clear all skip lists and restart history -# watchdog_skip_list_manager.sh --clear ContainerName — clear specific container +# ── WHAT THE SKIP LIST IS ───────────────────────────────────────────────────────────────────── +# docker_watchdog.sh adds a container to the skip list when it exceeds the restart loop +# limit (WATCHDOG_CONTAINER_RESTART_LIMIT in WATCHDOG_CONTAINER_RESTART_WINDOW hours). +# Once on the skip list the watchdog stops restarting it — prevents infinite restart loops. # -# After clearing a container from the skip list: +# Skip list persists on /boot/config — survives reboots. +# Auto-clears when docker_watchdog.sh sees the container running on a cycle. +# This script clears it manually when you have fixed the underlying problem. +# +# ── ACTIONS ─────────────────────────────────────────────────────────────────────────────────── +# --status — show skip list, container states, restart history +# --clear ContainerName — clear a specific container from skip list + history +# --clear-all — clear all skip lists and restart history +# +# ── AFTER CLEARING ──────────────────────────────────────────────────────────────────────────── # 1. Fix whatever was causing the container to fail -# 2. Start the container manually: docker start ContainerName -# 3. The watchdog will monitor it normally on the next cycle +# 2. Start it manually: docker start ContainerName +# 3. docker_watchdog.sh monitors it normally on the next cycle +# 4. If it crashes again → watchdog adds it back and notifies # -# Files managed: -# SYS_WATCHDOG_FAILED_FILE — persistent container skip list -# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection -# ----------------------------------------------------------------------------------------------- +# ── SKIP LIST AUTO-CLEAR ────────────────────────────────────────────────────────────────────── +# docker_watchdog.sh auto-clears a container from the skip list when it sees it running. +# So if a container recovers on its own (Docker restart policy eventually works), +# the watchdog will see it running, remove it from the skip list, and resume monitoring. +# Manual clear only needed when container is stuck stopped and needs intervention. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock — prevents concurrent access with docker_watchdog.sh writing files +# docker_watchdog check — warns if watchdog is running during clear (could re-add instantly) +# DOCKER_TIMEOUT — docker inspect calls protected against daemon hangs +# Confirmation required — interactive: YES | non-interactive: --force flag +# validate_unraid_cmd — notify validated before use +# +# ── FILES MANAGED ───────────────────────────────────────────────────────────────────────────── +# SYS_WATCHDOG_FAILED_FILE — persistent container skip list +# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# watchdog_skip_list_manager.sh — show status +# watchdog_skip_list_manager.sh --status — show status explicitly +# watchdog_skip_list_manager.sh --clear ContainerName — clear specific container +# watchdog_skip_list_manager.sh --clear-all — clear everything +# Any action supports --dry-run and --force +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" -parse_args "$@" +DOCKER_TIMEOUT=15 -# Parse action from args -ACTION="" +# ── Parse action flags before parse_args ────────────────────────────────────────────────────── +ACTION="status" TARGET_CONTAINER="" +FORCE=false +FILTERED_ARGS=() -for arg in "${PARSED_ARGS[@]}"; do +for arg in "$@"; do case "$arg" in --clear-all) ACTION="clear-all" ;; --clear) ACTION="clear" ;; --status) ACTION="status" ;; + --force) FORCE=true ;; *) - [[ "$ACTION" == "clear" && -z "$TARGET_CONTAINER" ]] && TARGET_CONTAINER="$arg" + if [[ "$ACTION" == "clear" && -z "$TARGET_CONTAINER" ]]; then + TARGET_CONTAINER="$arg" + else + FILTERED_ARGS+=("$arg") + fi ;; esac done -[[ -z "$ACTION" ]] && ACTION="status" - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" +parse_args "${FILTERED_ARGS[@]}" +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi -success "Running as root" -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" +acquire_lock + +# detect_hosts() sets MY_ID — used in output +detect_hosts + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" +[[ "$FORCE" == true ]] && warn "FORCE mode — confirmation prompt skipped" + +# Ensure state files exist touch "$SYS_WATCHDOG_FAILED_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null -# ----------------------------------------------------------------------------------------------- -# ━━━ STATUS ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Status — always shown regardless of action ━━━ +# ============================================================================================== echo "" -echo "━━━ $ICON_WATCHDOG Skip List Status ━━━" +echo "━━━ $ICON_WATCHDOG Skip List Status — $MY_ID ━━━" SKIP_COUNT=$(grep -c "." "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || echo 0) +SKIP_COUNT="${SKIP_COUNT//[^0-9]/}"; SKIP_COUNT="${SKIP_COUNT:-0}" RESTART_COUNT=$(wc -l < "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0) +RESTART_COUNT="${RESTART_COUNT//[^0-9]/}"; RESTART_COUNT="${RESTART_COUNT:-0}" +# docker_watchdog.sh running check +WATCHDOG_RUNNING=false +if pgrep -f "docker_watchdog.sh" >/dev/null 2>&1; then + WATCHDOG_RUNNING=true + warn "docker_watchdog.sh is currently RUNNING" + [[ "$ACTION" != "status" ]] && \ + warn "Clearing during an active cycle — watchdog may re-add container on next iteration" +fi + +echo "" if [[ "$SKIP_COUNT" -eq 0 ]]; then - success "Skip list is empty — all containers healthy" + log "Skip list: empty — all containers monitored normally ✅" else - warn "$SKIP_COUNT container(s) on skip list:" + warn "$SKIP_COUNT container(s) on skip list — manual intervention needed:" + echo "" while IFS= read -r container; do [[ -z "$container" ]] && continue - # Check if container is currently running - STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") - if [[ "$STATUS" == "true" ]]; then - echo " $ICON_RUNNING $container — currently RUNNING (will auto-clear on next watchdog cycle)" - elif [[ "$STATUS" == "false" ]]; then - echo " $ICON_STOPPED $container — currently STOPPED — fix and start manually" - else - echo " $ICON_INFO $container — container not found" - fi + STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ + '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") + case "$STATUS" in + true) + echo " $ICON_RUNNING $container — RUNNING (watchdog will auto-clear next cycle)" + ;; + false) + echo " $ICON_NOT_RUNNING $container — STOPPED — fix and start manually" + ;; + *) + echo " $ICON_WARN $container — not found on this server" + ;; + esac done < "$SYS_WATCHDOG_FAILED_FILE" fi echo "" echo "━━━ $ICON_WATCHDOG Restart History ━━━" if [[ "$RESTART_COUNT" -eq 0 ]]; then - success "No restart history" + log "No restart history" else - info "$RESTART_COUNT restart entries (window: ${WATCHDOG_CONTAINER_RESTART_WINDOW}h)" + log "$RESTART_COUNT restart entries (window: ${WATCHDOG_CONTAINER_RESTART_WINDOW}h)" echo "" - # Show per-container restart counts - awk -F'|' '{counts[$1]++} END {for (c in counts) printf " %-30s %d restart(s)\n", c, counts[c]}' \ - "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | sort + awk -F'|' '{counts[$1]++} END { + for (c in counts) + printf " %-30s %d restart(s)\n", c, counts[c] + }' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | sort fi [[ "$ACTION" == "status" ]] && exit 0 -# ----------------------------------------------------------------------------------------------- -# ━━━ CLEAR ALL ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Clear All ━━━ +# ============================================================================================== if [[ "$ACTION" == "clear-all" ]]; then echo "" echo "━━━ $ICON_TRASH Clear All Skip Lists ━━━" warn "This will clear the skip list and restart history for ALL containers" + echo "" if [[ "$DRY_RUN" == false ]]; then - read -r -p "Type YES to confirm: " CONFIRM - if [[ "$CONFIRM" != "YES" ]]; then - info "Cancelled" - exit 0 + if [[ "$FORCE" == true ]]; then + log "FORCE flag set — skipping confirmation" + elif [[ -t 0 ]]; then + read -r -p "Type YES to confirm: " CONFIRM + if [[ "$CONFIRM" != "YES" ]]; then + warn "Cancelled" + exit 0 + fi + else + error "Non-interactive mode — use --force flag to skip confirmation" + exit 1 fi > "$SYS_WATCHDOG_FAILED_FILE" > "$WATCHDOG_CONTAINER_RESTART_LOG" - success "Skip list cleared" - success "Restart history cleared" - notify "Watchdog skip list manually cleared on $(hostname) — all containers will be monitored normally" "Watchdog Manager" "normal" + warn "Skip list cleared ✅" + warn "Restart history cleared ✅" + [[ "$WATCHDOG_RUNNING" == true ]] && \ + warn "Note: watchdog is running — containers will be monitored on next cycle" + notify "Watchdog skip list cleared on $(hostname) ($MY_ID) — all containers will be monitored normally" \ + "Watchdog Manager" "warning" else warn "DRY RUN — would clear: $SYS_WATCHDOG_FAILED_FILE" warn "DRY RUN — would clear: $WATCHDOG_CONTAINER_RESTART_LOG" fi fi -# ----------------------------------------------------------------------------------------------- -# ━━━ CLEAR SPECIFIC CONTAINER ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Clear Specific Container ━━━ +# ============================================================================================== if [[ "$ACTION" == "clear" ]]; then echo "" echo "━━━ $ICON_TRASH Clear Container: $TARGET_CONTAINER ━━━" if [[ -z "$TARGET_CONTAINER" ]]; then - error "No container specified. Usage: --clear ContainerName" + error "No container specified" + error "Usage: watchdog_skip_list_manager.sh --clear ContainerName" exit 1 fi + # Remove from skip list if ! grep -q "^${TARGET_CONTAINER}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null; then warn "$TARGET_CONTAINER is not on the skip list" else if [[ "$DRY_RUN" == false ]]; then sed -i "/^${TARGET_CONTAINER}$/d" "$SYS_WATCHDOG_FAILED_FILE" - success "$TARGET_CONTAINER removed from skip list" + warn "$TARGET_CONTAINER removed from skip list ✅" else warn "DRY RUN — would remove $TARGET_CONTAINER from skip list" fi fi # Clear restart history for this container - HIST_COUNT=$(grep -c "^${TARGET_CONTAINER}|" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0) + HIST_COUNT=$(grep -c "^${TARGET_CONTAINER}|" \ + "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0) + HIST_COUNT="${HIST_COUNT//[^0-9]/}"; HIST_COUNT="${HIST_COUNT:-0}" + if [[ "$HIST_COUNT" -gt 0 ]]; then if [[ "$DRY_RUN" == false ]]; then sed -i "/^${TARGET_CONTAINER}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG" - success "Cleared $HIST_COUNT restart history entries for $TARGET_CONTAINER" + warn "Cleared $HIST_COUNT restart history entries for $TARGET_CONTAINER ✅" else warn "DRY RUN — would clear $HIST_COUNT restart history entries" fi else - info "No restart history for $TARGET_CONTAINER" + log "No restart history for $TARGET_CONTAINER" fi - echo "" - echo "$ICON_INFO Next steps:" - echo " 1. Fix whatever was causing $TARGET_CONTAINER to fail" - echo " 2. Start it manually: docker start $TARGET_CONTAINER" - echo " 3. Watchdog will monitor it normally on the next cycle" + [[ "$WATCHDOG_RUNNING" == true ]] && \ + warn "Note: watchdog is running — $TARGET_CONTAINER may be re-added if still failing" + + if [[ "$DRY_RUN" == false ]]; then + echo "" + echo "━━━ $ICON_INFO Next Steps ━━━" + echo " 1. Fix whatever was causing $TARGET_CONTAINER to fail" + echo " 2. Start it manually: docker start $TARGET_CONTAINER" + echo " 3. docker_watchdog.sh monitors it on the next cycle" + echo " 4. If it crashes again → watchdog adds it back and notifies" + notify "$TARGET_CONTAINER cleared from watchdog skip list on $(hostname) ($MY_ID)" \ + "Watchdog Manager" "warning" + fi fi echo "" -echo "━━━━━ $ICON_SUMMARY DONE ━━━━━" \ No newline at end of file +echo "━━━━━ $ICON_SUMMARY DONE — $MY_ID ━━━━━" \ No newline at end of file diff --git a/Tools/zfs_pool_scrub.sh b/Tools/zfs_pool_scrub.sh index 20cb172..6cece26 100644 --- a/Tools/zfs_pool_scrub.sh +++ b/Tools/zfs_pool_scrub.sh @@ -1,84 +1,128 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- ZFS Pool Scrub --------------------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= ZFS Pool Scrub ============================================ +# ============================================================================================== # Triggers a ZFS scrub on all pools (or a specific pool) and waits for completion. # Sends a notification when scrub completes with a summary of any errors found. # -# ZFS scrub reads every block on every pool and verifies checksums — it catches -# silent data corruption that would otherwise only surface when you try to read -# the corrupted data. Running monthly is recommended for all ZFS pools. +# ── WHAT ZFS SCRUB DOES ─────────────────────────────────────────────────────────────────────── +# Reads every block on every pool and verifies checksums against the stored hash. +# Catches silent data corruption that would otherwise only surface when you read the +# corrupted data — by then it may be too late for redundancy to help. # -# Usage: -# zfs_pool_scrub.sh — scrub all pools -# zfs_pool_scrub.sh poolname — scrub specific pool only -# zfs_pool_scrub.sh --status — show scrub status for all pools -# zfs_pool_scrub.sh --dry-run — show what would be scrubbed +# Scrub is safe to run while the pool is in use — it does not interrupt normal I/O. +# It does consume I/O bandwidth — run during off-peak hours or maintenance windows. +# Monthly is recommended for all pools. Quarterly minimum for large pools. # -# Pools in ZFS_REPORT_IGNORE_POOLS are skipped unless specified explicitly. -# Scrub runs in background — script polls until complete then reports. -# Safe to run while the pool is in use — scrub does not interrupt normal I/O. -# ----------------------------------------------------------------------------------------------- +# ── BEHAVIOUR ───────────────────────────────────────────────────────────────────────────────── +# Starts scrub on each pool then polls every 60 seconds until all complete. +# Progress shown via warn() every poll (visible) when scrub is running. +# Safe to leave running or interrupt — scrub continues even if script is stopped. +# On completion reports errors per pool and notifies if any found. +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS → ZFS_REPORT_IGNORE_POOLS. +# Pools in ZFS_REPORT_IGNORE_POOLS are skipped (single-disk VMs, temp pools etc.) +# unless specified explicitly as a positional argument. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock — prevents concurrent scrub starts on same server +# detect_hosts() — correct pool ignore list per host +# validate_unraid_cmd — zpool and notify validated before use +# Scrub-in-progress check — skips pools already scrubbing rather than erroring +# SIGTERM trap — poll loop exits cleanly on signal +# Silent when clean — only errors produce visible output and notification +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_ZFS_REPORT_IGNORE_POOLS — pools excluded from automatic scrub +# Aliased by detect_hosts() — script uses ZFS_REPORT_IGNORE_POOLS +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# zfs_pool_scrub.sh — scrub all non-ignored pools +# zfs_pool_scrub.sh poolname — scrub specific pool (bypasses ignore list) +# zfs_pool_scrub.sh --status — show scrub status for all pools +# zfs_pool_scrub.sh --dry-run — show what would be scrubbed +# zfs_pool_scrub.sh --log — verbose progress output +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" TARGET_POOL="${PARSED_ARGS[0]:-}" +SCRUB_RUNNING=true -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" - +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi -success "Running as root" +validate_unraid_cmd \ + "$(command -v zpool 2>/dev/null || echo /sbin/zpool)" \ + "--version" "" \ + "zpool" || { + error "ZFS not available on this system — zpool not found" + exit 1 + } + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" acquire_lock -if ! command -v zpool >/dev/null 2>&1; then - error "ZFS not available on this system" - exit 1 -fi +# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS +detect_hosts -success "ZFS available" -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no scrubs will be started" - -# Build ignore map +# Build ignore pool map declare -A IGNORE_MAP -for pool in "${ZFS_REPORT_IGNORE_POOLS[@]}"; do +for pool in "${ZFS_REPORT_IGNORE_POOLS[@]:-}"; do [[ -n "$pool" ]] && IGNORE_MAP["$pool"]=1 done -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +log "Identity: $MY_ID ($LOCAL_SERVER_NAME)" +log "Ignore pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}" + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no scrubs will be started" + +# SIGTERM trap — exit poll loop cleanly +trap 'warn "ZFS scrub script interrupted — scrub continues in background"; SCRUB_RUNNING=false; exit 0' \ + SIGTERM SIGINT + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" - echo "━━━━━ $ICON_SUMMARY SCRUB STATUS ━━━━━" - zpool list -H -o name 2>/dev/null | while read -r pool; do + echo "━━━━━ $ICON_SUMMARY ZFS SCRUB STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "" + while IFS= read -r pool; do + [[ -z "$pool" ]] && continue SCAN=$(zpool status "$pool" 2>/dev/null | grep "scan:") - echo " $ICON_ZFS $pool — $SCAN" - done + IGNORED="" + [[ -n "${IGNORE_MAP[$pool]:-}" ]] && IGNORED=" (ignored)" + echo " $ICON_ZFS $pool${IGNORED} — ${SCAN:-no scan data}" + done < <(zpool list -H -o name 2>/dev/null) + echo "" + echo " Ignored pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -# ----------------------------------------------------------------------------------------------- -# Build pool list to scrub -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ── Build pool list ──────────────────────────────────────────────────────────────────────────── +# ============================================================================================== POOLS_TO_SCRUB=() if [[ -n "$TARGET_POOL" ]]; then - # Specific pool requested — validate it exists + # Specific pool — bypass ignore list, validate exists if ! zpool list "$TARGET_POOL" >/dev/null 2>&1; then error "Pool not found: $TARGET_POOL" exit 1 @@ -89,7 +133,7 @@ else while IFS= read -r pool; do [[ -z "$pool" ]] && continue if [[ -n "${IGNORE_MAP[$pool]:-}" ]]; then - info "Skipping $pool (in ZFS_REPORT_IGNORE_POOLS)" + log "Skipping $pool (in ZFS_REPORT_IGNORE_POOLS)" continue fi POOLS_TO_SCRUB+=("$pool") @@ -97,99 +141,139 @@ else fi if [[ ${#POOLS_TO_SCRUB[@]} -eq 0 ]]; then - warn "No pools to scrub" + warn "No pools to scrub — all pools may be on the ignore list" + warn "Ignored: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}" exit 0 fi -info "Pools to scrub: ${POOLS_TO_SCRUB[*]}" +log "Pools to scrub: ${POOLS_TO_SCRUB[*]}" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_ZFS Start Scrubs ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Start Scrubs ━━━ +# ============================================================================================== echo "" -echo "━━━ $ICON_ZFS Starting ZFS Scrubs ━━━" +echo "━━━ $ICON_ZFS Starting ZFS Scrubs — $MY_ID ━━━" START=$(date +%s) +STARTED=() +SKIPPED_POOLS=() + for pool in "${POOLS_TO_SCRUB[@]}"; do - info "$ICON_ZFS Starting scrub on $pool..." - if [[ "$DRY_RUN" == false ]]; then - zpool scrub "$pool" 2>/dev/null && \ - success "$pool scrub started" || \ - error "Failed to start scrub on $pool" - else + + # Check if scrub already in progress + ALREADY=$(zpool status "$pool" 2>/dev/null | grep "scan:" | grep -c "in progress" || true) + if [[ "$ALREADY" -gt 0 ]]; then + warn "$pool — scrub already in progress — joining existing scrub" + STARTED+=("$pool") + continue + fi + + log "Starting scrub on $pool..." + + if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would scrub: $pool" + STARTED+=("$pool") + elif zpool scrub "$pool" 2>/dev/null; then + log "$pool scrub started ✅" + STARTED+=("$pool") + else + error "Failed to start scrub on $pool" + SKIPPED_POOLS+=("$pool") fi done -[[ "$DRY_RUN" == true ]] && { +if [[ "$DRY_RUN" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY ZFS SCRUB SUMMARY ━━━━━" - echo "$ICON_WARN Status: DRY RUN — no scrubs started" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + warn "DRY RUN — no scrubs started" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" exit 0 -} +fi -# ----------------------------------------------------------------------------------------------- -# ━━━ Poll until complete ━━━ -# ----------------------------------------------------------------------------------------------- +if [[ ${#STARTED[@]} -eq 0 ]]; then + error "No scrubs were started — check pool status" + exit 1 +fi + +# ============================================================================================== +# ━━━ Poll Until Complete ━━━ +# ============================================================================================== echo "" -echo "━━━ $ICON_TIME Waiting for scrubs to complete ━━━" -info "Polling every 60 seconds — this may take a while on large pools" -info "Safe to leave running — scrub continues even if this script is stopped" +echo "━━━ $ICON_TIME Waiting for Scrubs to Complete ━━━" +log "Polling every 60 seconds — scrubs may take hours on large pools" +log "Safe to interrupt — scrubs continue in background if script is stopped" -STILL_RUNNING=true -while [[ "$STILL_RUNNING" == true ]]; do +while [[ "$SCRUB_RUNNING" == true ]]; do sleep 60 + STILL_RUNNING=false - for pool in "${POOLS_TO_SCRUB[@]}"; do - STATUS=$(zpool status "$pool" 2>/dev/null | grep "scan:" | grep -c "in progress" || true) - if [[ "$STATUS" -gt 0 ]]; then + for pool in "${STARTED[@]}"; do + IN_PROGRESS=$(zpool status "$pool" 2>/dev/null | \ + grep "scan:" | grep -c "in progress" || true) + if [[ "$IN_PROGRESS" -gt 0 ]]; then STILL_RUNNING=true - REPAIRED=$(zpool status "$pool" 2>/dev/null | grep "scan:" | grep -oE "[0-9]+ repaired") - log "$pool — scrub in progress ${REPAIRED:+($REPAIRED)}" + # Show progress — always visible so user knows it's running + PROGRESS=$(zpool status "$pool" 2>/dev/null | \ + grep "scan:" | grep -oE "[0-9]+\.[0-9]+% done") + REPAIRED=$(zpool status "$pool" 2>/dev/null | \ + grep "scan:" | grep -oE "[0-9]+ repaired") + warn "$pool — scrub in progress ${PROGRESS:+$PROGRESS}${REPAIRED:+ ($REPAIRED)}" fi done + + [[ "$STILL_RUNNING" == false ]] && SCRUB_RUNNING=false done END=$(date +%s) -success "All scrubs complete" +warn "All scrubs complete — $(format_duration $(( END - START )))" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Results ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Results ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_ZFS Scrub Results ━━━" POOLS_OK=() POOLS_ERRORS=() -for pool in "${POOLS_TO_SCRUB[@]}"; do +for pool in "${STARTED[@]}"; do SCAN_LINE=$(zpool status "$pool" 2>/dev/null | grep "scan:") - ERRORS=$(zpool status "$pool" 2>/dev/null | grep "errors:" | grep -v "No known data errors") + ERRORS=$(zpool status "$pool" 2>/dev/null | \ + grep "errors:" | grep -v "No known data errors") if [[ -n "$ERRORS" ]]; then - error "$pool — $SCAN_LINE" - error "$pool — $ERRORS" + error "$pool — ERRORS FOUND" + error " $SCAN_LINE" + error " $ERRORS" POOLS_ERRORS+=("$pool") else - success "$pool — $SCAN_LINE" + log "$pool — $SCAN_LINE" POOLS_OK+=("$pool") fi done +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY ZFS SCRUB SUMMARY ━━━━━" -echo "$ICON_ZFS Pools scrubbed: ${#POOLS_TO_SCRUB[@]}" -echo "$ICON_SUCCESS Clean: ${#POOLS_OK[@]}" -echo "$ICON_ERROR Errors: ${#POOLS_ERRORS[@]}" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_ZFS Pools: ${#POOLS_TO_SCRUB[@]} to scrub" +echo "$ICON_SUCCESS Clean: ${#POOLS_OK[@]}" +[[ ${#POOLS_ERRORS[@]} -gt 0 ]] && echo "$ICON_ERROR Errors: ${#POOLS_ERRORS[@]}" +[[ ${#SKIPPED_POOLS[@]} -gt 0 ]] && warn "Failed start: ${SKIPPED_POOLS[*]}" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" if [[ ${#POOLS_ERRORS[@]} -gt 0 ]]; then echo "$ICON_ERROR Status: ERRORS FOUND — ${POOLS_ERRORS[*]}" - notify "ZFS scrub complete on $(hostname) — ERRORS found in pools: ${POOLS_ERRORS[*]}" "ZFS Scrub" "warning" -else - echo "$ICON_DONE Status: $ICON_SUCCESS ALL POOLS CLEAN" - notify "ZFS scrub complete on $(hostname) — ${#POOLS_OK[@]} pools clean in $(format_duration $((END - START)))" "ZFS Scrub" "normal" + notify "ZFS scrub errors on $(hostname) ($MY_ID) — pools with errors: ${POOLS_ERRORS[*]}" \ + "ZFS Scrub" "warning" +elif [[ ${#POOLS_OK[@]} -gt 0 ]]; then + log "$ICON_DONE Status: all ${#POOLS_OK[@]} pools clean ✅" fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ ${#POOLS_ERRORS[@]} -gt 0 ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/Transcodes/README-Transcoding.md b/Transcodes/README-Transcoding.md index 283e884..88bf41f 100644 --- a/Transcodes/README-Transcoding.md +++ b/Transcodes/README-Transcoding.md @@ -1,171 +1,418 @@ -# Transcodes +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# 🎬 TRANSCODING +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Ramdisk-based transcode storage management for Emby using filesystem symlink indirection. +**Ramdisk-based transcode storage with automatic SSD fallback, session-safe symlink +flipping, and stale file cleanup.** Emby transcodes to RAM at full speed. If the ramdisk +fills up, new sessions automatically shift to SSD — without interrupting anything +already playing. When pressure drops, new sessions shift back. + +> **This system has two subtle configuration requirements that are not obvious and +> both were discovered the hard way in production.** The Docker mount must use +> `bind-propagation=shared` or symlink flips are silently ignored after the first +> flip. The `transcoding-temp` directory must be pre-created on the ramdisk or Emby +> finds the SSD version and routes all sessions there until restarted. Both are +> documented and both will bite you if missed. --- -## The Problem - -Emby transcodes video on the fly for clients that can't play the source format directly. Each transcode session generates hundreds of small HLS segment files that are written and read continuously. Where those files live has a significant impact on performance: - -- **Hard drives** — too slow for simultaneous multi-stream transcoding. Seek times cause buffering. -- **SSD (cache pool)** — fast enough, but constant small file writes accelerate wear over time. -- **RAM (tmpfs)** — fastest possible, no wear, disappears cleanly when sessions end. - -A ramdisk is the ideal transcode location. The only risk is running out of RAM during heavy load — which is where this system comes in. +## ━━━ THE PROBLEM THAT BUILT THIS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ --- -## The Design +### 🔴 Three Storage Options, None Perfect on Their Own -### Symlink Indirection +Emby transcodes generate hundreds of small HLS segment files written and read +continuously at high throughput. Where those files live matters a lot: -Emby is pointed at a fixed path that never changes: - -``` -/mnt/ram-transcode → [currently: /mnt/ramdisk_transcodes] +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Hard drives: +# Seek times cause buffering on simultaneous multi-stream transcoding. +# 5 streams trying to seek on spinning disks = constant buffering for everyone. +# +# SSD (cache pool): +# Fast enough for any realistic load. +# But: constant small file writes at Emby volume accelerate SSD wear. +# A busy Live TV night writes and deletes thousands of segment files. +# Over months, this adds up. +# +# RAM (tmpfs): +# Fastest possible — no disk I/O at all. +# No wear — RAM doesn't have write cycles. +# Files disappear instantly on session end — no cleanup needed for normal exits. +# One risk: running out of RAM during heavy load. +# ───────────────────────────────────────────────────────────────────────────── ``` -This is a symlink. Emby doesn't know or care what's on the other end — it just writes to `/mnt/ram-transcode`. The transcode manager controls where that path actually points by updating the symlink target. - -**The critical insight:** ffmpeg resolves the symlink path **once** at session start. After that, it has a direct reference to the actual directory. This means: - -- **Existing sessions are never affected by symlink changes** -- When the symlink flips from ramdisk to SSD, sessions already in progress keep writing to ramdisk until they end naturally -- Only **new** sessions care about where the symlink currently points - -This is what makes the fallback seamless. Users never experience a glitch. +The correct answer is RAM by default, SSD as a safety net. The ramdisk handles normal +operation. The SSD absorbs unexpected load spikes. The system manages the transition +automatically. --- -## Docker Mount — Critical +### 🔴 Changing Transcode Location Requires Restarting Emby -**This must be configured correctly or the symlink system will not work.** +The obvious approach — configure Emby to use the ramdisk, configure SSD as fallback in +Emby's settings — requires restarting Emby to switch between them. Restarting Emby +during active streams drops everyone. A 7-person household with 5 Live TV streams +running at 9pm is not a good moment to restart Emby. -Emby must be configured using `--mount` in Extra Parameters — **not** as a standard path mapping in the unRAID template. +The fix: symlink indirection. Emby points at a fixed path that never changes. +The symlink target changes. ffmpeg resolves the symlink once at session start and +holds the resolved path — existing sessions are completely unaffected by symlink +changes. Only new sessions care about where the symlink currently points. -**In Emby Extra Parameters:** +The transition is seamless. Sessions in progress when the flip happens continue +writing to wherever they started. Only new sessions after the flip go to the new +location. No restart. No interruption. + +--- + +### 🔴 Docker Bind Mount Silently Ignored After First Flip + +Got the symlink system working. First flip from ramdisk to SSD: works. Flip back to +ramdisk: nothing. All new sessions still land on SSD. The symlink on the host clearly +points at the ramdisk — `readlink /mnt/ram-transcode` shows the correct path — but +Emby keeps writing to SSD. + +The cause: Docker's default bind mount uses `rprivate` propagation. With `rprivate`, +Docker resolves the symlink target at first mount and locks that inode. When the +symlink flips, Docker ignores it — it already has a private SSD binding locked in. +The container sees the path but the mount behind it doesn't update. + +The fix: `bind-propagation=shared` in Extra Parameters. With shared propagation, host +mount changes propagate into the container in real time. Symlink flips on the host are +immediately visible inside the container. This requires using `--mount` syntax instead +of a standard template path mapping — that syntax supports propagation, the path +mapping UI does not. + +--- + +### 🔴 Sessions Drifting to SSD After a Day of Operation + +System working correctly for hours. Then gradually sessions start landing on SSD even +though the ramdisk has plenty of space and the symlink points at the ramdisk. Next day +all sessions are on SSD. + +The cause: cleanup was removing the empty `transcoding-temp` directory from the +ramdisk. When `transcoding-temp` doesn't exist on the ramdisk, Emby searches its +accessible paths for an existing one. It finds the SSD fallback version. All subsequent +sessions route there until Emby is restarted. + +The fix: `transcoding-temp` is protected from cleanup — excluded by name from the +`find` command. And `ramdisk_setup.sh` pre-creates it at mount time so Emby always +finds it on the ramdisk first. Both fixes together prevent this permanently. + +--- + +### 🔴 lsof Per File on a Live TV System + +Early cleanup implementation called `lsof filename` per file to check if anything had +it open. On a busy Live TV night with 5 simultaneous streams, the ramdisk contains +thousands of HLS segment files. Calling `lsof` once per file was creating thousands +of subprocess calls every 3 minutes. The cleanup script was spending more time on +lsof calls than on actual cleanup. + +The fix: `lsof` is called once per location to build a complete open-file map. All +subsequent file checks are O(1) lookups against that pre-built map. Thousands of files, +one `lsof` call, no performance penalty. + +--- + +## ━━━ THE DESIGN ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +### ── The Symlink Architecture ───────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Emby is configured to write transcodes to TRANSCODE_LINK. +# TRANSCODE_LINK is a symlink — its target is managed at runtime. +# +# Normal operation (ramdisk has headroom): +# /mnt/ram-transcode → /mnt/ramdisk_transcodes/ (ramdisk, fast, no wear) +# +# Heavy load (ramdisk filling up): +# /mnt/ram-transcode → /mnt/cache/Temp_Storage/Emby/Transcodes/ (SSD) +# +# Emby doesn't know this symlink exists. It writes to /mnt/ram-transcode. +# ffmpeg resolves the symlink ONCE when a session starts. +# After that it holds a direct reference to the actual directory. +# Flipping the symlink has ZERO effect on sessions already in progress. +# Only NEW sessions care about where the symlink currently points. +# ───────────────────────────────────────────────────────────────────────────── ``` + +--- + +### ── The Threshold Logic ────────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Two thresholds with a hysteresis gap to prevent flip-flopping. +# + RAMDISK_WARN_GB=8.8 # flip to SSD when ramdisk usage exceeds this + RAMDISK_LOW_GB=6.5 # flip back to ramdisk when usage drops below this +# +# The 2.3GB hysteresis gap: +# Without this gap: usage hovers at 8.7GB → flip to SSD → sessions drain +# → usage drops to 8.5GB → flip back → new sessions fill → flip again +# The symlink would oscillate every few minutes under moderate load. +# +# With the gap: usage must drop all the way to 6.5GB before flipping back. +# That requires multiple sessions to end completely — a genuine recovery, +# not a brief fluctuation. Stable, predictable behaviour. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── What Lives Where ────────────────────────────────────────────────────────── + +``` +/mnt/ramdisk_transcodes/ ← tmpfs (HOST*_RAMDISK_SIZE ceiling) + transcoding-temp/ ← pre-created by ramdisk_setup.sh — always here + E0D8DC/ ← Emby session (Live TV HLS segments) + F1A9BB/ ← another session + +/mnt/ram-transcode ← symlink — managed at runtime by transcode_manager.sh + currently points at: /mnt/ramdisk_transcodes/ + +/mnt/cache/Temp_Storage/Emby/Transcodes/ ← SSD fallback + transcoding-temp/ ← also pre-created — Emby finds ramdisk version first + xyz789/ ← sessions that started when ramdisk was full +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ⚙️ DOCKER MOUNT — READ THIS BEFORE ANYTHING ELSE +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +> **This is the most important configuration requirement in the entire folder.** +> Get this wrong and symlink flips silently stop working after the first flip. +> The system appears to work initially and fails subtly. + +--- + +### ── Required Mount Configuration ─────────────────────────────────────────── + +```bash +# In Emby's Extra Parameters in the unRAID Docker template: +# ───────────────────────────────────────────────────────────────────────────── --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared +# ───────────────────────────────────────────────────────────────────────────── +# This REPLACES the transcode path in the standard template path mapping UI. +# Do NOT add this via the path mapping UI — that UI does not support propagation. +# Use Extra Parameters only. ``` -**Why `shared` propagation is required:** +--- -Standard bind mounts in unRAID use `rprivate` propagation by default. With `rprivate`, Docker resolves the symlink target once — at the moment of the first mount change — and locks that inode for the lifetime of the container. When the symlink flips from ramdisk to SSD, Docker takes a private copy of that SSD binding. When the symlink later flips back to ramdisk, the container ignores it — it already has a private SSD binding locked in. All new sessions land on SSD permanently until Emby restarts. +### ── Why `shared` Is Required ──────────────────────────────────────────────── -With `shared` propagation, host mount changes propagate into the container in real time. Symlink flips on the host are immediately visible inside the container. The system works as designed. - -**Verify the mount is configured correctly:** ```bash +# ───────────────────────────────────────────────────────────────────────────── +# rprivate (Docker's default): +# Docker resolves the symlink target at first mount and locks that inode. +# Flip: ramdisk → SSD → flip works. +# Flip: SSD → ramdisk → Docker ignores it. Container still sees SSD binding. +# All sessions continue to land on SSD forever until Emby restarts. +# Symptom: symlink on host is correct, Emby still uses SSD. Confusing. +# +# shared: +# Host mount changes propagate into the container in real time. +# Flip: ramdisk → SSD → visible in container immediately ✅ +# Flip: SSD → ramdisk → visible in container immediately ✅ +# System works as designed. Every flip is immediate and correct. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Verify the Mount ───────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Check the propagation — must show "shared" not "rprivate": docker inspect Emby | grep -A4 "ext-ram" -# Should show: "Propagation": "shared" -# NOT: "Propagation": "rprivate" -``` +# Expected output includes: "Propagation": "shared" +# Wrong output: "Propagation": "rprivate" -**Do NOT add a static SSD transcode path as a second mount.** If the SSD path is mounted inside the container, Emby can see it as an accessible transcode location and will route sessions there independently of the symlink — completely bypassing the management system. - -> **Both of these issues were discovered in production.** The static SSD mount caused sessions to bypass the symlink. The rprivate propagation caused sessions to lock onto SSD after the first flip. Both are now fixed in the correct configuration. - ---- - -## The Scripts - -### `ramdisk_setup.sh` -**Run at array start. Run once.** - -Creates the tmpfs ramdisk, the SSD fallback directory, the symlink, and — critically — the `transcoding-temp` subdirectory on the ramdisk. - -```bash -# Scheduled as: At Startup of Array -/mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh -``` - -**Why `transcoding-temp` must be pre-created:** - -Emby creates a `transcoding-temp` subdirectory inside its configured transcode path when it first needs to write. If `transcoding-temp` doesn't exist on the ramdisk, Emby may find and use an existing one on the SSD fallback path instead — locking all sessions onto SSD until Emby restarts. - -`ramdisk_setup.sh` creates `transcoding-temp` on the ramdisk at mount time so Emby always finds it there first. - -What it creates: -``` -/mnt/ramdisk_transcodes/ ← tmpfs mount (RAMDISK_SIZE ceiling) -/mnt/ramdisk_transcodes/transcoding-temp ← pre-created so Emby uses ramdisk -/mnt/ram-transcode ← symlink pointing at ramdisk -/mnt/cache/Temp_Storage/Emby/Transcodes/ ← SSD fallback directory -``` - -After running, verify: -```bash -mountpoint /mnt/ramdisk_transcodes # should say "is a mountpoint" -readlink /mnt/ram-transcode # should show /mnt/ramdisk_transcodes -ls /mnt/ramdisk_transcodes/ # should show transcoding-temp/ +# Check Emby's transcode path setting (inside container): +docker exec Emby cat /config/config/encoding.xml | grep TranscodingTempPath +# Expected: /ext-ram-transcode +# ───────────────────────────────────────────────────────────────────────────── ``` --- -### `transcode_management.sh` (Orchestrators/) -**Run every 3 minutes via cron. Replaces separate manager and cleanup cron entries.** - -Runs `transcode_cleanup.sh` first then `transcode_manager.sh` in the correct order. Cleanup runs first so the manager sees accurate post-cleanup usage before making threshold decisions. +### ── What NOT to Do ────────────────────────────────────────────────────────── ```bash -# Scheduled as: */3 * * * * -/mnt/user/appdata/unraid_scripts/Orchestrators/transcode_management.sh +# ───────────────────────────────────────────────────────────────────────────── +# DO NOT add a static SSD transcode path as a second path mapping in the template: +# /mnt/cache/Temp_Storage/Emby/Transcodes → /ssd-transcode +# +# If the SSD path is mounted inside the container, Emby can see it as an +# accessible transcode location. It will route sessions there independently of +# the symlink — completely bypassing the management system. Sessions land on SSD +# regardless of symlink state. The whole system stops working. +# ───────────────────────────────────────────────────────────────────────────── ``` -Also tracks daily transcode statistics to `/boot/config/transcode_daily.db` — read by `weekly_health_digest.sh` for the weekly report. - --- -### `transcode_manager.sh` -**Called by `transcode_management.sh` — not scheduled directly.** +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🚀 ramdisk_setup.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Monitors ramdisk usage and manages the symlink direction. +Creates the ramdisk, SSD fallback directory, symlink, and `transcoding-temp` on the +ramdisk. Run once at array start. Idempotent — if the ramdisk is already mounted it +reports status and exits cleanly. -#### Operating Modes - -Set `TRANSCODE_MANAGER_MODE` in `Master.conf`: - -| Mode | Behavior | Use When | -|------|----------|----------| -| `smart` | Auto-flips between ramdisk and SSD based on thresholds | Normal operation — default | -| `ramdisk` | Always uses ramdisk, never flips to SSD | Light load, guaranteed RAM performance | -| `ssd` | Always uses SSD, never uses ramdisk | Maintenance, post-flip drain | - -#### Smart Mode — How the Flip Works - -``` -Ramdisk usage rises above RAMDISK_WARN_GB (8.8GB) - → Symlink flips to SSD - → New sessions land on SSD - → Existing sessions continue on ramdisk until they end - -Ramdisk usage drops below RAMDISK_LOW_GB (6.5GB) - → Symlink flips back to ramdisk - → New sessions land on ramdisk again +```bash +# Scheduled: At Startup of Array (via array_start.sh) +# This runs BEFORE Emby starts — order matters in ARRAY_START_SCRIPTS ``` -The 2.3GB gap between `RAMDISK_WARN_GB` and `RAMDISK_LOW_GB` is the **hysteresis gap**. It prevents the symlink from flip-flopping when usage hovers near the threshold. +--- -#### Safety Checks - -Every run, regardless of mode: - -| Condition | Action | -|-----------|--------| -| Ramdisk not mounted | Flip to SSD immediately, notify warning | -| SSD path missing | Disable fallback, notify warning | -| Symlink missing | Recreate pointing at ramdisk, notify | -| Symlink target gone | Reset to ramdisk, notify | -| `transcoding-temp` missing from ramdisk | Create it — prevents Emby falling back to SSD | -| Permissions drift | Fix silently every run | -| Emby not running | Skip threshold checks, verify symlink only | - -#### Session Display - -Each run queries the Emby API and shows active streams: +### ── What It Creates ────────────────────────────────────────────────────────── +```bash +# ───────────────────────────────────────────────────────────────────────────── +# 1. RAMDISK_PATH (/mnt/ramdisk_transcodes) +# mount -t tmpfs -o size=HOST*_RAMDISK_SIZE tmpfs /mnt/ramdisk_transcodes +# In-memory tmpfs — uses only as much RAM as actually needed. +# RAMDISK_SIZE is a ceiling, not a reservation — an empty ramdisk uses ~0 RAM. +# +# 2. transcoding-temp inside the ramdisk +# mkdir -p /mnt/ramdisk_transcodes/transcoding-temp +# Pre-created so Emby always finds it here first. +# Without this: Emby creates transcoding-temp at its first writable location, +# which may be the SSD fallback even when the symlink points at the ramdisk. +# chown nobody:users — correct ownership for Emby to write as PUID=99 +# +# 3. TRANSCODE_SSD (/mnt/cache/Temp_Storage/Emby/Transcodes/) +# mkdir -p — creates if missing, silent if exists +# Also pre-creates transcoding-temp/ inside SSD fallback for consistency +# +# 4. TRANSCODE_LINK (/mnt/ram-transcode) +# ln -sfn /mnt/ramdisk_transcodes /mnt/ram-transcode +# Always reset to ramdisk at array start — clean state every boot +# ───────────────────────────────────────────────────────────────────────────── ``` + +--- + +### ── Verify After Setup ─────────────────────────────────────────────────────── + +```bash +# Run these after first setup — all three must pass: + +mountpoint /mnt/ramdisk_transcodes +# Expected: /mnt/ramdisk_transcodes is a mountpoint + +readlink /mnt/ram-transcode +# Expected: /mnt/ramdisk_transcodes + +ls /mnt/ramdisk_transcodes/ +# Expected: transcoding-temp/ +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +ramdisk_setup.sh # normal run (at array start via array_start.sh) +ramdisk_setup.sh --dry-run # show what would be created without creating +ramdisk_setup.sh --status # show current ramdisk, symlink, and SSD state +ramdisk_setup.sh --log # verbose — show each creation step +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔄 transcode_manager.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Monitors ramdisk usage and manages the symlink direction. Called by +`transcode_management.sh` — not scheduled directly. Every 3 minutes it checks usage, +makes a flip decision if needed, runs safety checks, and shows active sessions. + +```bash +# Called by: transcode_management.sh (every 3 minutes) +# Not scheduled directly — use transcode_management.sh +``` + +--- + +### ── Three Modes ────────────────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── + TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd +# +# smart: +# Auto-flips between ramdisk and SSD based on usage thresholds. +# Normal operation — use this in production. +# Ramdisk above RAMDISK_WARN_GB → flip to SSD. +# Ramdisk below RAMDISK_LOW_GB → flip back to ramdisk. +# +# ramdisk: +# Always uses ramdisk. Never flips to SSD. +# Use: light load server, guaranteed RAM performance, testing ramdisk behaviour. +# Warning logged if usage exceeds threshold — no automatic action. +# +# ssd: +# Always uses SSD. Never uses ramdisk. +# Use: post-flip drain (waiting for ramdisk sessions to end naturally), +# maintenance windows, ramdisk capacity testing. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Safety Checks — Every Run Regardless of Mode ─────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# These run on EVERY cycle — they protect against state drift: +# +# Symlink missing or broken +# → Recreate pointing at ramdisk, notify +# → Can happen after manual intervention or filesystem issue +# +# Ramdisk disappeared (unmounted) +# → Auto-flip to SSD immediately, notify warning +# → Can happen if tmpfs was manually unmounted or system ran out of memory +# +# SSD path missing +# → Disable SSD fallback (mode=ssd: error) +# → Can happen if SSD pool is not mounted +# +# transcoding-temp missing from ramdisk +# → Recreate immediately, no notification +# → Prevents sessions silently routing to SSD version +# +# Permissions drift +# → Fix silently every run +# → nobody:users ownership, TRANSCODE_CHMOD mode +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Session Display ────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Each run shows active sessions from all configured TRANSCODE_SERVERS: +# ━━━ 🎬 Active Emby Sessions ━━━ 🎬 Total: 7 | 💨 Live TV: 5 | 🔄 Transcoding: 5 | 🏁 Direct: 2 🔗 Storage: 💨 ramdisk @@ -173,184 +420,305 @@ Each run queries the Emby API and shows active streams: 🎬 Sunny — ABC (WTAE) — Live TV — Transcode 🎬 Mama Bear — Cinemax — Live TV — Transcode 🎬 Gmer4Lfe — WAN Show — TV Show — Transcode -``` -**Split state** is detected and displayed when sessions exist on both ramdisk and SSD simultaneously — normal during a symlink flip: - -``` +# Split state — sessions on both ramdisk and SSD simultaneously: +# Normal during a flip — ramdisk sessions draining, new sessions on SSD +# ⚠️ Split state — 4 folder(s) on ramdisk / 2 on SSD 🔗 Storage: 💨 ramdisk (4) + 💾 SSD (2) +# ───────────────────────────────────────────────────────────────────────────── ``` --- -### `transcode_cleanup.sh` -**Called by `transcode_management.sh` — not scheduled directly.** - -Removes old inactive transcode files from both ramdisk and SSD. Never deletes files that are currently open by any process. - -**Deletion rules — a file is eligible only when ALL are true:** -1. Older than `TRANSCODE_MAX_AGE` minutes -2. Not currently open by any process - -**`transcoding-temp` directory is protected from deletion.** Even when empty, `transcoding-temp` is never removed by cleanup. Deleting it causes Emby to fall back to the SSD version on next session start — this was the root cause of sessions drifting to SSD after a day of operation. - -**Performance design:** `lsof` is called once per location to build a complete open file list — not once per file. On a busy Live TV system with thousands of HLS segments this is critical for performance. - ---- - -## Configuration - -All configuration in `Master.conf` under `── TRANSCODES ──`: - -```bash -RAMDISK_PATH="/mnt/ramdisk_transcodes" -RAMDISK_SIZE="10G" # bumped from 8G — peak usage ~5.2GB on busy nights -TRANSCODE_LINK="/mnt/ram-transcode" -TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" - -RAMDISK_WARN_GB=8.8 # flip to SSD above this — 1.2GB headroom from ceiling -RAMDISK_LOW_GB=6.5 # flip back to ramdisk below this — 2.3GB hysteresis gap -RAMDISK_SSD_MIN_GB=20 # minimum SSD free space before allowing flip - -TRANSCODE_MAX_AGE=20 # minutes before file eligible for cleanup -TRANSCODE_ORPHAN_AGE=30 # minutes for orphaned files - -TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times per hour - -TRANSCODE_OWNER="nobody:users" -TRANSCODE_CHMOD="755" - -TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd - -TRANSCODE_CHECK_EMBY=true # skips threshold checks when no servers active - -TRANSCODE_DAILY_LOG="/boot/config/transcode_daily.db" -TRANSCODE_LOG_RETENTION=90 -``` - -### Multi-Server Configuration — `TRANSCODE_SERVERS` - -The transcode manager supports any number of media servers sharing the same ramdisk scratch space. One server or ten — the same script handles it. The threshold decision is based on total ramdisk usage across all servers combined. +### ── Multi-Server Configuration ───────────────────────────────────────────── ```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Any number of media servers can share the same ramdisk scratch space. +# They never touch each other's files — each writes to its own session subfolder. +# Format: "ContainerName|URL|APIKey|Type" +# TRANSCODE_SERVERS=( - "Emby|http://localhost:8096|your-api-key|emby" - # "Emby-Jayred365|http://HOST2-IP:8096|his-api-key|emby" # temporarily cover his Emby - # "Jellyfin|http://localhost:8097|jellyfin-api-key|jellyfin" # test Jellyfin instance - # "Plex|http://localhost:32400|plex-token|plex" # Plex if needed + "Emby|http://localhost:8096|0c27448d93a7431f9ac63569f7655829|emby" + + # Temporarily cover HOST2's Emby during maintenance: + # "Emby-Jayred365|http://100.x.x.x:8096|HOST2-api-key|emby" + + # Jellyfin instance (separate port): + # "Jellyfin|http://localhost:8097|jellyfin-api-key|jellyfin" ) +# +# Type field controls which API endpoint format is used: +# emby → /Sessions endpoint +# jellyfin → /Sessions endpoint (same format, same code path) +# plex → /status/sessions (different format) +# +# ⚠️ Tdarr does NOT belong here. +# Tdarr encodes full video files — large working files would fill the ramdisk +# rapidly and cause constant flips. Tdarr belongs on SSD permanently. +# dedicated tdarr_cleanup.sh handles Tdarr orphan management separately. +# +# Entries with placeholder API keys are skipped automatically. +# Comment out unused entries rather than deleting — placeholders show what's available. +# ───────────────────────────────────────────────────────────────────────────── ``` -**Format:** `"ContainerName|URL|APIKey|Type"` - -| Field | Description | -|-------|-------------| -| `ContainerName` | Exact Docker container name — used for running check | -| `URL` | API base URL including port | -| `APIKey` | Server API key or token | -| `Type` | `emby` \| `jellyfin` \| `plex` — controls API endpoint format | - -**How it works with multiple servers:** - -Each media server writes its transcode sessions to its own subfolder inside `transcoding-temp`: - -``` -/mnt/ram-transcode/transcoding-temp/ - E0D8DC/ ← Emby session (Live TV segment files) - abc123/ ← Jellyfin session - xyz789/ ← his Emby session -``` - -They never touch each other's files. The ramdisk is shared scratch space — each server is unaware of the others. The manager aggregates session counts from all servers and makes one threshold decision based on total ramdisk usage. - -**Each server container needs the shared propagation mount in Extra Parameters:** -``` ---mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared -``` - -Set the transcode path in each server's settings to `/ext-ram-transcode` (or whatever target path you use). - -**Entries with placeholder API key values are skipped automatically** — comment out entries that aren't in use rather than removing them. The placeholders make it clear what's available to enable. - -**⚠️ Tdarr does NOT belong in this array.** Tdarr encodes full files — large working files would fill the ramdisk rapidly and cause constant flips. Keep Tdarr on SSD. A dedicated `tdarr_cleanup.sh` handles Tdarr orphan management separately. - -### Sizing the Ramdisk - -`tmpfs` only uses RAM actually needed — `RAMDISK_SIZE` is a ceiling, not a reservation. - -**Production data from this setup:** -``` -Normal load (2-3 streams) → ~1.5-2.0GB -Busy evening (5-6 streams) → ~3.5-4.5GB -Peak (8 streams, live TV) → ~5.2GB -Current ramdisk → 10G with 8.8GB threshold -``` - -### Sizing Thresholds - -When adjusting `RAMDISK_SIZE`, adjust thresholds to match. Keep a 1.5-2.5GB hysteresis gap between WARN and LOW: - -| Ramdisk Size | RAMDISK_WARN_GB | RAMDISK_LOW_GB | -|-------------|-----------------|----------------| -| 6G | 4.8 | 3.5 | -| 8G | 6.8 | 5.5 | -| 10G | 8.8 | 6.5 | -| 12G | 10.5 | 8.5 | - --- -## Scheduled Run Summary +### ── Sizing the Ramdisk ─────────────────────────────────────────────────────── -| Script | Schedule | Purpose | -|--------|----------|---------| -| `ramdisk_setup.sh` | At Startup of Array | Create ramdisk, symlink, transcoding-temp | -| `transcode_management.sh` | `*/3 * * * *` | Cleanup then manager — correct order, daily stats | - -`transcode_manager.sh` and `transcode_cleanup.sh` are called by `transcode_management.sh` — do not schedule them separately. - ---- - -## Troubleshooting - -**Sessions landing on SSD despite symlink pointing at ramdisk:** - -1. Check Docker mount propagation: - ```bash - docker inspect Emby | grep Propagation - # Must show: "shared" not "rprivate" - ``` - Fix: Add `--mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared` to Extra Parameters and restart Emby. - -2. Check `transcoding-temp` exists on ramdisk: - ```bash - ls /mnt/ramdisk_transcodes/ - # Must show: transcoding-temp/ - ``` - Fix: `mkdir -p /mnt/ramdisk_transcodes/transcoding-temp && chown nobody:users /mnt/ramdisk_transcodes/transcoding-temp` - -3. Check for duplicate SSD mount in Emby template — remove any static SSD transcode path mapping. - -**`[LOG] Permissions fixed` on every run:** - -Permissions are applied every run regardless — this is by design. If it logs every cycle it means Emby is resetting permissions on write. Not harmful — just informational. - -**Flip count high — 3+ per hour:** - -Ramdisk filling up regularly. Consider increasing `RAMDISK_SIZE` by 2GB and adjusting thresholds accordingly. - ---- - -## Emergency Manual Flip - -If you need to manually redirect all new transcodes to SSD: ```bash +# master_host1.conf +# ───────────────────────────────────────────────────────────────────────────── +# tmpfs uses only as much RAM as actually needed — RAMDISK_SIZE is a ceiling. +# An empty ramdisk uses essentially zero RAM. +# + HOST1_RAMDISK_SIZE="10G" # 10GB ceiling — verified against production usage below + +# Production data from this setup (7-household Live TV system): +# Normal (2-3 streams) → ~1.5-2.0GB +# Busy evening (5-6 streams) → ~3.5-4.5GB +# Peak (8 streams, Live TV) → ~5.2GB +# Current ramdisk → 10G with 8.8GB threshold → 1.2GB safety headroom +# ───────────────────────────────────────────────────────────────────────────── + +# Threshold sizing — keep ~1.5-2.5GB hysteresis gap: +# +# ┌──────────────┬─────────────────┬────────────────┐ +# │ RAMDISK_SIZE │ RAMDISK_WARN_GB │ RAMDISK_LOW_GB │ +# ├──────────────┼─────────────────┼────────────────┤ +# │ 6G │ 4.8 │ 3.5 │ +# │ 8G │ 6.8 │ 5.5 │ +# │ 10G │ 8.8 │ 6.5 │ +# │ 12G │ 10.5 │ 8.5 │ +# └──────────────┴─────────────────┴────────────────┘ +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🧹 transcode_cleanup.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Removes stale transcode files from both ramdisk and SSD fallback locations. Called by +`transcode_management.sh` before `transcode_manager.sh` — order is critical. + +```bash +# Called by: transcode_management.sh (cleanup runs BEFORE manager) +# Not scheduled directly — use transcode_management.sh +``` + +--- + +### ── Deletion Rules ──────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# A file is eligible for deletion only when ALL conditions are true: +# +# 1. Older than TRANSCODE_MAX_AGE minutes (mtime — last write time) +# Active segments are being written every few seconds. +# A file not touched in 20 minutes is from a session that ended. +# +# 2. Not currently open by any process +# lsof map built once per location — O(1) lookup per file. +# If ffmpeg has a file open, it is not deleted regardless of age. +# "Session ended in API but ffmpeg still writing" → safe, not deleted. +# +# transcoding-temp directory: +# NEVER deleted, even when empty. +# Protected by name exclusion in find command. +# Deleting it causes Emby to find the SSD version and route there permanently. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Why Not Session-Aware Cleanup ──────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# ffmpeg generates its own folder names inside transcoding-temp, independently +# of the media server API session IDs. There is no reliable mapping between +# API session IDs and the actual folder names on disk. +# +# Attempting to correlate them: session E0D8DC → folder E0D8DC → safe assumption? +# No. The folder name is an internal ffmpeg identifier. It may match, may not. +# Using this correlation would falsely treat active sessions as ended. +# +# lsof is the correct check: +# If ffmpeg has a file open, the file is active regardless of session state. +# If no process has the file open, the file is safely deletable. +# No correlation needed. No race condition. Always correct. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Flip-Back After Cleanup ────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# transcode_cleanup.sh checks ramdisk usage AFTER removing stale files. +# If usage dropped below RAMDISK_LOW_GB → triggers flip-back to ramdisk. +# This handles the recovery direction so transcode_manager.sh doesn't have to. +# +# Without cleanup running first, the manager would see inflated usage from stale +# files and potentially flip to SSD unnecessarily. +# With cleanup running first, the manager always sees real active session usage. +# This is why the order in transcode_management.sh is non-negotiable. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +## ━━━ FULL CONFIGURATION REFERENCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── + +# ── Paths ────────────────────────────────────────────────────────────────── + TRANSCODE_LINK="/mnt/ram-transcode" # the symlink Emby points at + TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" # SSD fallback + +# ── Per-Host (master_host*.conf) ────────────────────────────────────────── + HOST1_RAMDISK_PATH="/mnt/ramdisk_transcodes" # ramdisk mount point + HOST1_RAMDISK_SIZE="10G" # tmpfs ceiling (not a reservation) + HOST1_RAMDISK_WARN_GB=8.8 # flip to SSD above this + HOST1_RAMDISK_LOW_GB=6.5 # flip back below this + +# ── Thresholds ───────────────────────────────────────────────────────────── + RAMDISK_SSD_MIN_GB=20 # minimum SSD free space before allowing SSD flip + # prevents filling the SSD cache pool accidentally + TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in an hour + # high flip count = ramdisk undersized + +# ── Cleanup ──────────────────────────────────────────────────────────────── + TRANSCODE_MAX_AGE=20 # minutes — files older than this are stale + TRANSCODE_ORPHAN_AGE=30 # minutes — orphaned session folders removed after this + +# ── Manager Mode ─────────────────────────────────────────────────────────── + TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd + TRANSCODE_CHECK_EMBY=true # skip threshold checks when Emby not running + # prevents unnecessary flips at night + +# ── Permissions ──────────────────────────────────────────────────────────── + TRANSCODE_OWNER="nobody:users" # matches PUID=99 PGID=100 container env + TRANSCODE_CHMOD="755" + +# ── Daily Log ────────────────────────────────────────────────────────────── + TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db" + TRANSCODE_LOG_RETENTION=90 # days — bounded, trimmed on every write + TRANSCODE_STATE_FILE="/tmp/transcode_state.db" # /tmp — resets on reboot + +# ── Multi-Server ─────────────────────────────────────────────────────────── + TRANSCODE_SERVERS=( + "ContainerName|http://host:port|api-key|type" # emby|jellyfin|plex + ) +``` + +--- + +## ━━━ SCHEDULE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +``` +At Startup of Array: + ramdisk_setup.sh ← via array_start.sh — creates ramdisk + symlink + transcoding-temp + +Every 3 minutes: + transcode_management.sh ← cleanup first, then manager — order non-negotiable + 1. transcode_cleanup.sh ← remove stale files, check if flip-back possible + 2. transcode_manager.sh ← check usage, flip if needed, show sessions + +Do NOT schedule transcode_manager.sh or transcode_cleanup.sh directly. +``` + +--- + +## ━━━ TROUBLESHOOTING ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +--- + +### 🔴 Sessions Landing on SSD Despite Symlink Pointing at Ramdisk + +```bash +# Check 1 — Docker mount propagation (most common cause): +docker inspect Emby | grep Propagation +# Expected: "Propagation": "shared" +# Wrong: "Propagation": "rprivate" +# +# Fix: Add to Emby Extra Parameters and restart Emby: +# --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared +# Remove any standard path mapping for the transcode directory. + +# Check 2 — transcoding-temp exists on ramdisk: +ls /mnt/ramdisk_transcodes/ +# Expected: transcoding-temp/ +# +# Fix if missing: +mkdir -p /mnt/ramdisk_transcodes/transcoding-temp +chown nobody:users /mnt/ramdisk_transcodes/transcoding-temp + +# Check 3 — no duplicate SSD mount in Emby template: +docker inspect Emby | grep -A3 "Mounts" +# Should show: only /mnt/ram-transcode → /ext-ram-transcode +# Should NOT show: /mnt/cache/Temp_Storage/... as a second mount +``` + +--- + +### 🔴 Flip Count High — 3+ Per Hour + +```bash +# Ramdisk filling up regularly — sessions draining before more arrive. +# Check peak usage from the weekly coffee report: +# Transcodes section → "Week peak: X.XGB" +# +# If peak is close to RAMDISK_WARN_GB → increase ramdisk: +# master_host1.conf +HOST1_RAMDISK_SIZE="12G" # increase by 2G +HOST1_RAMDISK_WARN_GB=10.5 # adjust thresholds accordingly +HOST1_RAMDISK_LOW_GB=8.5 +# +# Then re-run ramdisk_setup.sh to remount at new size: +bash /mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh +``` + +--- + +### 🔴 Ramdisk Not Mounting at Array Start + +```bash +# Check if tmpfs mounted: +mountpoint /mnt/ramdisk_transcodes +# "not a mountpoint" → setup failed or not run yet + +# Run manually to see the error: +bash /mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh --log + +# Common causes: +# /mnt/ramdisk_transcodes directory missing → mkdir -p /mnt/ramdisk_transcodes +# Insufficient RAM → check free RAM: free -h +# RAMDISK_SIZE too large for available RAM → reduce HOST*_RAMDISK_SIZE +``` + +--- + +### 🔴 Emergency Manual Flip + +```bash +# Flip to SSD immediately — all new sessions go to SSD: ln -sfn /mnt/cache/Temp_Storage/Emby/Transcodes /mnt/ram-transcode -``` -To flip back to ramdisk: -```bash +# Flip back to ramdisk — all new sessions go to ramdisk: ln -sfn /mnt/ramdisk_transcodes /mnt/ram-transcode -``` -Existing sessions are unaffected. Only new sessions follow the new target. \ No newline at end of file +# Check current symlink target: +readlink /mnt/ram-transcode + +# Existing sessions in progress are NEVER affected by these changes. +# Only new sessions follow the new symlink target. +``` \ No newline at end of file diff --git a/Transcodes/ramdisk_setup.sh b/Transcodes/ramdisk_setup.sh index 6f2c15e..fcecbae 100644 --- a/Transcodes/ramdisk_setup.sh +++ b/Transcodes/ramdisk_setup.sh @@ -1,79 +1,136 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Ramdisk Setup Script --------------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= Ramdisk Setup ============================================== +# ============================================================================================== # Creates a tmpfs ramdisk for Emby transcodes and points the transcode symlink at it. # Run once at array start via User Scripts plugin — scheduled as "At Startup of Array". -# If ramdisk already mounted reports status and exits cleanly without remounting. +# If ramdisk is already mounted reports status and exits cleanly without remounting. # -# Creates: -# RAMDISK_PATH — tmpfs mount point (fast, in-memory transcode location) -# TRANSCODE_SSD — SSD fallback directory (created if missing) -# TRANSCODE_LINK — symlink pointing at ramdisk by default +# ── WHAT IT CREATES ─────────────────────────────────────────────────────────────────────────── +# RAMDISK_PATH — tmpfs mount point (in-memory transcode location) +# Size: HOST*_RAMDISK_SIZE (e.g. 8G) — must fit in available RAM +# TRANSCODE_SSD — SSD fallback directory (created if missing) +# transcode_manager.sh flips symlink here if ramdisk fills up +# TRANSCODE_LINK — symlink pointing at RAMDISK_PATH by default +# transcoding-temp/ — pre-created inside ramdisk so Emby always finds it there +# Without this Emby creates it at its own first-writable location +# which may be SSD even when symlink points at ramdisk # -# All configuration in Master.conf under Transcode Manager section. -# Supports --dry-run to preview what would be created without making changes. -# ----------------------------------------------------------------------------------------------- +# ── TRANSCODE_LINK SYMLINK ──────────────────────────────────────────────────────────────────── +# Emby's transcode path is set to TRANSCODE_LINK in Emby config. +# transcode_manager.sh flips the symlink between RAMDISK_PATH and TRANSCODE_SSD at runtime +# based on ramdisk usage — Emby sessions automatically follow without restart. +# This script always sets the link to RAMDISK_PATH at array start (clean state). +# +# ── STATE FILE ──────────────────────────────────────────────────────────────────────────────── +# Initialises /tmp/transcode_state.db with current target and flip tracking counters. +# /tmp resets on reboot — correct, transcode state is ephemeral. +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID and aliases HOST*_RAMDISK_SIZE → RAMDISK_SIZE. +# HOST*_RAMDISK_SIZE, HOST*_RAMDISK_WARN_GB, HOST*_RAMDISK_LOW_GB must all be +# configured per host — different servers have different amounts of RAM available. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# Root check — mount and symlink require root +# acquire_lock — prevents duplicate runs at array start +# detect_hosts() — correct RAMDISK_SIZE per host +# Already mounted — exits cleanly without remounting (idempotent) +# validate_unraid_cmd — notify validated before use +# Silent on success — startup script runs every boot — no noise when healthy +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_RAMDISK_SIZE — tmpfs size (e.g. 8G) — must change together with WARN_GB/LOW_GB +# HOST*_RAMDISK_WARN_GB — warn threshold in GB +# HOST*_RAMDISK_LOW_GB — flip to SSD threshold in GB +# HOST*_TRANSCODE_SSD — SSD fallback path +# HOST*_TRANSCODE_SERVERS — which servers run transcoding +# Aliased by detect_hosts() +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# TRANSCODE_LINK — symlink path Emby uses as transcode directory +# TRANSCODE_CHMOD — permissions applied to ramdisk and fallback +# TRANSCODE_OWNER — owner applied (default nobody:users) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# ramdisk_setup.sh — normal setup (runs at array start) +# ramdisk_setup.sh --dry-run — preview without making changes +# ramdisk_setup.sh --status — show current ramdisk and symlink state +# ramdisk_setup.sh --log — verbose output +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" - +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root" + error "Must be run as root — mount and symlink require root" exit 1 fi -success "Running as root" +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +acquire_lock + +# detect_hosts() sets MY_ID and aliases RAMDISK_SIZE, TRANSCODE_SSD etc. +detect_hosts + +log "Identity: $MY_ID ($LOCAL_SERVER_NAME)" +log "Ramdisk: $RAMDISK_PATH ($RAMDISK_SIZE)" +log "Fallback: $TRANSCODE_SSD" +log "Symlink: $TRANSCODE_LINK" + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_RAM Ramdisk path: $RAMDISK_PATH" - echo "$ICON_RAM Ramdisk size: $RAMDISK_SIZE" - echo "$ICON_LINK Transcode link: $TRANSCODE_LINK" - echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD" - echo "$ICON_GEAR Owner: $TRANSCODE_OWNER" - echo "$ICON_GEAR Mode: $TRANSCODE_MODE" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_RAM Ramdisk path: $RAMDISK_PATH" + echo "$ICON_RAM Ramdisk size: $RAMDISK_SIZE" + echo "$ICON_RAM Warn at: ${RAMDISK_WARN_GB}GB" + echo "$ICON_RAM Flip at: ${RAMDISK_LOW_GB}GB" + echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD" + echo "$ICON_LINK Symlink: $TRANSCODE_LINK" + echo "$ICON_GEAR Owner: $TRANSCODE_OWNER" + echo "$ICON_GEAR Mode: $TRANSCODE_CHMOD" + echo "" - # Show current mount state if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then - CURRENT_USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}') - CURRENT_AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}') - echo "$ICON_RAM Current usage: $CURRENT_USAGE used / $CURRENT_AVAIL available" + USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}') + AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}') + echo " $ICON_RAM Ramdisk: mounted — $USAGE used / $AVAIL available ✅" else - echo "$ICON_RAM Ramdisk: not mounted" + echo " $ICON_RAM Ramdisk: NOT mounted" fi if [[ -L "$TRANSCODE_LINK" ]]; then - echo "$ICON_LINK Current target: $(readlink "$TRANSCODE_LINK")" + TARGET=$(readlink "$TRANSCODE_LINK") + echo " $ICON_LINK Symlink: $TRANSCODE_LINK → $TARGET" else - echo "$ICON_LINK Symlink: not set" + echo " $ICON_LINK Symlink: not set" fi + echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_RAM Ramdisk Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Ramdisk ━━━ +# ============================================================================================== echo "" -echo "━━━ $ICON_RAM Ramdisk Setup ━━━" +echo "━━━ $ICON_RAM Ramdisk — $MY_ID ━━━" echo "$ICON_RAM Path: $RAMDISK_PATH" echo "$ICON_RAM Size: $RAMDISK_SIZE" echo "" @@ -81,39 +138,39 @@ echo "" START=$(date +%s) SETUP_SUCCESS=true -# Check if ramdisk already mounted if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then - CURRENT_USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}') - CURRENT_AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}') - info "$ICON_RAM Ramdisk already mounted — $CURRENT_USAGE used / $CURRENT_AVAIL available" - info "Skipping mount — verifying symlink and permissions" + USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}') + AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}') + log "Ramdisk already mounted — $USAGE used / $AVAIL available" + log "Skipping mount — verifying symlink and permissions" else - # Create mount point if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would create $RAMDISK_PATH" warn "DRY RUN — would mount tmpfs ${RAMDISK_SIZE} at $RAMDISK_PATH" else - info "Creating ramdisk mount point: $RAMDISK_PATH" + log "Creating ramdisk mount point: $RAMDISK_PATH" mkdir -p "$RAMDISK_PATH" || { error "Failed to create $RAMDISK_PATH" - notify "Ramdisk setup failed on $(hostname) — could not create mount point" "Ramdisk Setup" "warning" + notify "Ramdisk setup failed on $(hostname) ($MY_ID) — could not create mount point" \ + "Ramdisk Setup" "warning" exit 1 } - info "$ICON_RAM Mounting tmpfs ${RAMDISK_SIZE} at $RAMDISK_PATH..." + log "Mounting tmpfs ${RAMDISK_SIZE} at $RAMDISK_PATH..." if mount -t tmpfs -o size="$RAMDISK_SIZE" tmpfs "$RAMDISK_PATH"; then - success "Ramdisk mounted — ${RAMDISK_SIZE} at $RAMDISK_PATH" + warn "Ramdisk mounted — ${RAMDISK_SIZE} at $RAMDISK_PATH ✅" else error "Failed to mount ramdisk at $RAMDISK_PATH" - notify "Ramdisk setup failed on $(hostname) — mount failed" "Ramdisk Setup" "warning" + notify "Ramdisk setup failed on $(hostname) ($MY_ID) — mount failed" \ + "Ramdisk Setup" "warning" exit 1 fi fi fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_DISK SSD Fallback ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ SSD Fallback ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_DISK SSD Fallback ━━━" @@ -121,11 +178,11 @@ if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would create SSD fallback: $TRANSCODE_SSD" else if [[ -d "$TRANSCODE_SSD" ]]; then - info "$ICON_DISK SSD fallback already exists: $TRANSCODE_SSD" + log "SSD fallback already exists: $TRANSCODE_SSD" else - info "Creating SSD fallback directory: $TRANSCODE_SSD" + log "Creating SSD fallback directory: $TRANSCODE_SSD" if mkdir -p "$TRANSCODE_SSD"; then - success "SSD fallback created: $TRANSCODE_SSD" + log "SSD fallback created: $TRANSCODE_SSD ✅" else error "Failed to create SSD fallback: $TRANSCODE_SSD" SETUP_SUCCESS=false @@ -133,22 +190,21 @@ else fi fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_LINK Symlink ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Symlink ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_LINK Symlink ━━━" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would set $TRANSCODE_LINK → $RAMDISK_PATH" else - # Remove existing symlink or directory at link path if [[ -L "$TRANSCODE_LINK" ]]; then CURRENT_TARGET=$(readlink "$TRANSCODE_LINK") if [[ "$CURRENT_TARGET" == "$RAMDISK_PATH" ]]; then - info "$ICON_LINK Symlink already points to ramdisk — no change needed" + log "Symlink already points to ramdisk — no change needed ✅" else - info "$ICON_LINK Updating symlink: $CURRENT_TARGET → $RAMDISK_PATH" + log "Updating symlink: $CURRENT_TARGET → $RAMDISK_PATH" ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK" || { error "Failed to update symlink" SETUP_SUCCESS=false @@ -162,8 +218,7 @@ else SETUP_SUCCESS=false } else - info "Creating symlink: $TRANSCODE_LINK → $RAMDISK_PATH" - # Ensure parent directory exists + log "Creating symlink: $TRANSCODE_LINK → $RAMDISK_PATH" mkdir -p "$(dirname "$TRANSCODE_LINK")" ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK" || { error "Failed to create symlink" @@ -171,19 +226,15 @@ else } fi - if [[ "$SETUP_SUCCESS" == true ]]; then - success "$ICON_LINK $TRANSCODE_LINK → $RAMDISK_PATH" - fi + [[ "$SETUP_SUCCESS" == true ]] && log "Symlink: $TRANSCODE_LINK → $RAMDISK_PATH ✅" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Transcoding-temp Directory ━━━ -# Creates the transcoding-temp folder on the ramdisk proactively. -# If this folder doesn't exist on the ramdisk Emby creates it wherever it finds -# a writable path first — which may be the SSD fallback — locking all sessions -# onto SSD even when the symlink points at the ramdisk. -# Creating it here guarantees Emby always finds it on the ramdisk at session start. -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Transcoding-temp Directory ━━━ +# ============================================================================================== +# Pre-created inside ramdisk so Emby always finds it there at session start. +# Without this Emby creates it at its own first-writable path — which may be +# SSD even when the symlink points at the ramdisk — locking all sessions onto SSD. echo "" echo "━━━ $ICON_GEAR Transcoding Temp Directory ━━━" @@ -193,42 +244,44 @@ if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would create $TRANSCODE_TEMP_DIR" else if [[ -d "$TRANSCODE_TEMP_DIR" ]]; then - info "transcoding-temp already exists on ramdisk" + log "transcoding-temp already exists on ramdisk" else - mkdir -p "$TRANSCODE_TEMP_DIR" && \ - success "Created transcoding-temp on ramdisk: $TRANSCODE_TEMP_DIR" || \ - { error "Failed to create transcoding-temp on ramdisk"; SETUP_SUCCESS=false; } + if mkdir -p "$TRANSCODE_TEMP_DIR"; then + log "Created transcoding-temp on ramdisk ✅" + else + error "Failed to create transcoding-temp on ramdisk" + SETUP_SUCCESS=false + fi fi - # Apply correct permissions so Emby (abc/nobody:users) can write to it if [[ -d "$TRANSCODE_TEMP_DIR" ]]; then chmod "$TRANSCODE_CHMOD" "$TRANSCODE_TEMP_DIR" chown "$TRANSCODE_OWNER" "$TRANSCODE_TEMP_DIR" - success "Permissions set on transcoding-temp" + log "Permissions set on transcoding-temp ($TRANSCODE_CHMOD $TRANSCODE_OWNER)" fi fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Permissions ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Permissions ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_GEAR Permissions ━━━" if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would apply $TRANSCODE_MODE $TRANSCODE_OWNER to $RAMDISK_PATH and $TRANSCODE_SSD" + warn "DRY RUN — would apply $TRANSCODE_CHMOD $TRANSCODE_OWNER to $RAMDISK_PATH and $TRANSCODE_SSD" else for path in "$RAMDISK_PATH" "$TRANSCODE_SSD"; do if [[ -d "$path" ]]; then - chmod "$TRANSCODE_MODE" "$path" + chmod "$TRANSCODE_CHMOD" "$path" chown "$TRANSCODE_OWNER" "$path" - success "Permissions set: $path" + log "Permissions set: $path ($TRANSCODE_CHMOD $TRANSCODE_OWNER)" fi done fi -# ----------------------------------------------------------------------------------------------- -# Initialise state file -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Initialise State File ━━━ +# ============================================================================================== if [[ "$DRY_RUN" == false ]]; then STATE_FILE="/tmp/transcode_state.db" NOW=$(date +%s) @@ -243,25 +296,26 @@ fi END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY RAMDISK SETUP SUMMARY ━━━━━" -echo "$ICON_RAM Ramdisk: $RAMDISK_PATH ($RAMDISK_SIZE)" -echo "$ICON_DISK Fallback: $TRANSCODE_SSD" -echo "$ICON_LINK Symlink: $TRANSCODE_LINK" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no changes made" -elif [[ "$SETUP_SUCCESS" == true ]]; then - echo "$ICON_DONE Status: $ICON_SUCCESS DONE" - notify "Ramdisk setup complete on $(hostname) — ${RAMDISK_SIZE} mounted at $RAMDISK_PATH" "Ramdisk Setup" "normal" -else - echo "$ICON_ERROR Status: $ICON_ERROR SETUP HAD ERRORS" - notify "Ramdisk setup completed with errors on $(hostname)" "Ramdisk Setup" "warning" -fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_RAM Ramdisk: $RAMDISK_PATH ($RAMDISK_SIZE)" +echo "$ICON_DISK Fallback: $TRANSCODE_SSD" +echo "$ICON_LINK Symlink: $TRANSCODE_LINK" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" +echo "" -[[ "$SETUP_SUCCESS" == false ]] && exit 1 -exit 0 \ No newline at end of file +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — no changes made" +elif [[ "$SETUP_SUCCESS" == true ]]; then + log "$ICON_DONE Status: done ✅" +else + echo "$ICON_ERROR Status: SETUP HAD ERRORS" + notify "Ramdisk setup errors on $(hostname) ($MY_ID) — check output" \ + "Ramdisk Setup" "warning" + exit 1 +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Transcodes/transcode_cleanup.sh b/Transcodes/transcode_cleanup.sh index a30a043..8a985e3 100644 --- a/Transcodes/transcode_cleanup.sh +++ b/Transcodes/transcode_cleanup.sh @@ -1,115 +1,178 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Transcode Cleanup ------------------------------------------ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= Transcode Cleanup ========================================== +# ============================================================================================== # Removes old inactive transcode files from both ramdisk and SSD fallback locations. +# Called every 5 minutes by transcode_manager.sh — must be fast and non-blocking. # Never deletes files that are currently open by any process. -# After cleanup checks if ramdisk usage dropped enough to flip symlink back to ramdisk. # -# Safety rules — a file is eligible for deletion only if ALL conditions are true: -# 1. Older than TRANSCODE_MAX_AGE minutes (mtime) -# 2. Not currently open by any process (single lsof call per location — not per file) +# ── SAFETY RULES ────────────────────────────────────────────────────────────────────────────── +# A file is eligible for deletion only if ALL conditions are true: +# 1. Older than TRANSCODE_MAX_AGE minutes (mtime — last modified time) +# 2. Not currently open by any process (checked via lsof pre-built map) # -# Performance note: -# lsof is called ONCE per location to build an open file list — not once per file. -# This is critical for locations with hundreds or thousands of segment files. -# A per-file lsof approach stalls on busy systems with live TV buffering. +# ── WHY NOT SESSION-AWARE CLEANUP ───────────────────────────────────────────────────────────── +# ffmpeg generates folder names independently of the media server API session IDs. +# There is no reliable correlation between API session IDs and transcoding-temp subfolder +# names — matching them would falsely treat active sessions as ended. +# lsof is the correct and reliable active file check — if ffmpeg has a file open, +# lsof sees it regardless of folder naming or session state. # -# Run every 5 minutes via cron/User Scripts plugin. -# All configuration in Master.conf under Transcode Manager section. -# Supports --dry-run to preview what would be deleted without making changes. -# ----------------------------------------------------------------------------------------------- +# ── TRANSCODING-TEMP PROTECTION ─────────────────────────────────────────────────────────────── +# The transcoding-temp directory is excluded from deletion even when empty. +# If cleanup removes the empty transcoding-temp folder from the ramdisk, Emby finds +# the SSD version instead and all new sessions land on SSD until Emby restarts. +# ! -name "transcoding-temp" exclusion in find prevents this permanently. +# +# ── PERFORMANCE ─────────────────────────────────────────────────────────────────────────────── +# lsof is called ONCE per location — never once per file. +# Per-file lsof stalls on busy systems with live TV buffering hundreds of segments. +# +# Open file check uses in-memory associative array (OPEN_FILES_MAP): +# Was: echo "$OPEN_FILES" | grep -qF "$file" — O(n) per file → O(n²) total +# Now: [[ -n "${OPEN_FILES_MAP[$file]:-}" ]] — O(1) per file → O(n) total +# Same lesson as TRACKED_MAP in arr cleanup scripts. +# +# ── POST-CLEANUP SYMLINK FLIP ───────────────────────────────────────────────────────────────── +# After cleanup, if ramdisk has recovered below RAMDISK_LOW_GB and symlink currently +# points at SSD → triggers transcode_manager.sh to flip back to ramdisk. +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID and aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_LOW_GB. +# Each server cleans its own transcode locations at the correct thresholds. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock "wait" — wait if previous cleanup still running +# detect_hosts() — correct paths and thresholds per host +# lsof timeout — lsof call capped at 15 seconds per location +# OPEN_FILES_MAP — in-memory O(1) active file lookup +# transcoding-temp guard — never deletes this directory +# Silent by default — runs every 5 minutes, must not produce noise when healthy +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_RAMDISK_PATH / HOST*_TRANSCODE_SSD / HOST*_RAMDISK_LOW_GB +# Aliased by detect_hosts() +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# TRANSCODE_MAX_AGE — minutes before an inactive transcode file is eligible +# TRANSCODE_ORPHAN_AGE — minutes for orphan detection (informational — future use) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# transcode_cleanup.sh — normal cleanup run +# transcode_cleanup.sh --dry-run — show what would be deleted +# transcode_cleanup.sh --status — show current state +# transcode_cleanup.sh --log — verbose per-file output +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" STATE_FILE="/tmp/transcode_state.db" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" - +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi -success "Running as root" +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" acquire_lock "wait" -if ! command -v lsof >/dev/null 2>&1; then - warn "lsof not available — active file check will be skipped, all aged files will be eligible" - LSOF_AVAILABLE=false -else +# detect_hosts() sets MY_ID and aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_LOW_GB +detect_hosts + +# lsof availability check +LSOF_AVAILABLE=false +if command -v lsof >/dev/null 2>&1; then LSOF_AVAILABLE=true + log "lsof available — active file check enabled" +else + warn "lsof not available — active file check skipped, all aged files eligible for deletion" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +log "Identity: $MY_ID ($LOCAL_SERVER_NAME)" +log "Max age: ${TRANSCODE_MAX_AGE} minutes" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_RAM Ramdisk: $RAMDISK_PATH" - echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD" - echo "$ICON_TRASH Max age: ${TRANSCODE_MAX_AGE} minutes" - echo "$ICON_TRASH Orphan age: ${TRANSCODE_ORPHAN_AGE} minutes" - echo "$ICON_GEAR lsof check: $LSOF_AVAILABLE" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_RAM Ramdisk: $RAMDISK_PATH" + echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD" + echo "$ICON_TRASH Max age: ${TRANSCODE_MAX_AGE} minutes" + echo "$ICON_TRASH Orphan age: ${TRANSCODE_ORPHAN_AGE} minutes" + echo "$ICON_RAM Flip at: ${RAMDISK_LOW_GB}GB" + echo "$ICON_GEAR lsof check: $LSOF_AVAILABLE" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "" + + if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then + USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}') + AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}') + echo " $ICON_RAM Ramdisk: mounted — $USAGE used / $AVAIL available ✅" + else + echo " $ICON_RAM Ramdisk: not mounted" + fi + + CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d= -f2) + echo " $ICON_LINK Current target: ${CURRENT_TARGET:-unknown}" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted" -# ----------------------------------------------------------------------------------------------- -# CLEANUP FUNCTION +# ============================================================================================== +# ── CLEANUP FUNCTION ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== # Scans a location and removes eligible files. -# Calls lsof ONCE per location to build open file list — never per file. -# ----------------------------------------------------------------------------------------------- +# Calls lsof ONCE per location — builds in-memory OPEN_FILES_MAP for O(1) lookup. +# Returns via LOCATION_REMOVED, LOCATION_FREED, LOCATION_SKIPPED, LOCATION_ACTIVE + cleanup_location() { local location="$1" label="$2" max_age="$3" - local files_removed=0 - local bytes_freed=0 - local files_skipped=0 - local files_active=0 + local files_removed=0 bytes_freed=0 files_skipped=0 files_active=0 if [[ ! -d "$location" ]]; then - warn "$label does not exist — skipping" - LOCATION_REMOVED=0 - LOCATION_FREED="0B" - LOCATION_SKIPPED=0 + log "$label does not exist — skipping" + LOCATION_REMOVED=0 LOCATION_FREED="0B" LOCATION_SKIPPED=0 LOCATION_ACTIVE=0 return fi local file_count file_count=$(find "$location" -type f 2>/dev/null | wc -l) - info "$ICON_TRASH $label: $file_count files to scan (age threshold: ${max_age}min)" + log "$label: $file_count files to scan (age threshold: ${max_age}min)" - # Build open file list with a single lsof call — timeout prevents stalling - local OPEN_FILES="" + # Build in-memory open file map — O(1) lookup per file + # One lsof call per location — never per file + declare -A OPEN_FILES_MAP if [[ "$LSOF_AVAILABLE" == true ]]; then - info "Building open file list for $label..." - OPEN_FILES=$(timeout 15 lsof +D "$location" 2>/dev/null | awk 'NR>1 {print $9}' | sort -u) - local open_count - open_count=$(echo "$OPEN_FILES" | grep -c "." 2>/dev/null || echo 0) - info "$open_count files currently open in $label" + log "Building open file map for $label..." + while IFS= read -r open_file; do + [[ -n "$open_file" ]] && OPEN_FILES_MAP["$open_file"]=1 + done < <(timeout 15 lsof +D "$location" 2>/dev/null | awk 'NR>1 {print $9}' | sort -u) + log "${#OPEN_FILES_MAP[@]} files currently open in $label" fi - # Find files older than max_age and process them + # Process aged files while IFS= read -r file; do [[ -z "$file" ]] && continue - # Check if file is currently open — fast string match against pre-built list - if [[ "$LSOF_AVAILABLE" == true ]] && echo "$OPEN_FILES" | grep -qF "$file"; then - ((files_active++)) + # O(1) open file check — in-memory map + if [[ -n "${OPEN_FILES_MAP[$file]:-}" ]]; then + (( files_active++ )) log "Skipping open file: $file" continue fi @@ -118,30 +181,29 @@ cleanup_location() { file_size=$(stat -c%s "$file" 2>/dev/null || echo 0) if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would delete: $(basename "$file")" - ((files_skipped++)) + log "DRY RUN — would delete: $(basename "$file")" + (( files_skipped++ )) else if rm -f "$file" 2>/dev/null; then - ((files_removed++)) - bytes_freed=$((bytes_freed + file_size)) + (( files_removed++ )) + bytes_freed=$(( bytes_freed + file_size )) log "Deleted: $file" else warn "Could not delete: $file" - ((files_skipped++)) + (( files_skipped++ )) fi fi done < <(find "$location" -type f -mmin +"$max_age" 2>/dev/null) - # Remove empty directories left behind — but NEVER remove transcoding-temp itself - # transcoding-temp must always exist on the ramdisk so Emby finds it there first - # If deleted Emby falls back to the SSD version and all new sessions land on SSD + # Remove empty directories — but NEVER remove transcoding-temp + # transcoding-temp must always exist on ramdisk so Emby finds it there first if [[ "$DRY_RUN" == false ]]; then find "$location" -mindepth 1 -type d -empty \ ! -name "transcoding-temp" -delete 2>/dev/null fi - # Format bytes freed for display + # Format bytes freed local freed_human if (( bytes_freed > 1073741824 )); then freed_human=$(awk "BEGIN {printf \"%.1fGB\", $bytes_freed / 1073741824}") @@ -153,11 +215,7 @@ cleanup_location() { freed_human="0B" fi - if [[ "$DRY_RUN" == true ]]; then - info "$label — dry run complete ($file_count files scanned, $files_active active)" - else - success "$label — removed $files_removed files ($freed_human freed), $files_active active, $files_skipped skipped" - fi + log "$label — removed $files_removed files ($freed_human freed) | active: $files_active | skipped: $files_skipped" LOCATION_REMOVED=$files_removed LOCATION_FREED=$freed_human @@ -165,79 +223,65 @@ cleanup_location() { LOCATION_ACTIVE=$files_active } -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_TRASH Transcode Cleanup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_TRASH Transcode Cleanup — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -echo "" - +# ============================================================================================== +# ━━━ Transcode Cleanup ━━━ +# ============================================================================================== START=$(date +%s) -TOTAL_REMOVED=0 -TOTAL_SKIPPED=0 -TOTAL_ACTIVE=0 -RAMDISK_FREED="0B" -SSD_FREED="0B" +TOTAL_REMOVED=0 TOTAL_SKIPPED=0 TOTAL_ACTIVE=0 +RAMDISK_FREED="0B" SSD_FREED="0B" # Cleanup ramdisk if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then - echo "━━━ $ICON_RAM Ramdisk ━━━" cleanup_location "$RAMDISK_PATH" "Ramdisk" "$TRANSCODE_MAX_AGE" - TOTAL_REMOVED=$((TOTAL_REMOVED + LOCATION_REMOVED)) - TOTAL_SKIPPED=$((TOTAL_SKIPPED + LOCATION_SKIPPED)) - TOTAL_ACTIVE=$((TOTAL_ACTIVE + LOCATION_ACTIVE)) + TOTAL_REMOVED=$(( TOTAL_REMOVED + LOCATION_REMOVED )) + TOTAL_SKIPPED=$(( TOTAL_SKIPPED + LOCATION_SKIPPED )) + TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE )) RAMDISK_FREED=$LOCATION_FREED - echo "" else - warn "$ICON_RAM Ramdisk not mounted — skipping ramdisk cleanup" + log "Ramdisk not mounted — skipping ramdisk cleanup" fi # Cleanup SSD fallback if [[ -d "$TRANSCODE_SSD" ]]; then - echo "━━━ $ICON_DISK SSD Fallback ━━━" cleanup_location "$TRANSCODE_SSD" "SSD fallback" "$TRANSCODE_MAX_AGE" - TOTAL_REMOVED=$((TOTAL_REMOVED + LOCATION_REMOVED)) - TOTAL_SKIPPED=$((TOTAL_SKIPPED + LOCATION_SKIPPED)) - TOTAL_ACTIVE=$((TOTAL_ACTIVE + LOCATION_ACTIVE)) + TOTAL_REMOVED=$(( TOTAL_REMOVED + LOCATION_REMOVED )) + TOTAL_SKIPPED=$(( TOTAL_SKIPPED + LOCATION_SKIPPED )) + TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE )) SSD_FREED=$LOCATION_FREED - echo "" else - info "$ICON_DISK SSD fallback not found — skipping" + log "SSD fallback not found — skipping SSD cleanup" fi -# ----------------------------------------------------------------------------------------------- -# Post-cleanup — check if ramdisk recovered enough to flip symlink back -# ----------------------------------------------------------------------------------------------- +# Post-cleanup — check if ramdisk recovered enough to flip symlink back to ramdisk if [[ "$DRY_RUN" == false ]] && mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then - RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ') - RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}") + RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ') + RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}") LOW_RECOVERED=$(awk "BEGIN {print ($RAMDISK_USED_GB < $RAMDISK_LOW_GB) ? 1 : 0}") - CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d'=' -f2) + CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d= -f2) if [[ "$LOW_RECOVERED" == "1" && "$CURRENT_TARGET" == "$TRANSCODE_SSD" ]]; then - info "$ICON_RAM Ramdisk has space after cleanup — triggering manager to flip back" + log "Ramdisk has space after cleanup (${RAMDISK_USED_GB}GB < ${RAMDISK_LOW_GB}GB) — triggering manager to flip back" bash "$SCRIPT_DIR/transcode_manager.sh" --no-log fi fi END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "━━━━━ $ICON_SUMMARY TRANSCODE CLEANUP SUMMARY ━━━━━" -echo "$ICON_RAM Ramdisk freed: $RAMDISK_FREED" -echo "$ICON_DISK SSD freed: $SSD_FREED" -echo "$ICON_TRASH Removed: $TOTAL_REMOVED files" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_RAM Ramdisk freed: $RAMDISK_FREED" +echo "$ICON_DISK SSD freed: $SSD_FREED" +echo "$ICON_TRASH Removed: $TOTAL_REMOVED files" echo "$ICON_RUNNING Active: $TOTAL_ACTIVE files (open — protected)" -echo "$ICON_TRASH Skipped: $TOTAL_SKIPPED files" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_TRASH Skipped: $TOTAL_SKIPPED files" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" + if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no files deleted" + warn "DRY RUN — no files deleted" else - echo "$ICON_DONE Status: $ICON_SUCCESS DONE" - if [[ "$TOTAL_REMOVED" -gt 0 ]]; then - notify "Transcode cleanup on $(hostname) — removed $TOTAL_REMOVED files (RAM: $RAMDISK_FREED SSD: $SSD_FREED)" "Transcode Cleanup" "normal" - fi + log "$ICON_DONE Status: done ✅" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Transcodes/transcode_manager.sh b/Transcodes/transcode_manager.sh index 0f3eedc..5f638b5 100644 --- a/Transcodes/transcode_manager.sh +++ b/Transcodes/transcode_manager.sh @@ -1,63 +1,105 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Transcode Manager ------------------------------------------ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= Transcode Manager ========================================== +# ============================================================================================== # Manages Emby transcode storage using filesystem symlink indirection. -# ffmpeg resolves the symlink ONCE at session start — existing sessions are never affected. -# Only new sessions care about where the symlink currently points. +# Called every 5 minutes by User Scripts — must be fast, non-blocking, and silent when healthy. # -# Three operating modes — set TRANSCODE_MANAGER_MODE in Master.conf: +# ── HOW IT WORKS ────────────────────────────────────────────────────────────────────────────── +# Emby's transcode path is set to TRANSCODE_LINK (a symlink). +# ffmpeg resolves the symlink ONCE at session start — existing sessions are never affected. +# Only NEW sessions care about where the symlink currently points. +# Flipping the symlink mid-stream is safe — in-progress transcodes continue uninterrupted. +# +# ── THREE MODES ─────────────────────────────────────────────────────────────────────────────── # smart — auto-flips between ramdisk and SSD based on usage thresholds (default) -# ramdisk — always uses ramdisk, never flips to SSD regardless of usage +# ramdisk above RAMDISK_WARN_GB → flip to SSD +# ramdisk below RAMDISK_LOW_GB → flip back to ramdisk +# ramdisk — always uses ramdisk, warns if above threshold, never flips # ssd — always uses SSD, never uses ramdisk # -# Safety checks on every run regardless of mode: -# Ramdisk disappeared → auto-flip to SSD, notify warning -# SSD path missing → disable SSD fallback, notify warning -# Symlink broken → auto-recreate, notify -# Symlink missing → auto-recreate, notify -# Permissions drift → fix silently -# Emby not running → skip threshold checks, verify symlink only +# ── SAFETY CHECKS — EVERY RUN ───────────────────────────────────────────────────────────────── +# Symlink missing/broken → auto-recreate pointing at ramdisk, notify +# Ramdisk disappeared → auto-flip to SSD, notify warning +# SSD path missing → disable SSD fallback / error if mode=ssd +# transcoding-temp missing → recreate on ramdisk immediately +# Permissions drift → fix silently +# Emby not running → skip threshold checks, verify symlink only # -# Session display shows active Emby streams with storage state. -# Split state detected when sessions exist on both ramdisk and SSD simultaneously — +# ── SESSION DISPLAY ─────────────────────────────────────────────────────────────────────────── +# Shows active Emby/Jellyfin/Plex streams with user, title, type, and play method. +# Split state shown when sessions exist on both ramdisk and SSD simultaneously — # this happens naturally when symlink flips mid-session. # -# All configuration in Master.conf under Transcode Manager section. -# Supports --dry-run and --status. -# ----------------------------------------------------------------------------------------------- +# ── DAILY LOG ───────────────────────────────────────────────────────────────────────────────── +# Appends to TRANSCODE_DAILY_LOG after each run — read by weekly_health_digest.sh. +# Format: DATE|RAMDISK_USED_GB|FLIP_COUNT|RAM_SESSION_COUNT|SSD_SESSION_COUNT|FILES_CLEANED +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID and aliases HOST*_TRANSCODE_SERVERS, HOST*_RAMDISK_PATH, +# HOST*_TRANSCODE_SSD, HOST*_RAMDISK_WARN_GB, HOST*_RAMDISK_LOW_GB, HOST*_RAMDISK_SIZE. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock "wait" — wait if previous run still active +# detect_hosts() — correct paths and thresholds per host +# DOCKER_TIMEOUT — all docker calls protected against daemon hangs +# validate_unraid_cmd — notify validated before use +# Silent by default — runs every 5 minutes, only speaks when something changes +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# transcode_manager.sh — normal run +# transcode_manager.sh --dry-run — preview without making changes +# transcode_manager.sh --status — show current state and exit +# transcode_manager.sh --log — verbose output +# transcode_manager.sh --no-log — suppress daily log write (called by cleanup) +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" -parse_args "$@" +# Handle --no-log flag before parse_args +NO_LOG=false +FILTERED_ARGS=() +for arg in "$@"; do + case "$arg" in + --no-log) NO_LOG=true ;; + *) FILTERED_ARGS+=("$arg") ;; + esac +done -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" +parse_args "${FILTERED_ARGS[@]}" +DOCKER_TIMEOUT=15 +STATE_FILE="/tmp/transcode_state.db" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi -success "Running as root" +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" acquire_lock "wait" -# Primary server for container running check — first entry in TRANSCODE_SERVERS -# Full session display loops over all entries in the array +# detect_hosts() sets MY_ID and aliases all HOST*_TRANSCODE_* vars +detect_hosts + +# Primary container — first entry in TRANSCODE_SERVERS +TRANSCODE_EMBY_CONTAINER="Emby" if [[ "${#TRANSCODE_SERVERS[@]}" -gt 0 ]]; then IFS='|' read -r PRIMARY_CONTAINER _ _ _ <<< "${TRANSCODE_SERVERS[0]}" TRANSCODE_EMBY_CONTAINER="${PRIMARY_CONTAINER:-Emby}" fi case "$TRANSCODE_MANAGER_MODE" in - smart|ramdisk|ssd) info "$ICON_GEAR Mode: $TRANSCODE_MANAGER_MODE" ;; + smart|ramdisk|ssd) log "Mode: $TRANSCODE_MANAGER_MODE" ;; *) warn "Unknown TRANSCODE_MANAGER_MODE: $TRANSCODE_MANAGER_MODE — defaulting to smart" TRANSCODE_MANAGER_MODE="smart" @@ -66,12 +108,13 @@ esac [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" - echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "━━━━━ $ICON_SUMMARY TRANSCODE MANAGER STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_GEAR Mode: $TRANSCODE_MANAGER_MODE" echo "$ICON_RAM Ramdisk: $RAMDISK_PATH ($RAMDISK_SIZE)" echo "$ICON_LINK Symlink: $TRANSCODE_LINK" @@ -85,19 +128,19 @@ if [[ "$SHOW_STATUS" == true ]]; then CURRENT_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "missing") echo "$ICON_LINK Symlink now: $TRANSCODE_LINK → $CURRENT_TARGET" if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then - USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ') - USED_GB=$(awk "BEGIN {printf \"%.2f\", $USED_KB / 1048576}") + USED_GB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | \ + tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}') echo "$ICON_RAM Ramdisk now: ${USED_GB}GB used" else - echo "$ICON_RAM Ramdisk: NOT MOUNTED" + echo "$ICON_RAM Ramdisk: NOT MOUNTED" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -# ----------------------------------------------------------------------------------------------- -# HELPERS -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== flip_symlink() { local target="$1" reason="$2" @@ -106,7 +149,7 @@ flip_symlink() { return 0 fi ln -sfn "$target" "$TRANSCODE_LINK" - success "$ICON_LINK Symlink flipped to: $target ($reason)" + warn "$ICON_LINK Symlink flipped to: $target ($reason)" } fix_permissions() { @@ -118,30 +161,27 @@ fix_permissions() { } get_ramdisk_used_gb() { - local used_kb - used_kb=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ') - awk "BEGIN {printf \"%.2f\", ${used_kb:-0} / 1048576}" + df "$RAMDISK_PATH" --output=used 2>/dev/null | \ + tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}' } get_ramdisk_avail_gb() { - local avail_kb - avail_kb=$(df "$RAMDISK_PATH" --output=avail 2>/dev/null | tail -1 | tr -d ' ') - awk "BEGIN {printf \"%.2f\", ${avail_kb:-0} / 1048576}" + df "$RAMDISK_PATH" --output=avail 2>/dev/null | \ + tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}' } get_ssd_free_gb() { - local free_kb - free_kb=$(df "$TRANSCODE_SSD" --output=avail 2>/dev/null | tail -1 | tr -d ' ') - awk "BEGIN {printf \"%.2f\", ${free_kb:-0} / 1048576}" + df "$TRANSCODE_SSD" --output=avail 2>/dev/null | \ + tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}' } get_flip_count() { local state_file="/tmp/transcode_flip_state.db" local current_hour current_hour=$(date '+%Y-%m-%d-%H') - if [[ ! -f "$state_file" ]]; then echo "0"; return; fi + [[ ! -f "$state_file" ]] && echo "0" && return local stored_hour stored_count - stored_hour=$(awk -F'|' 'NR==1{print $1}' "$state_file" 2>/dev/null) + stored_hour=$(awk -F'|' 'NR==1{print $1}' "$state_file" 2>/dev/null) stored_count=$(awk -F'|' 'NR==1{print $2}' "$state_file" 2>/dev/null) [[ "$stored_hour" == "$current_hour" ]] && echo "${stored_count:-0}" || echo "0" } @@ -152,76 +192,55 @@ increment_flip_count() { current_hour=$(date '+%Y-%m-%d-%H') local current_count current_count=$(get_flip_count) - current_count=$((current_count + 1)) + current_count=$(( current_count + 1 )) echo "${current_hour}|${current_count}" > "$state_file" echo "$current_count" } -emby_api() { - local endpoint="$1" - [[ -z "$EMBY_API_KEY" ]] && return 1 - curl -sf \ - --max-time 10 \ - -H "X-Emby-Token: $EMBY_API_KEY" \ - "${EMBY_URL}/${endpoint}" 2>/dev/null -} - get_media_type_label() { - local type="$1" - case "$type" in - LiveTv) echo "Live TV" ;; - TvChannel) echo "Live TV" ;; - Episode) echo "TV Show" ;; - Movie) echo "Movie" ;; - Audio) echo "Music" ;; - MusicVideo) echo "Music Video" ;; - *) echo "$type" ;; + case "$1" in + LiveTv|TvChannel) echo "Live TV" ;; + Episode) echo "TV Show" ;; + Movie) echo "Movie" ;; + Audio) echo "Music" ;; + MusicVideo) echo "Music Video" ;; + *) echo "$1" ;; esac } get_play_method_label() { - local method="$1" - case "$method" in - Transcode) echo "Transcode" ;; - DirectStream) echo "Direct Stream" ;; - DirectPlay) echo "Direct Play" ;; - *) echo "$method" ;; + case "$1" in + Transcode) echo "Transcode" ;; + DirectStream) echo "Direct Stream" ;; + DirectPlay) echo "Direct Play" ;; + *) echo "$1" ;; esac } -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_RAM Transcode Manager ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_RAM Transcode Manager — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -echo "$ICON_GEAR Mode: $TRANSCODE_MANAGER_MODE" -echo "" - +# ============================================================================================== +# ━━━ Transcode Manager ━━━ +# ============================================================================================== START=$(date +%s) RAMDISK_HEALTHY=true SSD_HEALTHY=true EMBY_RUNNING=true +SOMETHING_HAPPENED=false # controls whether summary is printed -# ----------------------------------------------------------------------------------------------- -# CHECK 1 — Emby running -# ----------------------------------------------------------------------------------------------- +# ── Check 1 — Emby running ─────────────────────────────────────────────────────────────────── if [[ "$TRANSCODE_CHECK_EMBY" == true ]]; then - info "$ICON_CONTAINERS Checking $TRANSCODE_EMBY_CONTAINER..." - if ! docker inspect "$TRANSCODE_EMBY_CONTAINER" \ + log "Checking $TRANSCODE_EMBY_CONTAINER..." + if ! timeout "$DOCKER_TIMEOUT" docker inspect "$TRANSCODE_EMBY_CONTAINER" \ --format '{{.State.Running}}' 2>/dev/null | grep -q "true"; then warn "$TRANSCODE_EMBY_CONTAINER is not running — skipping threshold checks" EMBY_RUNNING=false + SOMETHING_HAPPENED=true else - success "$TRANSCODE_EMBY_CONTAINER is running" + log "$TRANSCODE_EMBY_CONTAINER is running ✅" fi fi -# ----------------------------------------------------------------------------------------------- -# CHECK 2 — Symlink integrity -# ----------------------------------------------------------------------------------------------- -echo "" -info "$ICON_LINK Checking symlink integrity..." - +# ── Check 2 — Symlink integrity ─────────────────────────────────────────────────────────────── +log "Checking symlink integrity..." CURRENT_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null) if [[ -z "$CURRENT_TARGET" ]]; then @@ -229,175 +248,162 @@ if [[ -z "$CURRENT_TARGET" ]]; then if [[ "$DRY_RUN" == false ]]; then ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK" CURRENT_TARGET="$RAMDISK_PATH" - notify "Transcode symlink was missing on $(hostname) — recreated" "Transcode Manager" "warning" + notify "Transcode symlink was missing on $(hostname) ($MY_ID) — recreated" \ + "Transcode Manager" "warning" fi + SOMETHING_HAPPENED=true elif [[ ! -e "$CURRENT_TARGET" ]]; then warn "$ICON_LINK Symlink target missing: $CURRENT_TARGET — resetting to ramdisk" if [[ "$DRY_RUN" == false ]]; then ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK" CURRENT_TARGET="$RAMDISK_PATH" - notify "Transcode symlink target was missing on $(hostname) — reset to ramdisk" "Transcode Manager" "warning" + notify "Transcode symlink target was missing on $(hostname) ($MY_ID) — reset to ramdisk" \ + "Transcode Manager" "warning" fi + SOMETHING_HAPPENED=true else - success "$ICON_LINK Symlink valid: $TRANSCODE_LINK → $CURRENT_TARGET" + log "Symlink valid: $TRANSCODE_LINK → $CURRENT_TARGET ✅" fi -# ----------------------------------------------------------------------------------------------- -# CHECK 3 — Ramdisk health -# ----------------------------------------------------------------------------------------------- -echo "" -info "$ICON_RAM Checking ramdisk..." +# ── Check 3 — Ramdisk health ────────────────────────────────────────────────────────────────── +log "Checking ramdisk..." if ! mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then - error "$ICON_RAM Ramdisk not mounted at $RAMDISK_PATH" + error "Ramdisk not mounted at $RAMDISK_PATH" RAMDISK_HEALTHY=false + SOMETHING_HAPPENED=true if [[ "$DRY_RUN" == false ]]; then warn "Flipping symlink to SSD — ramdisk unavailable" flip_symlink "$TRANSCODE_SSD" "ramdisk disappeared" CURRENT_TARGET="$TRANSCODE_SSD" - notify "Ramdisk disappeared on $(hostname) — transcodes falling back to SSD. Run ramdisk_setup.sh to restore." "Transcode Manager" "warning" + notify "Ramdisk disappeared on $(hostname) ($MY_ID) — transcodes falling back to SSD. Run ramdisk_setup.sh to restore." \ + "Transcode Manager" "warning" fi else - RAMDISK_SIZE_ACTUAL=$(df "$RAMDISK_PATH" --output=size -h | tail -1 | tr -d ' ') - success "$ICON_RAM Ramdisk mounted — size: $RAMDISK_SIZE_ACTUAL" + RAMDISK_SIZE_ACTUAL=$(df "$RAMDISK_PATH" --output=size -h 2>/dev/null | tail -1 | tr -d ' ') + log "Ramdisk mounted — size: $RAMDISK_SIZE_ACTUAL ✅" fix_permissions "$RAMDISK_PATH" # Guarantee transcoding-temp exists on ramdisk - # If missing Emby creates it wherever it finds a writable path first — - # which may be the SSD fallback — locking all new sessions onto SSD - # even when the symlink correctly points at the ramdisk. TRANSCODE_TEMP_RAM="${RAMDISK_PATH}/transcoding-temp" - TRANSCODE_TEMP_SSD="${TRANSCODE_SSD}/transcoding-temp" - if [[ ! -d "$TRANSCODE_TEMP_RAM" ]]; then warn "transcoding-temp missing from ramdisk — creating now" mkdir -p "$TRANSCODE_TEMP_RAM" chmod "$TRANSCODE_CHMOD" "$TRANSCODE_TEMP_RAM" chown "$TRANSCODE_OWNER" "$TRANSCODE_TEMP_RAM" - success "transcoding-temp created on ramdisk — new sessions will use ramdisk" + warn "transcoding-temp created on ramdisk — new sessions will use ramdisk ✅" + SOMETHING_HAPPENED=true else - log "transcoding-temp exists on ramdisk — ok" + log "transcoding-temp exists on ramdisk ✅" fi fi -# ----------------------------------------------------------------------------------------------- -# CHECK 4 — SSD health -# ----------------------------------------------------------------------------------------------- -echo "" -info "$ICON_DISK Checking SSD fallback..." +# ── Check 4 — SSD health ───────────────────────────────────────────────────────────────────── +log "Checking SSD fallback..." if [[ ! -d "$TRANSCODE_SSD" ]]; then - warn "$ICON_DISK SSD fallback path missing: $TRANSCODE_SSD" + warn "SSD fallback path missing: $TRANSCODE_SSD" SSD_HEALTHY=false + SOMETHING_HAPPENED=true if [[ "$TRANSCODE_MANAGER_MODE" == "ssd" ]]; then error "Mode is 'ssd' but SSD path is missing — cannot continue" - notify "Transcode SSD path missing on $(hostname) — mode is 'ssd', manual intervention needed" "Transcode Manager" "warning" + notify "Transcode SSD path missing on $(hostname) ($MY_ID) — mode is 'ssd', manual intervention needed" \ + "Transcode Manager" "warning" exit 1 else warn "SSD fallback disabled — will stay on ramdisk" fi else SSD_FREE_GB=$(get_ssd_free_gb) - success "$ICON_DISK SSD available — ${SSD_FREE_GB}GB free" + log "SSD available — ${SSD_FREE_GB}GB free ✅" fix_permissions "$TRANSCODE_SSD" fi -# ----------------------------------------------------------------------------------------------- -# USAGE STATS -# ----------------------------------------------------------------------------------------------- -echo "" +# ── Usage stats ─────────────────────────────────────────────────────────────────────────────── RAMDISK_USED_GB="0.00" RAMDISK_AVAIL_GB="0.00" +RAMDISK_FILES=0 if [[ "$RAMDISK_HEALTHY" == true ]]; then RAMDISK_USED_GB=$(get_ramdisk_used_gb) RAMDISK_AVAIL_GB=$(get_ramdisk_avail_gb) RAMDISK_FILES=$(find "$RAMDISK_PATH" -type f 2>/dev/null | wc -l) - RAMDISK_SIZE_H=$(du -sh "$RAMDISK_PATH" 2>/dev/null | cut -f1) fi SSD_FILES=$(find "$TRANSCODE_SSD" -type f 2>/dev/null | wc -l) -SSD_SIZE_H=$(du -sh "$TRANSCODE_SSD" 2>/dev/null | cut -f1) - -info "$ICON_RAM Ramdisk usage: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB available" -info "$ICON_RAM Ramdisk files: ${RAMDISK_FILES:-0} ($RAMDISK_SIZE_H)" -info "$ICON_DISK SSD files: $SSD_FILES ($SSD_SIZE_H)" - FLIP_COUNT=$(get_flip_count) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_EMBY Active Transcode Sessions ━━━ -# Queries all configured servers in TRANSCODE_SERVERS array -# Aggregates sessions, counts, and storage state across all servers -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_EMBY Active Transcode Sessions ━━━" +log "Ramdisk: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB available ($RAMDISK_FILES files)" +log "SSD: $SSD_FILES files" +log "Flips: $FLIP_COUNT this hour" +# ============================================================================================== +# ━━━ Active Transcode Sessions ━━━ +# ============================================================================================== TOTAL_SESSIONS=0 LIVE_TV=0 TRANSCODING=0 DIRECT=0 ANY_SERVER_RUNNING=false +RAM_SESSION_COUNT=0 +SSD_SESSION_COUNT=0 for server_entry in "${TRANSCODE_SERVERS[@]}"; do - # Parse entry — ContainerName|URL|APIKey|Type IFS='|' read -r SRV_CONTAINER SRV_URL SRV_KEY SRV_TYPE <<< "$server_entry" # Skip placeholder entries - if [[ "$SRV_KEY" == *"api-key"* ]] || [[ "$SRV_KEY" == *"token"* && ${#SRV_KEY} -lt 20 ]]; then + if [[ "$SRV_KEY" == *"api-key"* ]] || \ + [[ "$SRV_KEY" == *"token"* && ${#SRV_KEY} -lt 20 ]]; then log "Skipping $SRV_CONTAINER — placeholder API key" continue fi - # Check if container is running - if ! docker inspect "$SRV_CONTAINER" --format '{{.State.Running}}' 2>/dev/null | grep -q "true"; then - info "$ICON_CONTAINERS $SRV_CONTAINER — not running, skipping" + # Container running check with timeout + if ! timeout "$DOCKER_TIMEOUT" docker inspect "$SRV_CONTAINER" \ + --format '{{.State.Running}}' 2>/dev/null | grep -q "true"; then + log "$SRV_CONTAINER — not running, skipping" continue fi ANY_SERVER_RUNNING=true - # Check API reachability if ! check_api "$SRV_URL" "$SRV_CONTAINER" 5; then - warn "$SRV_CONTAINER — API unreachable, skipping (container up but API not responding)" + warn "$SRV_CONTAINER — API unreachable, skipping" continue fi - # Query sessions based on server type case "$SRV_TYPE" in emby|jellyfin) SESSION_DATA=$(curl -sf --max-time 10 \ -H "X-Emby-Token: $SRV_KEY" \ - "${SRV_URL}/Sessions" 2>/dev/null) - ;; + "${SRV_URL}/Sessions" 2>/dev/null) ;; plex) SESSION_DATA=$(curl -sf --max-time 10 \ -H "X-Plex-Token: $SRV_KEY" \ - "${SRV_URL}/status/sessions" 2>/dev/null) - ;; + "${SRV_URL}/status/sessions" 2>/dev/null) ;; *) warn "Unknown server type '$SRV_TYPE' for $SRV_CONTAINER — skipping" - continue - ;; + continue ;; esac - if [[ -z "$SESSION_DATA" ]] || ! command -v jq >/dev/null 2>&1; then - warn "Could not retrieve session data from $SRV_CONTAINER" - continue - fi + [[ -z "$SESSION_DATA" ]] && continue + ! command -v jq >/dev/null 2>&1 && continue - # Count sessions (Emby/Jellyfin format) if [[ "$SRV_TYPE" == "emby" || "$SRV_TYPE" == "jellyfin" ]]; then SRV_TOTAL=$(echo "$SESSION_DATA" | \ jq '[.[] | select(.NowPlayingItem != null)] | length' 2>/dev/null || echo 0) SRV_LIVE=$(echo "$SESSION_DATA" | \ - jq '[.[] | select(.NowPlayingItem != null) | select(.NowPlayingItem.Type == "LiveTv" or .NowPlayingItem.Type == "TvChannel")] | length' \ + jq '[.[] | select(.NowPlayingItem != null) | + select(.NowPlayingItem.Type == "LiveTv" or + .NowPlayingItem.Type == "TvChannel")] | length' \ 2>/dev/null || echo 0) SRV_TRANSCODE=$(echo "$SESSION_DATA" | \ - jq '[.[] | select(.NowPlayingItem != null) | select(.PlayState.PlayMethod == "Transcode")] | length' \ + jq '[.[] | select(.NowPlayingItem != null) | + select(.PlayState.PlayMethod == "Transcode")] | length' \ 2>/dev/null || echo 0) SRV_DIRECT=$(echo "$SESSION_DATA" | \ - jq '[.[] | select(.NowPlayingItem != null) | select(.PlayState.PlayMethod != "Transcode")] | length' \ + jq '[.[] | select(.NowPlayingItem != null) | + select(.PlayState.PlayMethod != "Transcode")] | length' \ 2>/dev/null || echo 0) TOTAL_SESSIONS=$(( TOTAL_SESSIONS + SRV_TOTAL )) @@ -405,138 +411,134 @@ for server_entry in "${TRANSCODE_SERVERS[@]}"; do TRANSCODING=$(( TRANSCODING + SRV_TRANSCODE )) DIRECT=$(( DIRECT + SRV_DIRECT )) - # Display server header if more than one server configured and running - if [[ "${#TRANSCODE_SERVERS[@]}" -gt 1 ]]; then - echo " $ICON_EMBY $SRV_CONTAINER — $SRV_TOTAL streams" - fi - - # List each active session if [[ "$SRV_TOTAL" -gt 0 ]]; then + SOMETHING_HAPPENED=true + [[ "${#TRANSCODE_SERVERS[@]}" -gt 1 ]] && \ + echo " $ICON_EMBY $SRV_CONTAINER — $SRV_TOTAL streams" + while IFS= read -r session; do - USER=$(echo "$session" | jq -r '.UserName // "Unknown"' 2>/dev/null) + USER=$(echo "$session" | jq -r '.UserName // "Unknown"' 2>/dev/null) TITLE=$(echo "$session" | jq -r '.NowPlayingItem.Name // "Unknown"' 2>/dev/null) - MEDIA_TYPE=$(echo "$session" | jq -r '.NowPlayingItem.Type // "Unknown"' 2>/dev/null) - PLAY_METHOD=$(echo "$session" | jq -r '.PlayState.PlayMethod // "Unknown"' 2>/dev/null) - - MEDIA_LABEL=$(get_media_type_label "$MEDIA_TYPE") - METHOD_LABEL=$(get_play_method_label "$PLAY_METHOD") - - echo " $ICON_EMBY $(printf '%-12s' "$USER") — $(printf '%-30s' "$TITLE") — $(printf '%-10s' "$MEDIA_LABEL") — $METHOD_LABEL" - done < <(echo "$SESSION_DATA" | \ - jq -c '.[] | select(.NowPlayingItem != null)' 2>/dev/null) + MTYPE=$(echo "$session" | jq -r '.NowPlayingItem.Type // "Unknown"' 2>/dev/null) + METH=$(echo "$session" | jq -r '.PlayState.PlayMethod // "Unknown"' 2>/dev/null) + echo " $ICON_EMBY $(printf '%-12s' "$USER") — $(printf '%-30s' "$TITLE") — $(get_media_type_label "$MTYPE") — $(get_play_method_label "$METH")" + done < <(echo "$SESSION_DATA" | jq -c '.[] | select(.NowPlayingItem != null)' 2>/dev/null) fi fi - done -# Totals header — shown after all servers -echo "" -echo " $ICON_EMBY Total: $TOTAL_SESSIONS | $ICON_RAM Live TV: $LIVE_TV | $ICON_SYNC Transcoding: $TRANSCODING | $ICON_DONE Direct: $DIRECT" +# Storage state +RAM_SESSION_COUNT=$(find "$RAMDISK_PATH/transcoding-temp" \ + -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l) +SSD_SESSION_COUNT=$(find "$TRANSCODE_SSD/transcoding-temp" \ + -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l) -# Storage state — based on total ramdisk/SSD session folders if [[ "$TOTAL_SESSIONS" -gt 0 ]]; then - TRANSCODE_TEMP="transcoding-temp" - RAM_SESSION_COUNT=$(find "$RAMDISK_PATH/$TRANSCODE_TEMP" \ - -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l) - SSD_SESSION_COUNT=$(find "$TRANSCODE_SSD/$TRANSCODE_TEMP" \ - -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l) + echo "" + echo " $ICON_EMBY Total: $TOTAL_SESSIONS | Live TV: $LIVE_TV | Transcoding: $TRANSCODING | Direct: $DIRECT" if [[ "$RAM_SESSION_COUNT" -gt 0 && "$SSD_SESSION_COUNT" -gt 0 ]]; then - warn "$ICON_WARN Split state — $RAM_SESSION_COUNT folder(s) on ramdisk / $SSD_SESSION_COUNT on SSD" + warn "Split state — $RAM_SESSION_COUNT folder(s) ramdisk / $SSD_SESSION_COUNT SSD" warn "Older sessions remain on original location until they end naturally" - echo " $ICON_LINK Storage: $ICON_RAM ramdisk ($RAM_SESSION_COUNT) + $ICON_DISK SSD ($SSD_SESSION_COUNT)" elif [[ "$RAM_SESSION_COUNT" -gt 0 ]]; then - echo " $ICON_LINK Storage: $ICON_RAM ramdisk" + log "Storage: ramdisk ($RAM_SESSION_COUNT sessions)" elif [[ "$SSD_SESSION_COUNT" -gt 0 ]]; then - echo " $ICON_LINK Storage: $ICON_DISK SSD" + log "Storage: SSD ($SSD_SESSION_COUNT sessions)" fi fi -if [[ "$ANY_SERVER_RUNNING" == false && "$TRANSCODE_CHECK_EMBY" == true ]]; then - info "No configured media servers are running — skipping threshold checks" - EMBY_RUNNING=false -fi - -# ----------------------------------------------------------------------------------------------- -# MODE LOGIC -# ----------------------------------------------------------------------------------------------- -echo "" +[[ "$ANY_SERVER_RUNNING" == false && "$TRANSCODE_CHECK_EMBY" == true ]] && \ + { log "No configured media servers running — skipping threshold checks"; EMBY_RUNNING=false; } +# ============================================================================================== +# ━━━ Mode Logic ━━━ +# ============================================================================================== case "$TRANSCODE_MANAGER_MODE" in ramdisk) - info "$ICON_RAM Mode: RAMDISK — forcing symlink to ramdisk" + log "Mode: RAMDISK — forcing symlink to ramdisk" if [[ "$RAMDISK_HEALTHY" == false ]]; then error "Ramdisk mode selected but ramdisk is not available" - notify "Transcode ramdisk mode failed on $(hostname) — ramdisk not mounted" "Transcode Manager" "warning" + notify "Transcode ramdisk mode failed on $(hostname) ($MY_ID) — ramdisk not mounted" \ + "Transcode Manager" "warning" + SOMETHING_HAPPENED=true else if [[ "$CURRENT_TARGET" != "$RAMDISK_PATH" ]]; then flip_symlink "$RAMDISK_PATH" "ramdisk mode" increment_flip_count > /dev/null - else - log "Symlink already points to ramdisk — no change" + FLIP_COUNT=$(get_flip_count) + SOMETHING_HAPPENED=true fi if (( $(awk "BEGIN {print ($RAMDISK_USED_GB >= $RAMDISK_WARN_GB) ? 1 : 0}") )); then - warn "$ICON_RAM Ramdisk usage ${RAMDISK_USED_GB}GB above threshold ${RAMDISK_WARN_GB}GB — consider switching to smart mode" - notify "Ramdisk usage high on $(hostname) — ${RAMDISK_USED_GB}GB used in ramdisk-only mode" "Transcode Manager" "warning" + warn "Ramdisk usage ${RAMDISK_USED_GB}GB above threshold ${RAMDISK_WARN_GB}GB — consider switching to smart mode" + notify "Ramdisk high on $(hostname) ($MY_ID) — ${RAMDISK_USED_GB}GB in ramdisk-only mode" \ + "Transcode Manager" "warning" + SOMETHING_HAPPENED=true fi fi ;; ssd) - info "$ICON_DISK Mode: SSD — forcing symlink to SSD" + log "Mode: SSD — forcing symlink to SSD" if [[ "$SSD_HEALTHY" == false ]]; then error "SSD mode selected but SSD path is not available" + SOMETHING_HAPPENED=true else if [[ "$CURRENT_TARGET" != "$TRANSCODE_SSD" ]]; then flip_symlink "$TRANSCODE_SSD" "ssd mode" increment_flip_count > /dev/null - else - log "Symlink already points to SSD — no change" + FLIP_COUNT=$(get_flip_count) + SOMETHING_HAPPENED=true fi fi ;; smart) - info "$ICON_GEAR Mode: SMART — auto threshold management" + log "Mode: SMART — auto threshold management" if [[ "$EMBY_RUNNING" == false ]]; then - info "No media servers running — skipping threshold checks" + log "No media servers running — skipping threshold checks" elif [[ "$RAMDISK_HEALTHY" == false ]]; then - info "Ramdisk unavailable — staying on SSD until ramdisk recovers" + log "Ramdisk unavailable — staying on SSD until ramdisk recovers" elif [[ "$CURRENT_TARGET" == "$RAMDISK_PATH" ]]; then if (( $(awk "BEGIN {print ($RAMDISK_USED_GB >= $RAMDISK_WARN_GB) ? 1 : 0}") )); then if [[ "$SSD_HEALTHY" == false ]]; then error "Ramdisk above threshold but SSD unavailable — cannot flip" - notify "Transcode ramdisk full on $(hostname) and SSD unavailable — intervention needed" "Transcode Manager" "warning" + notify "Transcode ramdisk full on $(hostname) ($MY_ID) and SSD unavailable" \ + "Transcode Manager" "warning" + SOMETHING_HAPPENED=true else SSD_FREE_GB=$(get_ssd_free_gb) if (( $(awk "BEGIN {print ($SSD_FREE_GB < $RAMDISK_SSD_MIN_GB) ? 1 : 0}") )); then - warn "$ICON_DISK SSD only ${SSD_FREE_GB}GB free — below ${RAMDISK_SSD_MIN_GB}GB minimum, not flipping" + warn "SSD only ${SSD_FREE_GB}GB free — below ${RAMDISK_SSD_MIN_GB}GB minimum, not flipping" + SOMETHING_HAPPENED=true else - warn "$ICON_RAM Ramdisk ${RAMDISK_USED_GB}GB — above ${RAMDISK_WARN_GB}GB, flipping to SSD" + warn "Ramdisk ${RAMDISK_USED_GB}GB — above ${RAMDISK_WARN_GB}GB, flipping to SSD" flip_symlink "$TRANSCODE_SSD" "threshold exceeded" CURRENT_TARGET="$TRANSCODE_SSD" NEW_COUNT=$(increment_flip_count) FLIP_COUNT=$NEW_COUNT + SOMETHING_HAPPENED=true if [[ "$NEW_COUNT" -ge "$TRANSCODE_FLIP_WARN" ]]; then - notify "Transcode flipped to SSD on $(hostname) — ${NEW_COUNT} flips this hour. Consider increasing RAMDISK_SIZE." "Transcode Manager" "warning" + notify "Transcode flipped to SSD on $(hostname) ($MY_ID) — ${NEW_COUNT} flips this hour — consider increasing HOST*_RAMDISK_SIZE" \ + "Transcode Manager" "warning" fi fi fi else - log "Symlink already points to ramdisk — no change" + log "Ramdisk ${RAMDISK_USED_GB}GB — below threshold — no action needed" fi elif [[ "$CURRENT_TARGET" == "$TRANSCODE_SSD" ]]; then if (( $(awk "BEGIN {print ($RAMDISK_USED_GB <= $RAMDISK_LOW_GB) ? 1 : 0}") )); then - info "$ICON_RAM Ramdisk ${RAMDISK_USED_GB}GB — below ${RAMDISK_LOW_GB}GB, flipping back to ramdisk" + warn "Ramdisk ${RAMDISK_USED_GB}GB — below ${RAMDISK_LOW_GB}GB, flipping back to ramdisk" flip_symlink "$RAMDISK_PATH" "usage recovered" CURRENT_TARGET="$RAMDISK_PATH" NEW_COUNT=$(increment_flip_count) FLIP_COUNT=$NEW_COUNT + SOMETHING_HAPPENED=true else log "On SSD — ramdisk ${RAMDISK_USED_GB}GB still above low threshold ${RAMDISK_LOW_GB}GB" fi @@ -547,16 +549,32 @@ esac END=$(date +%s) CURRENT_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown") -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━━━ $ICON_SUMMARY TRANSCODE MANAGER SUMMARY ━━━━━" -echo "$ICON_GEAR Mode: $TRANSCODE_MANAGER_MODE" -echo "$ICON_RAM Ramdisk: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB available" -echo "$ICON_LINK Symlink: $TRANSCODE_LINK → $CURRENT_TARGET" -echo "$ICON_LINK Flips: $FLIP_COUNT this hour (warn at $TRANSCODE_FLIP_WARN)" -echo "$ICON_EMBY Streams: ${TOTAL_SESSIONS} total | Live TV: ${LIVE_TV} | Transcoding: ${TRANSCODING} | Direct: ${DIRECT}" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -echo "$ICON_DONE Status: $ICON_SUCCESS DONE" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file +# ============================================================================================== +# ━━━ Daily Log Write ━━━ +# ============================================================================================== +# Read by weekly_health_digest.sh — format: DATE|USED_GB|FLIPS|RAM_SESSIONS|SSD_SESSIONS +if [[ "$DRY_RUN" == false && "$NO_LOG" == false && -n "${TRANSCODE_DAILY_LOG:-}" ]]; then + TODAY=$(date '+%Y-%m-%d') + mkdir -p "$(dirname "$TRANSCODE_DAILY_LOG")" + echo "${TODAY}|${RAMDISK_USED_GB}|${FLIP_COUNT}|${RAM_SESSION_COUNT}|${SSD_SESSION_COUNT}" \ + >> "$TRANSCODE_DAILY_LOG" 2>/dev/null || true + log "Daily log written: $TRANSCODE_DAILY_LOG" +fi + +# ============================================================================================== +# ━━━ Summary — only shown when something happened ━━━ +# ============================================================================================== +if [[ "$SOMETHING_HAPPENED" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY TRANSCODE MANAGER SUMMARY ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_GEAR Mode: $TRANSCODE_MANAGER_MODE" + echo "$ICON_RAM Ramdisk: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB available" + echo "$ICON_LINK Symlink: $TRANSCODE_LINK → $CURRENT_TARGET" + echo "$ICON_LINK Flips: $FLIP_COUNT this hour (warn at $TRANSCODE_FLIP_WARN)" + echo "$ICON_EMBY Streams: $TOTAL_SESSIONS total | Live TV: $LIVE_TV | Transcoding: $TRANSCODING | Direct: $DIRECT" + echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +else + log "Transcode manager — all healthy — $MY_ID ($(format_duration $(( END - START ))))" +fi \ No newline at end of file diff --git a/common.sh b/common.sh index c11c238..ce0be4e 100644 --- a/common.sh +++ b/common.sh @@ -1,81 +1,110 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ---------------------------- -# ----------------------------------------------------------------------------------------------- -# Version: 2.9 -# ----------------------------------------------------------------------------------------------- -# Changelog: -# v1.0 — Initial stable framework -# v1.1 — format_duration moved here from daily_sync_maintenance.sh for shared use -# SSH_KEY collision resolved — gitea key renamed GITEA_SSH_KEY in Master.conf -# Version and changelog tracking added -# v1.2 — Consistent function header comment blocks across all functions -# check_connectivity added as standalone function -# check_connectivity friendlier error output with tailscale hint -# v1.3 — check_remote_rootfs added — aborts if remote rootfs exceeds ROOTFS_WARN threshold -# check_remote_share added — aborts if target directory is missing or empty on remote -# Both protect against rsync running when remote array is down or drives are missing -# v1.4 — check_remote_disks added — verifies all physical disks backing a share are mounted -# Discovers disk layout automatically at runtime, no configuration required -# Aborts if any single disk backing the share is offline or unmounted -# v1.5 — Full icon set expanded — each operation and state has its own distinct icon -# All function output updated to use correct icon per context -# Icons grouped and commented by category for clarity -# v1.6 — ICON_CONTAINERS added — 📦 anchors all container sections for visual consistency -# ICON_NOT_RUNNING changed to ⭕ — distinct from ICON_STOPPED 🔴 -# Section dividers updated from --- to ━━━ for cleaner log readability -# Summary passed/failed lines use ICON_SUCCESS and ICON_ERROR consistently -# v1.7 — ICON_MOVER added for mover operations -# ICON_CONTAINERS replaces ICON_DOCKER for docker/container operations -# validate_int added — reusable integer validation for any script -# v1.8 — ICON_PHP added for PHP-FPM operations -# v1.9 — ICON_REBOOT added for server reboot operations -# v2.0 — ICON_PLUGIN added for User Scripts plugin operations -# v2.1 — ICON_ZFS and ICON_MEM added for ZFS and memory diagnostics -# Diagnostics icon group added to icon block -# v2.2 — ICON_WATCHDOG added for Docker watchdog monitoring operations -# v2.3 — ICON_NOTIFY added for notification operations -# notify() added — shared notification function supporting unRAID native and Discord -# NOTIFY_UNRAID and DISCORD_WEBHOOK configured in Master.conf -# v2.4 — ICON_CLEAN, ICON_TRASH added for media cleaner operations -# ICON_PERMS, ICON_UNLOCKED added for media permissions operations -# ICON_REBOOT_SMART added for smart conditional reboot -# v2.5 — ICON_RAM added for ramdisk operations -# ICON_LINK added for symlink state and management -# Transcode scripts group added to ecosystem -# v2.6 — ICON_FAILOVER added for failover operations -# check_local_array added — verifies local /mnt/user is mounted and healthy -# check_remote_array added — verifies remote /mnt/user is mounted and healthy -# check_remote_docker added — verifies remote Docker daemon is responding -# ping_remote added — non-fatal ping returning status for failover use -# ping_internet added — non-fatal external ping for failover use -# v2.7 — ICON_WEBGUI added for WebGUI watchdog operations -# ICON_DOCKER_NET added for Docker network connect operations -# v2.8 — ICON_CERT added for SSL certificate monitoring operations -# v2.9 — ICON_MONITOR added for monitoring section headers -# v3.0 — Script locking system added — prevents concurrent execution conflicts -# acquire_lock() — create PID lock file, register EXIT trap -# release_lock() — remove lock file on exit -# acquire_rsync_lock() — per-profile lock + global concurrent limit -# release_rsync_lock() — decrement global counter, remove profile lock -# check_api() — pre-flight API reachability check -# ICON_LOCK added — 🔏 script instance lock acquired/released -# ICON_SMART added for drive SMART health operations -# ICON_BANDWIDTH added for bandwidth tracking operations -# ICON_DIGEST added for health digest operations -# ICON_EMBY added for Emby session reporting -# ICON_VERIFY added for backup verification operations -# Monitor/ folder added to ecosystem -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= COMMON LIBRARY ============================================= +# ============================================================================================== +# Shared functions used by every script in the ecosystem. +# Sourced automatically by load_config.sh — do not source directly. +# +# ── WHAT THIS FILE PROVIDES ─────────────────────────────────────────────────────────────────── +# Icons — consistent visual language across all script output +# Output helpers — info, warn, error, success, log, notify +# Arg parser — unified --dry-run, --log, --status, KEY=VALUE handling +# Validation — require_var, validate_int +# Host detection — detect_hosts() identifies local/remote server, sets all aliases +# Connectivity — resolve_remote_ip, check_connectivity, ping_remote, ping_internet +# Health checks — local/remote array, docker daemon, rootfs, share, disk temps +# Disk verification — check_remote_disks() — XFS, ZFS, cache pools, nested paths +# Container mgmt — stop/start local and remote containers with state tracking +# Rsync helpers — get_rsync_opts, check_rsync_enabled +# Locking — acquire_lock, acquire_rsync_lock — concurrent execution protection +# Path translation — translate_path() — container → host path for arr cleanup scripts +# Arr utilities — check_arr_version() — API version safety gate +# Status display — show_status() — runtime config dump on --status flag +# +# ── CHANGELOG ───────────────────────────────────────────────────────────────────────────────── +# +# Version What changed +# ─────────────────────────────────────────────────────────────────────────────────────────── +# v1.0 Initial stable framework — output helpers, arg parser, host detection +# +# v1.1 format_duration() moved here from daily_sync_maintenance.sh for shared use +# SSH_KEY naming collision resolved — gitea key renamed GITEA_SSH_KEY in master.conf +# +# v1.2 check_connectivity() added — fatal ping with Tailscale hint on failure +# Function header comment blocks standardised across all functions +# +# v1.3 check_remote_rootfs() — aborts rsync if remote rootfs exceeds ROOTFS_WARN +# check_remote_share() — aborts if target directory missing or empty on remote +# Both protect against rsync running when remote array is down +# +# v1.4 check_remote_disks() — verifies all physical disks backing a share are mounted +# Auto-discovers disk layout at runtime from disks.ini — no config required +# Aborts if any disk backing the share is offline or unmounted +# +# v1.5-2.9 Icon set expanded progressively — each operation has its own distinct icon +# All function output updated to use correct icon per context +# notify() added — unRAID native + Discord webhook notifications +# Script locking system added — acquire_lock(), acquire_rsync_lock() +# translate_path() added — container → host path for arr cleanup scripts +# check_arr_version() added — API version safety gate before arr operations +# check_api() added — pre-flight API reachability check +# ping_remote(), ping_internet() — non-fatal ping for failover use +# check_local_array(), check_remote_array() — array health checks +# check_remote_docker() — Docker daemon health check +# get_unraid_temp_thresholds() — reads thresholds from dynamix.cfg +# check_local_disk_temps() — pre-rsync temp check with exit codes 0/1/2 +# stop/start local containers added alongside existing remote variants +# +# v3.1 Three-file config split — master.conf + master_host1.conf + master_host2.conf +# load_config.sh introduced — auto-discovers all master_host*.conf files +# detect_hosts() rewritten — sets MY_ID/REMOTE_ID and aliases all HOST* vars +# All scripts now source load_config.sh instead of conf files directly +# Adding a new server = add master_host*.conf, zero script changes required +# +# v3.2 check_remote_disks() rewritten — three-tier detection: +# Tier 1: array disk paths (/mnt/disk*/sharename) +# Tier 2: ZFS pools with find -maxdepth 2 (catches nested cache paths) +# Tier 3: shfs fallback — verifies /mnt/user is mounted +# Fixes: nested paths like /mnt/cache/appdata-Failover/Critical-Data +# Fixes: ZFS cache pools not detected when share is not at pool root +# +# v3.3 PROFILE_REMOTE_RESTART_CONTAINERS support added +# Dirty sync profiles (critical-failover, emby-failover) restart remote +# containers after sync if they were running before — picks up config changes +# Was stopped → stays stopped. Was running → gets restarted. ✅ +# +# v3.5 notify_emby_scan() added — triggers Emby "Clean Missing Files" task +# after arr cleanup scripts delete orphaned files +# Emby immediately removes ghost entries — no user-facing file-not-found errors +# Called by lidarr/sonarr/radarr_cleanup.sh when files are deleted +# +# v3.4 Silent-by-default output model +# info() and success() now gated by SILENT_MODE — only warn/error always visible +# Exception: monitor scripts designed to produce output stay verbose +# Reduces notification spam — ecosystem only speaks when something is wrong +# +# Three new safety functions added: +# check_unraid_version_parity() — refuses remote ops on version mismatch +# reads /etc/unraid-version local and remote via SSH +# major mismatch → abort | minor mismatch → configurable warn/abort +# check_remote_docker_daemon() — verifies remote Docker daemon before +# issuing any remote container commands — strike system → skip/retry/exit +# validate_unraid_cmd() — verifies unRAID-specific commands exist and +# produce expected output before use — notifies and exits calling script +# if command changed or disappeared after upgrade — other scripts unaffected +# +# ── VERSION ─────────────────────────────────────────────────────────────────────────────────── +# Current: v3.5 +# ============================================================================================== -# ----------------------------------------------------------------------------------------------- -# ICONS +# ============================================================================================== +# ── ICONS ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== # Each icon has one job — do not reuse across different contexts. -# Adding a new icon: add it to the appropriate group below with a comment describing its job. -# ----------------------------------------------------------------------------------------------- +# Adding a new icon: add to the appropriate group below with a comment describing its job. # System / Host -ICON_HOST="🖥️" # host detection +ICON_HOST="🖥️" # host detection and identity ICON_NET="🌐" # network / IP resolution ICON_PING="📡" # connectivity check ICON_GEAR="⚙️" # setup section header / profile load @@ -156,27 +185,49 @@ ICON_WARN="⚠️" ICON_ERROR="❌" ICON_SUCCESS="✅" -# ----------------------------------------------------------------------------------------------- -# OUTPUT HELPERS +# ============================================================================================== +# ── OUTPUT HELPERS ──────────────────────────────────────────────────────────────────────────── +# ============================================================================================== # Standardised output functions used across all scripts. -# log() is gated by ENABLE_LOGGING — set in Master.conf or via --log flag. -# ----------------------------------------------------------------------------------------------- -info() { echo "$ICON_INFO [INFO] $*"; } +# +# Silent-by-default model: +# SILENT_MODE=true (default) — only warn() and error() produce output +# SILENT_MODE=false — all functions produce output +# --log flag — enables ENABLE_LOGGING (detailed [LOG] lines) +# +# Rules: +# error() — always visible — something broke +# warn() — always visible — something needs attention +# info() — silent by default — operational detail, visible when SILENT_MODE=false +# success() — silent by default — confirmation, visible when SILENT_MODE=false +# log() — debug detail — only when ENABLE_LOGGING=true +# +# Exception — monitor scripts are designed to produce output and set SILENT_MODE=false +# at the top of the script. All other scripts use the silent default. +# +# All output goes to stdout — callers can redirect as needed. + +info() { [[ "${SILENT_MODE:-true}" == false ]] && echo "$ICON_INFO [INFO] $*"; return 0; } warn() { echo "$ICON_WARN [WARN] $*"; } error() { echo "$ICON_ERROR [ERROR] $*"; } -success() { echo "$ICON_SUCCESS [OK] $*"; } +success() { [[ "${SILENT_MODE:-true}" == false ]] && echo "$ICON_SUCCESS [OK] $*"; return 0; } log() { [[ "${ENABLE_LOGGING:-false}" == true ]] && echo "[LOG] $*" + return 0 } -# ----------------------------------------------------------------------------------------------- -# NOTIFICATION +# ============================================================================================== +# ── NOTIFICATION ────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== # Sends a notification via unRAID native system and/or Discord webhook. -# Both channels are optional and independently controlled via Master.conf. +# Both channels are optional and independently controlled: +# NOTIFY_UNRAID — shared toggle in master.conf +# MY_DISCORD_WEBHOOK — per-host in master_host*.conf, set by detect_hosts() +# # Severity levels: normal, warning, alert # Usage: notify "message" "subject" "severity" -# ----------------------------------------------------------------------------------------------- + notify() { local message="$1" local subject="${2:-unRAID Notification}" @@ -194,23 +245,27 @@ notify() { fi fi - if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then + # Uses MY_DISCORD_WEBHOOK — set by detect_hosts() from HOST*_DISCORD_WEBHOOK + if [[ -n "${MY_DISCORD_WEBHOOK:-}" ]]; then local payload payload=$(printf '{"content": "%s — **%s**\\n%s"}' \ "$ICON_NOTIFY" "$subject" "$message") if curl -s -H "Content-Type: application/json" \ - -d "$payload" "$DISCORD_WEBHOOK" >/dev/null 2>&1; then + -d "$payload" "$MY_DISCORD_WEBHOOK" >/dev/null 2>&1; then log "$ICON_NOTIFY Discord notification sent" else - warn "Discord notification failed — check DISCORD_WEBHOOK in Master.conf" + warn "Discord notification failed — check HOST*_DISCORD_WEBHOOK in master_host*.conf" fi fi } -# ----------------------------------------------------------------------------------------------- -# DURATION FORMATTER -# Converts raw seconds into a human readable string — e.g. 10m53s or 47s -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ── DURATION FORMATTER ──────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Converts raw seconds into a human readable string. +# Examples: 47 → "47s" | 653 → "10m53s" +# Usage: format_duration $SECONDS + format_duration() { local secs=$1 local mins=$((secs / 60)) @@ -218,13 +273,25 @@ format_duration() { [[ $mins -gt 0 ]] && echo "${mins}m${rem}s" || echo "${rem}s" } -# ----------------------------------------------------------------------------------------------- -# ARG PARSER +# ============================================================================================== +# ── ARG PARSER ──────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== # Processes all flags and key=value pairs passed to any script. -# Supported flags: --dry-run, --log, --no-log, --status, --help -# Supported key=value: LOG=true/false, or any declared variable e.g. BW_LIMIT=5000 +# Call early in every script: parse_args "$@" +# +# Supported flags: +# --dry-run | -n → DRY_RUN=true +# --log → ENABLE_LOGGING=true +# --no-log → ENABLE_LOGGING=false +# --status → SHOW_STATUS=true +# --help | -h → print usage and exit +# +# Supported key=value: +# LOG=true/false → toggle logging +# BW_LIMIT=5000 → override any declared master.conf variable for this run +# # Unparsed positional args returned in PARSED_ARGS array. -# ----------------------------------------------------------------------------------------------- + parse_args() { ENABLE_LOGGING=${ENABLE_LOGGING:-false} DRY_RUN=${DRY_RUN:-false} @@ -267,9 +334,9 @@ parse_args() { PARSED_ARGS=("${CLEAN_ARGS[@]}") } -# ----------------------------------------------------------------------------------------------- -# VALIDATION HELPERS -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ── VALIDATION HELPERS ──────────────────────────────────────────────────────────────────────── +# ============================================================================================== # Exits with error if a required variable is empty or unset. # Usage: require_var VAR_NAME @@ -282,7 +349,7 @@ require_var() { validate_int() { local name="$1" value="$2" if [[ -z "$value" ]]; then - error "$name is not set — check Master.conf" + error "$name is not set — check master.conf" exit 1 fi if ! [[ "$value" =~ ^[0-9]+$ ]]; then @@ -292,69 +359,290 @@ validate_int() { log "$name validated: $value" } -# ----------------------------------------------------------------------------------------------- -# HOST DETECTION +# ============================================================================================== +# ── HOST DETECTION ──────────────────────────────────────────────────────────────────────────── +# ============================================================================================== # Determines which server is local and which is remote by comparing hostname against -# HOST1 and HOST2 in Master.conf. Sets LOCAL_SERVER_NAME, REMOTE_SERVER_NAME and SSH_KEY. -# Both servers run identical scripts — this is what makes them bidirectional. -# ----------------------------------------------------------------------------------------------- -detect_hosts() { - LOCAL_HOSTNAME="$(hostname)" +# all HOST* values discovered from master_host*.conf files. +# +# Sets: +# MY_ID — "HOST1" or "HOST2" (the role key, not the hostname) +# REMOTE_ID — the peer's role key +# LOCAL_SERVER_NAME — local hostname string (e.g. "unRAID-Gmer4Lfe") +# REMOTE_SERVER_NAME — remote hostname string +# SSH_KEY — SSH key for server-to-server operations +# +# Also sets aliases for all host-specific arrays so scripts use unprefixed names: +# DAILY_SYNC_SHARES ← HOST*_DAILY_SYNC_SHARES +# WEEKLY_SYNC_SHARES ← HOST*_WEEKLY_SYNC_SHARES +# CRITICAL_SYNC_SHARES ← HOST*_CRITICAL_SYNC_SHARES +# DAILY_RESTART_CONTAINERS ← HOST*_DAILY_RESTART_CONTAINERS +# WEEKLY_RESTART_CONTAINERS ← HOST*_WEEKLY_RESTART_CONTAINERS +# WATCHDOG_CONTAINERS ← HOST*_WATCHDOG_CONTAINERS (associative) +# WATCHDOG_CONTAINER_URLS ← HOST*_WATCHDOG_CONTAINER_URLS (associative) +# WATCHDOG_REQUIRED_CONTAINERS ← HOST*_WATCHDOG_REQUIRED_CONTAINERS +# WATCHDOG_SCAN_IGNORE ← HOST*_WATCHDOG_SCAN_IGNORE +# WATCHDOG_DEPENDENCIES ← HOST*_WATCHDOG_DEPENDENCIES (associative) +# NETWORK_CONNECT_CONTAINERS ← HOST*_NETWORK_CONNECT_CONTAINERS +# NETWORK_CONNECT_NETWORKS ← HOST*_NETWORK_CONNECT_NETWORKS +# MEDIA_PERMISSION_SHARES ← HOST*_MEDIA_PERMISSION_SHARES +# ANIME_CLEAN_FOLDERS ← HOST*_ANIME_CLEAN_FOLDERS +# MEDIA_CLEAN_FOLDERS ← HOST*_MEDIA_CLEAN_FOLDERS +# CERT_MONITOR_DOMAINS ← HOST*_CERT_MONITOR_DOMAINS +# SMART_IGNORE_DRIVES ← HOST*_SMART_IGNORE_DRIVES +# ZFS_REPORT_IGNORE_POOLS ← HOST*_ZFS_REPORT_IGNORE_POOLS +# TRANSCODE_SSD ← HOST*_TRANSCODE_SSD +# TRANSCODE_SERVERS ← HOST*_TRANSCODE_SERVERS +# RAMDISK_SIZE ← HOST*_RAMDISK_SIZE +# RAMDISK_WARN_GB ← HOST*_RAMDISK_WARN_GB +# RAMDISK_LOW_GB ← HOST*_RAMDISK_LOW_GB +# BACKUP_VERIFY_SHARES ← HOST*_BACKUP_VERIFY_SHARES +# DDNS_CONTAINERS ← HOST*_DDNS_CONTAINERS +# PARTNERSHIP_AUTH_WEBUIS ← HOST*_PARTNERSHIP_AUTH_WEBUIS +# PARTNERSHIP_MIRROR_BACKUPS ← HOST*_PARTNERSHIP_MIRROR_BACKUPS +# MY_DISCORD_WEBHOOK ← HOST*_DISCORD_WEBHOOK +# EMBY_CONTAINER ← HOST*_EMBY_CONTAINER +# EMBY_URL ← HOST*_EMBY_URL +# EMBY_API_KEY ← HOST*_EMBY_API_KEY +# +# Arr-specific vars set by detect_hosts() when needed by arr scripts: +# LIDARR_URL / LIDARR_API_KEY / LIDARR_MUSIC_ROOT (MY_ID only — Lidarr is HOST1 only) +# SONARR_URL / SONARR_API_KEY / SONARR_TV_ROOT +# RADARR_URL / RADARR_API_KEY / RADARR_MOVIES_ROOT +# SLSKD_URL / SLSKD_API_KEY / SLSKD_FAILED_IMPORTS_DIR (MY_ID only) +# SABNZBD_URL / SABNZBD_API_KEY (MY_ID only) +# QBIT_URL / QBIT_USERNAME / QBIT_PASSWORD (MY_ID only) - if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then - LOCAL_SERVER_NAME="$HOST1" - REMOTE_SERVER_NAME="$HOST2" - elif [[ "$LOCAL_HOSTNAME" == "$HOST2" ]]; then - LOCAL_SERVER_NAME="$HOST2" - REMOTE_SERVER_NAME="$HOST1" - else - error "Unknown host: $LOCAL_HOSTNAME" +detect_hosts() { + local local_hostname + local_hostname="$(hostname)" + + # ── Find MY_ID by matching hostname against all HOST* vars ──────────────── + MY_ID="" + local host_var host_val + + # Scan all HOST* vars that hold a hostname value + # HOST1, HOST2, HOST3 etc. — all defined in master_host*.conf files + for host_var in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do + host_val="${!host_var:-}" + [[ -z "$host_val" ]] && continue + if [[ "$local_hostname" == "$host_val" ]]; then + MY_ID="$host_var" + break + fi + done + + if [[ -z "$MY_ID" ]]; then + error "Unknown host: $local_hostname" + error "Hostname must match a HOST* value in master_host*.conf" + error "Available: $(for h in HOST1 HOST2 HOST3 HOST4; do + [[ -n "${!h:-}" ]] && echo -n "${!h} "; done)" exit 1 fi - declare -A SSH_KEYS - SSH_KEYS["$HOST1|$HOST2"]="$HOST1_SSH_KEY" - SSH_KEYS["$HOST2|$HOST1"]="$HOST2_SSH_KEY" - SSH_KEY="${SSH_KEYS[$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME]}" + # ── Find REMOTE_ID — first HOST* that isn't MY_ID ──────────────────────── + REMOTE_ID="" + for host_var in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do + [[ "$host_var" == "$MY_ID" ]] && continue + host_val="${!host_var:-}" + [[ -z "$host_val" ]] && continue + REMOTE_ID="$host_var" + break + done - [[ -z "$SSH_KEY" ]] && error "Missing SSH key mapping" && exit 1 + # ── Set host name strings ───────────────────────────────────────────────── + LOCAL_SERVER_NAME="${!MY_ID}" + REMOTE_SERVER_NAME="${!REMOTE_ID:-unknown}" + + # ── Set SSH key ─────────────────────────────────────────────────────────── + local ssh_key_var="${MY_ID}_SSH_KEY" + SSH_KEY="${!ssh_key_var:-}" + if [[ -z "$SSH_KEY" ]]; then + error "Missing SSH key: ${ssh_key_var} not set in master_host*.conf" + exit 1 + fi + + # ── Set scalar aliases ──────────────────────────────────────────────────── + MY_DISCORD_WEBHOOK="${!MY_ID}_DISCORD_WEBHOOK" + MY_DISCORD_WEBHOOK="${!MY_DISCORD_WEBHOOK:-}" + + EMBY_CONTAINER_VAR="${MY_ID}_EMBY_CONTAINER" + EMBY_CONTAINER="${!EMBY_CONTAINER_VAR:-}" + EMBY_URL_VAR="${MY_ID}_EMBY_URL" + EMBY_URL="${!EMBY_URL_VAR:-}" + EMBY_API_KEY_VAR="${MY_ID}_EMBY_API_KEY" + EMBY_API_KEY="${!EMBY_API_KEY_VAR:-}" + + # ── System watchdog check toggles — per-host ───────────────────────────── + # Aliased as unprefixed SYS_WATCHDOG_CHECK_* for use in system_watchdog.sh + local _wd_checks=( + SYS_WATCHDOG_NIC + SYS_WATCHDOG_CHECK_DOCKER_DAEMON SYS_WATCHDOG_CHECK_ROOTFS + SYS_WATCHDOG_CHECK_KERNEL_OOPS SYS_WATCHDOG_CHECK_FD + SYS_WATCHDOG_CHECK_BOOT SYS_WATCHDOG_CHECK_OOM + SYS_WATCHDOG_CHECK_RAM SYS_WATCHDOG_CHECK_LOG + SYS_WATCHDOG_CHECK_ARC SYS_WATCHDOG_CHECK_CPU_TEMP + SYS_WATCHDOG_CHECK_LOAD SYS_WATCHDOG_CHECK_ZOMBIES + SYS_WATCHDOG_CHECK_CONTAINERS SYS_WATCHDOG_CHECK_TMP + SYS_WATCHDOG_CHECK_MDSTAT SYS_WATCHDOG_CHECK_NETWORK + SYS_WATCHDOG_CHECK_SSHD SYS_WATCHDOG_CHECK_RUNAWAY + ) + for _wd_var in "${_wd_checks[@]}"; do + local _wd_src="${MY_ID}_${_wd_var}" + # Only alias if the host-specific var is set — preserves master.conf defaults + [[ -n "${!_wd_src+x}" ]] && eval "${_wd_var}="\${!_wd_src}"" + done + + TRANSCODE_SSD_VAR="${MY_ID}_TRANSCODE_SSD" + TRANSCODE_SSD="${!TRANSCODE_SSD_VAR:-}" + RAMDISK_SIZE_VAR="${MY_ID}_RAMDISK_SIZE" + RAMDISK_SIZE="${!RAMDISK_SIZE_VAR:-8G}" + RAMDISK_WARN_GB_VAR="${MY_ID}_RAMDISK_WARN_GB" + RAMDISK_WARN_GB="${!RAMDISK_WARN_GB_VAR:-6.8}" + RAMDISK_LOW_GB_VAR="${MY_ID}_RAMDISK_LOW_GB" + RAMDISK_LOW_GB="${!RAMDISK_LOW_GB_VAR:-5.5}" + + # Arr credentials — MY_ID only (remote arr accessed via its own host) + LIDARR_URL_VAR="${MY_ID}_LIDARR_URL"; LIDARR_URL="${!LIDARR_URL_VAR:-}" + LIDARR_API_KEY_VAR="${MY_ID}_LIDARR_API_KEY"; LIDARR_API_KEY="${!LIDARR_API_KEY_VAR:-}" + LIDARR_MUSIC_ROOT_VAR="${MY_ID}_LIDARR_MUSIC_ROOT"; LIDARR_MUSIC_ROOT="${!LIDARR_MUSIC_ROOT_VAR:-}" + + SONARR_URL_VAR="${MY_ID}_SONARR_URL"; SONARR_URL="${!SONARR_URL_VAR:-}" + SONARR_API_KEY_VAR="${MY_ID}_SONARR_API_KEY"; SONARR_API_KEY="${!SONARR_API_KEY_VAR:-}" + SONARR_TV_ROOT_VAR="${MY_ID}_SONARR_TV_ROOT"; SONARR_TV_ROOT="${!SONARR_TV_ROOT_VAR:-}" + + RADARR_URL_VAR="${MY_ID}_RADARR_URL"; RADARR_URL="${!RADARR_URL_VAR:-}" + RADARR_API_KEY_VAR="${MY_ID}_RADARR_API_KEY"; RADARR_API_KEY="${!RADARR_API_KEY_VAR:-}" + RADARR_MOVIES_ROOT_VAR="${MY_ID}_RADARR_MOVIES_ROOT"; RADARR_MOVIES_ROOT="${!RADARR_MOVIES_ROOT_VAR:-}" + + SLSKD_URL_VAR="${MY_ID}_SLSKD_URL"; SLSKD_URL="${!SLSKD_URL_VAR:-}" + SLSKD_API_KEY_VAR="${MY_ID}_SLSKD_API_KEY"; SLSKD_API_KEY="${!SLSKD_API_KEY_VAR:-}" + SLSKD_FAILED_IMPORTS_DIR_VAR="${MY_ID}_SLSKD_FAILED_IMPORTS_DIR" + SLSKD_FAILED_IMPORTS_DIR="${!SLSKD_FAILED_IMPORTS_DIR_VAR:-}" + + SABNZBD_URL_VAR="${MY_ID}_SABNZBD_URL"; SABNZBD_URL="${!SABNZBD_URL_VAR:-}" + SABNZBD_API_KEY_VAR="${MY_ID}_SABNZBD_API_KEY"; SABNZBD_API_KEY="${!SABNZBD_API_KEY_VAR:-}" + + QBIT_URL_VAR="${MY_ID}_QBIT_URL"; QBIT_URL="${!QBIT_URL_VAR:-}" + QBIT_USERNAME_VAR="${MY_ID}_QBIT_USERNAME"; QBIT_USERNAME="${!QBIT_USERNAME_VAR:-}" + QBIT_PASSWORD_VAR="${MY_ID}_QBIT_PASSWORD"; QBIT_PASSWORD="${!QBIT_PASSWORD_VAR:-}" + + # ── Set array aliases — indexed arrays ──────────────────────────────────── + # Each eval copies the host-specific array into the unprefixed name scripts use + + _alias_array() { + local alias_name="$1" + local source_var="${MY_ID}_${alias_name}" + eval "${alias_name}=(\"\${${source_var}[@]:-}\")" + } + + _alias_array "DAILY_SYNC_SHARES" + _alias_array "PERSONAL_SHARES" + _alias_array "WEEKLY_SYNC_SHARES" + _alias_array "CRITICAL_SYNC_SHARES" + _alias_array "BACKUP_VERIFY_SHARES" + _alias_array "DAILY_RESTART_CONTAINERS" + _alias_array "WEEKLY_RESTART_CONTAINERS" + _alias_array "WATCHDOG_REQUIRED_CONTAINERS" + _alias_array "WATCHDOG_SCAN_IGNORE" + _alias_array "NETWORK_CONNECT_CONTAINERS" + _alias_array "NETWORK_CONNECT_NETWORKS" + _alias_array "MEDIA_PERMISSION_SHARES" + _alias_array "ANIME_CLEAN_FOLDERS" + _alias_array "MEDIA_CLEAN_FOLDERS" + _alias_array "CERT_MONITOR_DOMAINS" + _alias_array "SMART_IGNORE_DRIVES" + _alias_array "ZFS_REPORT_IGNORE_POOLS" + _alias_array "TRANSCODE_SERVERS" + _alias_array "DDNS_CONTAINERS" + _alias_array "PARTNERSHIP_AUTH_WEBUIS" + _alias_array "PARTNERSHIP_MIRROR_BACKUPS" + + # ── Set array aliases — associative arrays ──────────────────────────────── + # Associative arrays cannot be copied with eval — must be rebuilt key by key + + _alias_assoc() { + local alias_name="$1" + local source_var="${MY_ID}_${alias_name}" + # Declare the alias as associative + eval "declare -gA ${alias_name}" + # Copy each key from source to alias + local key + eval "for key in \"\${!${source_var}[@]:-}\"; do + eval \"${alias_name}[\\\"\$key\\\"]=\\\"\${${source_var}[\\\"\$key\\\"]}\\\"\" + done" + } + + _alias_assoc "WATCHDOG_CONTAINERS" + _alias_assoc "WATCHDOG_CONTAINER_URLS" + _alias_assoc "WATCHDOG_DEPENDENCIES" + + # ── Output ──────────────────────────────────────────────────────────────── info "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" } -# ----------------------------------------------------------------------------------------------- -# REMOTE IP RESOLUTION +# ============================================================================================== +# ── REMOTE IP RESOLUTION ────────────────────────────────────────────────────────────────────── +# ============================================================================================== # Resolves the Tailscale IPv4 address of the remote server. # Sets REMOTE_SERVER used by all subsequent SSH and rsync calls. # Exits if resolution fails — Tailscale may be down or peer offline. -# ----------------------------------------------------------------------------------------------- + resolve_remote_ip() { log "Resolving remote IP for $REMOTE_SERVER_NAME..." REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) - [[ -z "$REMOTE_SERVER" ]] && error "Failed to resolve Tailscale IP for $REMOTE_SERVER_NAME" && exit 1 + if [[ -z "$REMOTE_SERVER" ]]; then + error "Failed to resolve Tailscale IP for $REMOTE_SERVER_NAME" + error "Check: tailscale status | grep $REMOTE_SERVER_NAME" + exit 1 + fi info "$ICON_NET Remote IP: $REMOTE_SERVER" } -# ----------------------------------------------------------------------------------------------- -# CONNECTIVITY CHECK — fatal, used by rsync scripts -# Pings remote and exits if unreachable. -# For failover use ping_remote() which returns status without exiting. -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ── CONNECTIVITY CHECKS ─────────────────────────────────────────────────────────────────────── +# ============================================================================================== -# ----------------------------------------------------------------------------------------------- -# check_rsync_enabled — two-tier rsync gate check -# Tier 1: RSYNC_ENABLED — global, overrides everything -# Tier 2: orchestrator-specific flag passed as argument +# Fatal connectivity check — used by rsync and other scripts that must abort if unreachable. +# For failover use ping_remote() which returns status without exiting. +check_connectivity() { + log "Checking connectivity to $REMOTE_SERVER..." + if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then + error "$ICON_PING Remote $REMOTE_SERVER ($REMOTE_SERVER_NAME) is unreachable" + info "Hint: tailscale status | grep $REMOTE_SERVER_NAME" + exit 1 + fi + info "$ICON_PING $REMOTE_SERVER_NAME is reachable" +} + +# Non-fatal ping — used by failover.sh which handles its own state machine. +# Returns 0 if reachable, 1 if not — does NOT exit. +ping_remote() { + ping -c2 -W3 "$REMOTE_SERVER" &>/dev/null +} + +# Non-fatal external connectivity check — used by failover.sh. +# Returns 0 if internet reachable, 1 if not — does NOT exit. +ping_internet() { + ping -c2 -W3 "${EXTERNAL_IP:-8.8.8.8}" &>/dev/null +} + +# ============================================================================================== +# ── RSYNC GATE CHECK ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Two-tier rsync enable/disable system. +# Tier 1: RSYNC_ENABLED — global gate, overrides everything +# Tier 2: orchestrator-specific flag — fine-grained control per window # # Usage: # check_rsync_enabled "DAILY" ← checks RSYNC_ENABLED + DAILY_RSYNC_ENABLED # check_rsync_enabled "WEEKLY" ← checks RSYNC_ENABLED + WEEKLY_RSYNC_ENABLED +# check_rsync_enabled "CRITICAL" ← checks RSYNC_ENABLED + CRITICAL_RSYNC_ENABLED # check_rsync_enabled "FAILOVER" ← checks RSYNC_ENABLED + FAILOVER_RSYNC_ENABLED -# check_rsync_enabled ← checks RSYNC_ENABLED only (rsync.sh direct call) +# check_rsync_enabled ← checks RSYNC_ENABLED only (direct rsync.sh call) # -# Returns: -# 0 = rsync enabled, proceed -# 1 = rsync disabled, skip cleanly -# ----------------------------------------------------------------------------------------------- +# Returns: 0 = enabled, proceed | 1 = disabled, skip cleanly + check_rsync_enabled() { local orchestrator="${1:-}" @@ -378,38 +666,13 @@ check_rsync_enabled() { return 0 } -check_connectivity() { - log "Checking connectivity to $REMOTE_SERVER..." - if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then - error "$ICON_PING Remote $REMOTE_SERVER ($REMOTE_SERVER_NAME) is unreachable" - info "Hint: tailscale status | grep $REMOTE_SERVER_NAME" - exit 1 - fi - info "$ICON_PING $REMOTE_SERVER_NAME is reachable" -} +# ============================================================================================== +# ── LOCAL HEALTH CHECKS ─────────────────────────────────────────────────────────────────────── +# ============================================================================================== -# ----------------------------------------------------------------------------------------------- -# PING REMOTE — non-fatal, used by failover -# Returns 0 if reachable, 1 if not — does NOT exit. -# ----------------------------------------------------------------------------------------------- -ping_remote() { - ping -c2 -W3 "$REMOTE_SERVER" &>/dev/null -} - -# ----------------------------------------------------------------------------------------------- -# PING INTERNET — non-fatal external connectivity check -# Returns 0 if internet reachable, 1 if not — does NOT exit. -# ----------------------------------------------------------------------------------------------- -ping_internet() { - ping -c2 -W3 "${EXTERNAL_IP:-8.8.8.8}" &>/dev/null -} - -# ----------------------------------------------------------------------------------------------- -# LOCAL ARRAY CHECK — non-fatal, returns status # Verifies local /mnt/user is mounted and has shares. +# Non-fatal — returns status for caller to decide. # Used by failover before starting remote containers locally. -# Returns 0 if healthy, 1 if not. -# ----------------------------------------------------------------------------------------------- check_local_array() { log "Checking local array..." if ! mountpoint -q /mnt/user 2>/dev/null; then @@ -426,12 +689,13 @@ check_local_array() { return 0 } -# ----------------------------------------------------------------------------------------------- -# REMOTE ARRAY CHECK — non-fatal, returns status +# ============================================================================================== +# ── REMOTE HEALTH CHECKS ───────────────────────────────────────────────────────────────────── +# ============================================================================================== + # Verifies remote /mnt/user is mounted via SSH. -# Used before handback rsync — syncing to remote with no array fills rootfs. -# Returns 0 if healthy, 1 if not. -# ----------------------------------------------------------------------------------------------- +# Non-fatal — returns status. Used before handback rsync. +# Syncing to remote with no array fills rootfs rapidly. check_remote_array() { log "Checking remote array on $REMOTE_SERVER_NAME..." local result @@ -445,12 +709,8 @@ check_remote_array() { return 0 } -# ----------------------------------------------------------------------------------------------- -# REMOTE DOCKER CHECK — non-fatal, returns status -# Verifies remote Docker daemon is responding before container operations. -# A hung daemon means start/stop commands will silently fail. -# Returns 0 if healthy, 1 if not. -# ----------------------------------------------------------------------------------------------- +# Verifies remote Docker daemon is responding. +# Non-fatal — returns status. A hung daemon means container commands silently fail. check_remote_docker() { log "Checking remote Docker daemon on $REMOTE_SERVER_NAME..." if ! ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ @@ -462,44 +722,44 @@ check_remote_docker() { return 0 } -# ----------------------------------------------------------------------------------------------- -# REMOTE ROOTFS SPACE CHECK — fatal -# Aborts if remote rootfs exceeds ROOTFS_WARN threshold. -# ----------------------------------------------------------------------------------------------- +# Aborts if remote rootfs (/) usage is at or above ROOTFS_WARN threshold. +# Fatal — exits the calling script. +# When remote array is down rsync writes land on rootfs and fill it rapidly. check_remote_rootfs() { log "Checking remote rootfs usage..." - REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + local remote_usage + remote_usage=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null) - if [[ -z "$REMOTE_USAGE" ]]; then + if [[ -z "$remote_usage" ]]; then error "Could not retrieve rootfs usage from $REMOTE_SERVER_NAME" exit 1 fi - if [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then - error "$ICON_HEALTH Remote rootfs ${REMOTE_USAGE}% — threshold ${ROOTFS_WARN:-75}%" + if [[ "$remote_usage" -ge "${ROOTFS_WARN:-75}" ]]; then + error "$ICON_HEALTH Remote rootfs ${remote_usage}% — threshold ${ROOTFS_WARN:-75}%" exit 1 fi - info "$ICON_HEALTH Remote rootfs: ${REMOTE_USAGE}% (threshold: ${ROOTFS_WARN:-75}%)" + info "$ICON_HEALTH Remote rootfs: ${remote_usage}% (threshold: ${ROOTFS_WARN:-75}%)" } -# ----------------------------------------------------------------------------------------------- -# REMOTE SHARE VALIDATION — fatal # Verifies target directory exists and is not empty on remote. +# Fatal — exits the calling script. # Usage: check_remote_share "/mnt/user/Movies" -# ----------------------------------------------------------------------------------------------- check_remote_share() { local dir="$1" log "Checking remote share: $dir..." - SHARE_EXISTS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + local share_exists + share_exists=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null) - if [[ "$SHARE_EXISTS" != "yes" ]]; then + if [[ "$share_exists" != "yes" ]]; then error "$ICON_HEALTH Remote share does not exist: $dir" exit 1 fi - SHARE_EMPTY=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + local share_empty + share_empty=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null) - if [[ "$SHARE_EMPTY" == "yes" ]]; then + if [[ "$share_empty" == "yes" ]]; then warn "$ICON_HEALTH Remote share exists but is empty: $dir — aborting to protect data" exit 1 fi @@ -507,38 +767,38 @@ check_remote_share() { info "$ICON_HEALTH Remote share verified: $dir" } -# ----------------------------------------------------------------------------------------------- -# get_unraid_temp_thresholds — read disk temp thresholds from unRAID's dynamix.cfg +# ============================================================================================== +# ── DISK TEMPERATURE CHECKS ─────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# Reads disk temperature thresholds from unRAID's dynamix.cfg. # Sets globals: UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX -# Falls back to safe defaults if file not found -# ----------------------------------------------------------------------------------------------- +# Falls back to values in master.conf (SMART_TEMP_WARN/CRIT) if file not found. get_unraid_temp_thresholds() { local cfg="/boot/config/plugins/dynamix/dynamix.cfg" if [[ -f "$cfg" ]]; then - UNRAID_DISK_HOT=$(grep '^hot=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"') - UNRAID_DISK_MAX=$(grep '^max=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"') - UNRAID_SSD_HOT=$(grep '^hotssd=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"') - UNRAID_SSD_MAX=$(grep '^maxssd=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"') + UNRAID_DISK_HOT=$(grep '^hot=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"') + UNRAID_DISK_MAX=$(grep '^max=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"') + UNRAID_SSD_HOT=$(grep '^hotssd=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"') + UNRAID_SSD_MAX=$(grep '^maxssd=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"') fi - # Safe defaults if not found - UNRAID_DISK_HOT="${UNRAID_DISK_HOT:-45}" - UNRAID_DISK_MAX="${UNRAID_DISK_MAX:-55}" + # Fall back to master.conf values if dynamix.cfg not found or values empty + UNRAID_DISK_HOT="${UNRAID_DISK_HOT:-${SMART_TEMP_WARN:-45}}" + UNRAID_DISK_MAX="${UNRAID_DISK_MAX:-${SMART_TEMP_CRIT:-55}}" UNRAID_SSD_HOT="${UNRAID_SSD_HOT:-60}" UNRAID_SSD_MAX="${UNRAID_SSD_MAX:-70}" } -# ----------------------------------------------------------------------------------------------- -# check_local_disk_temps — check local disk temps before rsync -# Reads temps and rotational flag from /var/local/emhttp/disks.ini -# Uses unRAID's own thresholds from dynamix.cfg +# Checks local disk temperatures before rsync. +# Reads temps and rotational flag from /var/local/emhttp/disks.ini. +# Uses unRAID's own thresholds from dynamix.cfg via get_unraid_temp_thresholds(). # # Returns: -# 0 = all temps OK -# 1 = warn threshold exceeded (skip this profile) -# 2 = critical threshold exceeded (abort all remaining profiles) +# 0 = all temps OK — proceed +# 1 = warn threshold exceeded — skip this profile only +# 2 = critical threshold exceeded — abort all remaining profiles # -# Sets global TEMP_CHECK_RESULT with human readable summary -# ----------------------------------------------------------------------------------------------- +# Sets TEMP_CHECK_RESULT with human readable summary for logging. check_local_disk_temps() { get_unraid_temp_thresholds @@ -559,7 +819,7 @@ check_local_disk_temps() { check_drive() { [[ -z "$current_name" ]] || [[ -z "$current_temp" ]] && return - [[ "$current_temp" -eq 0 ]] && return # spun down + [[ "$current_temp" -eq 0 ]] && return # spun down — skip local warn_thresh crit_thresh if [[ "$current_rotational" == "0" ]]; then @@ -581,7 +841,6 @@ check_local_disk_temps() { while IFS= read -r ini_line; do if echo "$ini_line" | grep -qE '^\["(disk[0-9]+|parity[0-9]?|cache[0-9]?)"\]'; then - # Save previous drive before starting new one check_drive current_name=$(echo "$ini_line" | grep -o '"[^"]*"' | head -1 | tr -d '"') current_device="" @@ -615,17 +874,23 @@ check_local_disk_temps() { fi } -# ----------------------------------------------------------------------------------------------- -# check_remote_disks — verify all disks backing a share are healthy on the remote server -# Auto-detects filesystem type from disks.ini — handles XFS and ZFS correctly -# No SKIP_DISK_CHECK needed — detection is automatic +# ============================================================================================== +# ── REMOTE DISK VERIFICATION ────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Verifies all disks backing a share are healthy on the remote server. +# Auto-detects filesystem type from disks.ini — handles XFS, ZFS, and cache pools. +# No configuration required — discovers disk layout at runtime. # -# XFS array disks: checks mountpoint is active via mountpoint -q -# ZFS disks/pools: checks zpool status is ONLINE -# Shares spanning multiple disks: all must pass +# Three-tier detection handles all share locations: +# Tier 1: Array disk paths — /mnt/disk*/sharename (XFS or ZFS per-disk) +# Tier 2: ZFS standalone pools — finds nested paths up to 2 levels deep +# catches /mnt/cache/appdata-Failover/Critical-Data correctly +# Tier 3: shfs fallback — if share exists under /mnt/user, resolve backing pool +# or verify shfs itself is mounted if pool can't be determined # +# Fatal — exits the calling script if any disk is offline. # Usage: check_remote_disks "/mnt/user/Movies" -# ----------------------------------------------------------------------------------------------- + check_remote_disks() { local dir="$1" local share_name @@ -633,7 +898,7 @@ check_remote_disks() { info "$ICON_DISK Checking disks backing $share_name on $REMOTE_SERVER_NAME..." - # Get disk→fsType mapping from remote disks.ini + # Read remote disks.ini for fsType mapping local disks_ini_content disks_ini_content=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "cat /var/local/emhttp/disks.ini 2>/dev/null" 2>/dev/null) @@ -643,33 +908,27 @@ check_remote_disks() { exit 1 fi - # Find which disk(s) back this share on remote — check full path first - # Share may be nested (e.g. /mnt/user/appdata-Failover/Critical-Data lives at - # /mnt/cache/appdata-Failover/Critical-Data on the remote cache pool) + # Tier 1 — array disk paths (/mnt/disk*/sharename) local backing_disks backing_disks=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "ls -d /mnt/disk*/$share_name 2>/dev/null | awk -F/ '{print \$3}'" 2>/dev/null) - # Check ZFS standalone pools for the share — by basename AND nested path + # Tier 2 — ZFS standalone pools, including nested paths local zfs_pool_paths zfs_pool_paths=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "zpool list -H -o name 2>/dev/null | while read pool; do - # Check direct: /mnt/pool/sharename [[ -d \"/mnt/\${pool}/$share_name\" ]] && echo \"\$pool\" && continue - # Check nested: /mnt/pool/**/sharename (one level deep) find \"/mnt/\${pool}\" -maxdepth 2 -name '$share_name' -type d 2>/dev/null | \ grep -q . && echo \"\$pool\" done | sort -u" 2>/dev/null) - # Also check unRAID shfs — share exists under /mnt/user which is the union filesystem - # If the path exists under /mnt/user on remote it's accessible regardless of backing disk + # Tier 3 — shfs fallback via /mnt/user if [[ -z "$backing_disks" ]] && [[ -z "$zfs_pool_paths" ]]; then local on_user on_user=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "find /mnt/user -maxdepth 3 -name '$share_name' -type d 2>/dev/null | head -1" \ 2>/dev/null) if [[ -n "$on_user" ]]; then - # Share exists on remote shfs — find which pool backs it local actual_path actual_path=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "readlink -f '$on_user' 2>/dev/null || echo '$on_user'" 2>/dev/null) @@ -679,7 +938,7 @@ check_remote_disks() { zfs_pool_paths="$pool_name" info "$ICON_DISK $share_name found at $actual_path on $REMOTE_SERVER_NAME" else - # Accessible via shfs — verify shfs is mounted + # Can't determine pool — verify shfs itself is mounted local shfs_ok shfs_ok=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "mountpoint -q /mnt/user && echo yes || echo no" 2>/dev/null) @@ -757,48 +1016,55 @@ check_remote_disks() { success "All disks backing $share_name are online ✅" } -# ----------------------------------------------------------------------------------------------- -# CONTAINER MANAGEMENT — STOP (remote via SSH) -# Stops containers in CRITICAL_CONTAINER_NAMES on remote server. -# Tracks running containers in RUNNING_CONTAINERS for restart after rsync. -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ── CONTAINER MANAGEMENT ────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Four functions — stop/start on both local and remote. +# State tracking: only containers that WERE running get restarted. +# Was stopped → stays stopped. Was running → gets restarted. ✅ + +# Tracks which remote containers were running before stop — used by start_containers() RUNNING_CONTAINERS=() +# Stop containers on the REMOTE server via SSH. +# Reads CRITICAL_CONTAINER_NAMES — set from profile arrays by rsync.sh. +# Tracks running state in RUNNING_CONTAINERS for restart after sync. stop_containers() { if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \ [[ "${CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then - log "No remote containers configured for this profile, skipping stop." + log "No remote containers configured for this profile, skipping remote stop." return fi + echo "━━━ $ICON_STOP $ICON_CONTAINERS Containers ━━━" info "Stopping remote containers..." RUNNING_CONTAINERS=() for c in "${CRITICAL_CONTAINER_NAMES[@]}"; do [[ -z "$c" ]] && continue - STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + local status + status=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "docker inspect -f '{{.State.Running}}' $c 2>/dev/null" 2>/dev/null || echo "unknown") - if [[ "$STATUS" == "true" ]]; then - echo "$ICON_STOP Stopping $c..." + if [[ "$status" == "true" ]]; then + echo "$ICON_STOP Stopping remote $c..." RUNNING_CONTAINERS+=("$c") if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $c" >/dev/null; then echo "$ICON_STOPPED $c stopped" else - error "Failed to stop $c" + error "Failed to stop remote $c" fi - elif [[ "$STATUS" == "false" ]]; then - echo "$ICON_NOT_RUNNING $c is not running, skipping" + elif [[ "$status" == "false" ]]; then + echo "$ICON_NOT_RUNNING $c is not running — skipping" else log "$c not found on remote — skipping" fi done } -# ----------------------------------------------------------------------------------------------- -# CONTAINER MANAGEMENT — STOP LOCAL -# Stops containers on the LOCAL server before rsync pushes data out. -# Uses PROFILE_LOCAL_CRITICAL_CONTAINER_NAMES — same naming scheme as remote. -# If container not found on this server → skipped gracefully, not errored. -# Only containers that were running get tracked for restart. -# ----------------------------------------------------------------------------------------------- +# Tracks which local containers were running before stop — used by start_local_containers() +LOCAL_RUNNING_CONTAINERS=() + +# Stop containers on the LOCAL server. +# Reads LOCAL_CRITICAL_CONTAINER_NAMES — set from profile arrays by rsync.sh. +# Flushes local databases cleanly before pushing data to remote. stop_local_containers() { if [[ ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \ [[ "${LOCAL_CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then @@ -809,8 +1075,9 @@ stop_local_containers() { LOCAL_RUNNING_CONTAINERS=() for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]}"; do [[ -z "$c" ]] && continue - STATUS=$(docker inspect -f '{{.State.Running}}' "$c" 2>/dev/null || echo "unknown") - if [[ "$STATUS" == "true" ]]; then + local status + status=$(docker inspect -f '{{.State.Running}}' "$c" 2>/dev/null || echo "unknown") + if [[ "$status" == "true" ]]; then echo "$ICON_STOP Stopping local $c..." LOCAL_RUNNING_CONTAINERS+=("$c") if docker stop "$c" >/dev/null; then @@ -818,19 +1085,17 @@ stop_local_containers() { else error "Failed to stop local $c" fi - elif [[ "$STATUS" == "false" ]]; then - echo "$ICON_NOT_RUNNING $c is not running, skipping" + elif [[ "$status" == "false" ]]; then + echo "$ICON_NOT_RUNNING $c is not running — skipping" else log "$c not found locally — skipping" fi done } -# ----------------------------------------------------------------------------------------------- -# CONTAINER MANAGEMENT — START (remote via SSH) -# Restarts only containers tracked in RUNNING_CONTAINERS. -# Delayed containers receive CONTAINER_DELAY seconds before starting. -# ----------------------------------------------------------------------------------------------- +# Restart containers on the REMOTE server that were running before sync. +# Reads RUNNING_CONTAINERS set by stop_containers(). +# Respects DELAYED_CONTAINERS — waits CONTAINER_DELAY seconds before starting them. start_containers() { if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then log "No remote containers to restart." @@ -840,28 +1105,25 @@ start_containers() { for c in "${RUNNING_CONTAINERS[@]}"; do [[ -z "$c" ]] && continue local needs_delay=false - for d in "${DELAYED_CONTAINERS[@]}"; do + for d in "${DELAYED_CONTAINERS[@]:-}"; do [[ "$c" == "$d" ]] && needs_delay=true && break done if [[ "$needs_delay" == true ]]; then info "Waiting ${CONTAINER_DELAY}s before starting $c..." sleep "$CONTAINER_DELAY" fi - echo "$ICON_START Starting $c..." + echo "$ICON_START Starting remote $c..." if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $c" >/dev/null 2>&1; then echo "$ICON_STARTED $c started" else - error "Failed to start $c — start manually if needed" + error "Failed to start remote $c — start manually if needed" fi done } -# ----------------------------------------------------------------------------------------------- -# CONTAINER MANAGEMENT — START LOCAL -# Restarts only containers tracked in LOCAL_RUNNING_CONTAINERS. -# Respects DELAYED_CONTAINERS and CONTAINER_DELAY same as remote start. -# If container not found → skipped gracefully. -# ----------------------------------------------------------------------------------------------- +# Restart containers on the LOCAL server that were running before sync. +# Reads LOCAL_RUNNING_CONTAINERS set by stop_local_containers(). +# Respects DELAYED_CONTAINERS and CONTAINER_DELAY same as remote. start_local_containers() { if [[ ${#LOCAL_RUNNING_CONTAINERS[@]} -eq 0 ]]; then log "No local containers to restart." @@ -871,7 +1133,7 @@ start_local_containers() { for c in "${LOCAL_RUNNING_CONTAINERS[@]}"; do [[ -z "$c" ]] && continue local needs_delay=false - for d in "${DELAYED_CONTAINERS[@]}"; do + for d in "${DELAYED_CONTAINERS[@]:-}"; do [[ "$c" == "$d" ]] && needs_delay=true && break done if [[ "$needs_delay" == true ]]; then @@ -887,11 +1149,13 @@ start_local_containers() { done } -# ----------------------------------------------------------------------------------------------- -# RSYNC OPTIONS -# Loads rsync options for current profile. Falls back to DEFAULT_RSYNC_OPTS if no match. -# Profile opts do NOT inherit from defaults — list all desired flags explicitly. -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ── RSYNC OPTIONS ───────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Loads rsync options for the current profile. +# Falls back to DEFAULT_RSYNC_OPTS from master.conf if no profile match found. +# Profile opts do NOT inherit from defaults — list all desired flags explicitly in master.conf. + get_rsync_opts() { if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then read -r -a RSYNC_OPTS <<< "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]}" @@ -902,20 +1166,22 @@ get_rsync_opts() { fi } -# ----------------------------------------------------------------------------------------------- -# SCRIPT LOCKING — v3.0 +# ============================================================================================== +# ── SCRIPT LOCKING ──────────────────────────────────────────────────────────────────────────── +# ============================================================================================== # Prevents multiple instances of the same script running simultaneously. # All lock files live in /tmp/unraid_locks/ — auto-cleared on reboot. # -# Usage in scripts: -# acquire_lock — strict: exit immediately if already running -# acquire_lock "wait" — wait mode: wait briefly then exit if still locked -# acquire_rsync_lock "$profile" — per-profile + global concurrent limit +# Modes: +# strict — exit immediately if already running (default) +# wait — wait LOCK_WAIT_TIMEOUT seconds then exit if still locked +# continuous — for long-running scripts: skip gracefully if healthy instance running # -# Stale lock detection — if lock file exists but PID is dead, clears and proceeds. -# Lock age warning — if lock is older than expected, warns but does not override. -# EXIT trap registered automatically — lock always released on exit, crash, or kill. -# ----------------------------------------------------------------------------------------------- +# Features: +# Stale lock detection — if lock PID is dead, clears and proceeds +# PID reuse protection — lock stores PID:scriptname, validates both +# Age warning — warns if lock older than LOCK_WARN_AGE (skipped for continuous) +# EXIT trap — lock always released on exit, crash, or kill signal LOCK_DIR="/tmp/unraid_locks" @@ -924,10 +1190,11 @@ LOCK_DIR="/tmp/unraid_locks" if [[ -n "${DATA_DIR:-}" ]] && [[ ! -d "$DATA_DIR" ]]; then mkdir -p "$DATA_DIR" 2>/dev/null || true fi + RSYNC_COUNT_FILE="$LOCK_DIR/rsync_active_count" RSYNC_MAX_CONCURRENT=3 -LOCK_WARN_AGE=300 # seconds — warn if lock older than this (5min default) -LOCK_WAIT_TIMEOUT=30 # seconds — how long "wait" mode waits before giving up +LOCK_WARN_AGE=300 # seconds — warn if lock older than this (5min default) +LOCK_WAIT_TIMEOUT=30 # seconds — how long "wait" mode waits before giving up # Internal — script name used as lock identifier _lock_name() { @@ -939,21 +1206,14 @@ _lock_file() { echo "$LOCK_DIR/${1:-$(_lock_name)}.lock" } -# Internal — release lock on exit +# Internal — release lock on exit (registered via EXIT trap) _release_on_exit() { local lockfile="$1" [[ -f "$lockfile" ]] && rm -f "$lockfile" } -# ----------------------------------------------------------------------------------------------- -# acquire_lock — acquire exclusive lock for this script -# Mode: strict (default) — exit immediately if locked -# wait — wait LOCK_WAIT_TIMEOUT seconds then exit -# continuous — for long-running scripts: skip gracefully if healthy -# Lock file stores PID:scriptname — prevents PID reuse false positives -# Stale lock: if PID dead OR PID belongs to different process → clear and acquire -# Age warning: if lock older than LOCK_WARN_AGE → warn (skipped for continuous) -# ----------------------------------------------------------------------------------------------- +# Acquire exclusive lock for this script. +# Usage: acquire_lock [strict|wait|continuous] acquire_lock() { local mode="${1:-strict}" local script_name @@ -963,7 +1223,6 @@ acquire_lock() { mkdir -p "$LOCK_DIR" - # Check for existing lock if [[ -f "$lockfile" ]]; then local lock_content existing_pid locked_name lock_content=$(cat "$lockfile" 2>/dev/null) @@ -975,13 +1234,13 @@ acquire_lock() { warn "Stale lock detected for $script_name (PID $existing_pid gone) — clearing" rm -f "$lockfile" - # PID reuse — PID alive but belongs to a different process + # PID reuse — PID alive but belongs to different process elif [[ "$locked_name" != "$script_name" ]]; then - warn "Lock PID $existing_pid reused by different process ($locked_name ≠ $script_name) — clearing stale lock" + warn "Lock PID $existing_pid reused by different process ($locked_name ≠ $script_name) — clearing" rm -f "$lockfile" else - # Lock is genuinely active — check age (skip warning for continuous scripts) + # Lock is genuinely active — check age local lock_age lock_age=$(( $(date +%s) - $(stat -c %Y "$lockfile" 2>/dev/null || echo 0) )) if [[ "$lock_age" -gt "$LOCK_WARN_AGE" ]] && [[ "$mode" != "continuous" ]]; then @@ -989,7 +1248,6 @@ acquire_lock() { fi if [[ "$mode" == "continuous" ]]; then - # Continuous scripts — healthy instance = always skip gracefully log "$ICON_SKIP $script_name already running healthy (PID $existing_pid) — skipping" exit 0 elif [[ "$mode" == "wait" ]]; then @@ -1018,7 +1276,7 @@ acquire_lock() { else error "Another instance of $script_name is already running (PID $existing_pid) — exiting" case "$script_name" in - failover|transcode_management|media_management|daily_sync_maintenance|system_watchdog) + failover|transcode_management|daily_sync_maintenance|system_watchdog) notify "$script_name lock collision on $(hostname) — concurrent instance detected" "$script_name" "warning" ;; esac @@ -1029,17 +1287,13 @@ acquire_lock() { # Acquire lock — store PID:scriptname to prevent PID reuse false positives echo "$$:$script_name" > "$lockfile" - - # Register EXIT trap to always release lock trap "_release_on_exit '$lockfile'" EXIT - log "$ICON_LOCK Lock acquired: $script_name (PID $$)" } -# ----------------------------------------------------------------------------------------------- -# acquire_rsync_lock — per-profile lock + global concurrent limit -# Prevents same profile running twice and limits total concurrent rsync instances -# ----------------------------------------------------------------------------------------------- +# Acquire per-profile rsync lock + enforce global concurrent limit. +# Prevents same profile running twice and limits total concurrent rsync instances. +# Usage: acquire_rsync_lock "$PROFILE_NAME" acquire_rsync_lock() { local profile="$1" local profile_lock @@ -1053,7 +1307,8 @@ acquire_rsync_lock() { lock_content=$(cat "$profile_lock" 2>/dev/null) existing_pid="${lock_content%%:*}" locked_name="${lock_content##*:}" - if [[ -n "$existing_pid" ]] && kill -0 "$existing_pid" 2>/dev/null && [[ "$locked_name" == "rsync_${profile}" ]]; then + if [[ -n "$existing_pid" ]] && kill -0 "$existing_pid" 2>/dev/null && \ + [[ "$locked_name" == "rsync_${profile}" ]]; then error "rsync profile '$profile' is already running (PID $existing_pid) — exiting" exit 1 else @@ -1062,11 +1317,10 @@ acquire_rsync_lock() { fi fi - # Global concurrent limit + # Global concurrent limit — validate count against actual live locks local current_count=0 if [[ -f "$RSYNC_COUNT_FILE" ]]; then current_count=$(cat "$RSYNC_COUNT_FILE" 2>/dev/null || echo 0) - # Validate count — clean up if stale local actual_count=0 for lf in "$LOCK_DIR"/rsync_*.lock; do [[ -f "$lf" ]] || continue @@ -1083,25 +1337,22 @@ acquire_rsync_lock() { if [[ "$current_count" -ge "$RSYNC_MAX_CONCURRENT" ]]; then error "Maximum concurrent rsync limit ($RSYNC_MAX_CONCURRENT) reached — exiting" - info "Active rsync locks: $(ls "$LOCK_DIR"/rsync_*.lock 2>/dev/null | xargs -I{} basename {} .lock | tr '\n' ' ')" + info "Active rsync locks: $(ls "$LOCK_DIR"/rsync_*.lock 2>/dev/null | \ + xargs -I{} basename {} .lock | tr '\n' ' ')" exit 1 fi # Acquire profile lock and increment counter echo "$$:rsync_${profile}" > "$profile_lock" echo $(( current_count + 1 )) > "$RSYNC_COUNT_FILE" - - # Register EXIT trap trap "_release_rsync_on_exit '$profile_lock'" EXIT - log "$ICON_LOCK rsync lock acquired: profile '$profile' (PID $$, active: $(( current_count + 1 ))/$RSYNC_MAX_CONCURRENT)" } -# Internal — release rsync lock on exit +# Internal — release rsync lock and decrement global counter on exit _release_rsync_on_exit() { local profile_lock="$1" [[ -f "$profile_lock" ]] && rm -f "$profile_lock" - # Decrement global counter if [[ -f "$RSYNC_COUNT_FILE" ]]; then local count count=$(cat "$RSYNC_COUNT_FILE" 2>/dev/null || echo 1) @@ -1111,30 +1362,27 @@ _release_rsync_on_exit() { fi } -# ----------------------------------------------------------------------------------------------- -# check_api — pre-flight API reachability check -# Verifies API endpoint is reachable before attempting operations -# Usage: check_api "http://localhost:8989" "Sonarr" || exit 1 -# ----------------------------------------------------------------------------------------------- -# ----------------------------------------------------------------------------------------------- -# translate_path — translate container path to host path -# Uses ARR_PATH_MAP associative array — must be declared in calling script before use +# ============================================================================================== +# ── PATH TRANSLATION ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Translates container-internal file paths to host paths for arr cleanup scripts. +# Arr APIs return file paths as seen inside the container — scripts need host paths to scan. +# +# Uses ARR_PATH_MAP associative array — must be declared before calling. +# Longest-match wins — prevents partial path collisions. # # Usage: # declare -A ARR_PATH_MAP=(["/ext-music"]="/mnt/user/Music-New") # translate_path "/ext-music/Artist/Album/track.flac" # Returns: /mnt/user/Music-New/Artist/Album/track.flac # -# If no match found — returns path unchanged -# Handles longest-match first to avoid partial path collisions -# ARR_PATH_MAP must be exported or declared global before calling -# ----------------------------------------------------------------------------------------------- +# If no match found — returns path unchanged (safe fallback). + translate_path() { local api_path="$1" local best_match="" local best_len=0 - # Find longest matching container path prefix for container_path in "${!ARR_PATH_MAP[@]}"; do if [[ "$api_path" == "$container_path"* ]]; then if [[ "${#container_path}" -gt "$best_len" ]]; then @@ -1151,9 +1399,12 @@ translate_path() { fi } -# ----------------------------------------------------------------------------------------------- -# check_arr_version — verify arr major version matches tested version in Master.conf -# Exits the calling script if version doesn't match — prevents running against untested API +# ============================================================================================== +# ── ARR VERSION CHECK ───────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Verifies arr major version matches tested version in master.conf. +# Exits if version doesn't match — prevents running against untested API structure. +# Warns and proceeds if version endpoint is unreachable. # # Usage: # check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" "Radarr" @@ -1162,9 +1413,9 @@ translate_path() { # $1 = arr base URL # $2 = API key # $3 = API path prefix (v3 or v1) -# $4 = expected major version number +# $4 = expected major version number from master.conf # $5 = arr name for error messages -# ----------------------------------------------------------------------------------------------- + check_arr_version() { local url="$1" local api_key="$2" @@ -1202,12 +1453,88 @@ check_arr_version() { else error "$arr_name version mismatch — running v${major_version}, tested against v${expected_major}" error "The API endpoint structure may have changed — exiting to protect your library" - error "Update ${arr_name^^}_VERSION_MAJOR in Master.conf after verifying the script works with v${major_version}" + error "Update ${arr_name^^}_VERSION_MAJOR in master.conf after verifying the script works with v${major_version}" notify "$arr_name version mismatch on $(hostname) — running v${major_version}, script tested against v${expected_major}" "$arr_name Cleanup" "warning" exit 1 fi } +# ============================================================================================== +# ── EMBY LIBRARY SCAN ───────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Triggers Emby "Clean Missing Files" scheduled task after arr cleanup scripts delete files. +# Without this Emby keeps showing deleted files as ghost entries until its next scheduled scan. +# User clicks a ghost entry → file not found error. This prevents that entirely. +# +# Uses EMBY_URL and EMBY_API_KEY — aliased by detect_hosts() from HOST*_EMBY_* vars. +# Skips cleanly if Emby is not configured on this host (empty URL or API key). +# +# Emby task discovery: +# GET /ScheduledTasks → find task where Name contains "Clean Missing" +# POST /ScheduledTasks/Running/{taskId} → trigger it +# +# Non-fatal — logs warning if task cannot be triggered but does not exit. +# Clean Missing Files runs across all libraries at once — one call handles +# music, TV, and movies together regardless of which arr triggered the cleanup. +# +# Usage: notify_emby_scan (no args — always triggers Clean Missing Files) + +notify_emby_scan() { + # Skip if Emby not configured on this host + if [[ -z "${EMBY_URL:-}" ]] || [[ -z "${EMBY_API_KEY:-}" ]]; then + log "Emby not configured on $MY_ID — skipping library scan notification" + return 0 + fi + + log "Notifying Emby to clean missing files..." + + # Get all scheduled tasks + local tasks_response + tasks_response=$(curl -sf --max-time 15 -H "X-Api-Key: $EMBY_API_KEY" "${EMBY_URL}/ScheduledTasks" 2>/dev/null) + + if [[ -z "$tasks_response" ]]; then + warn "Could not reach Emby scheduled tasks API at $EMBY_URL — skipping scan" + return 0 + fi + + # Find the "Clean Missing Files" task ID + local task_id + task_id=$(echo "$tasks_response" | grep -o '"Id":"[^"]*","Name":"[^"]*Clean Missing[^"]*"' | grep -o '"Id":"[^"]*"' | sed 's/"Id":"//;s/"//' | head -1) + + # Fallback — try "Scan Media Library" if Clean Missing not found + if [[ -z "$task_id" ]]; then + task_id=$(echo "$tasks_response" | grep -o '"Id":"[^"]*","Name":"[^"]*Scan Media Library[^"]*"' | grep -o '"Id":"[^"]*"' | sed 's/"Id":"//;s/"//' | head -1) + [[ -n "$task_id" ]] && log "Clean Missing Files not found — using Scan Media Library" + fi + + if [[ -z "$task_id" ]]; then + warn "Could not find Emby Clean Missing Files or Scan Media Library task" + warn "Check Emby scheduled tasks — ghost entries will persist until next Emby scan" + return 0 + fi + + log "Found Emby task ID: $task_id" + + # Trigger the task + local http_code + http_code=$(curl -sf --max-time 15 -X POST -H "X-Api-Key: $EMBY_API_KEY" -w "%{http_code}" -o /dev/null "${EMBY_URL}/ScheduledTasks/Running/${task_id}" 2>/dev/null) + + if [[ "$http_code" == "204" ]] || [[ "$http_code" == "200" ]]; then + warn "$ICON_EMBY Emby Clean Missing Files triggered — ghost entries will be removed" + else + warn "Emby task trigger returned HTTP $http_code — ghost entries may persist" + warn "Check Emby dashboard — manually run Clean Missing Files if needed" + fi +} + +# ============================================================================================== +# ── API REACHABILITY CHECK ──────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Pre-flight check — verifies an API endpoint is reachable before attempting operations. +# Non-fatal — returns 0 if reachable, 1 if not. Caller decides whether to exit. +# +# Usage: check_api "http://localhost:8989" "Sonarr" || exit 1 + check_api() { local url="$1" local service="${2:-API}" @@ -1222,20 +1549,191 @@ check_api() { fi } -# ----------------------------------------------------------------------------------------------- -# STATUS DISPLAY -# Prints current runtime configuration — triggered by --status flag. -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ── UNRAID VERSION PARITY CHECK ─────────────────────────────────────────────────────────────── +# ============================================================================================== +# Verifies local and remote servers are running compatible unRAID versions before any +# remote operation. Version mismatches can mean changed APIs, commands, or behaviours +# that silently break remote container operations, rsync, or failover logic. +# +# Reads /etc/unraid-version on both sides — format: version="7.2.3" +# +# Mismatch behaviour (UNRAID_VERSION_MISMATCH_ACTION in master.conf): +# "warn" — log warning and continue (default for minor/patch differences) +# "abort" — exit the calling script (default for major version differences) +# Major version mismatch always aborts regardless of setting +# +# Usage: check_unraid_version_parity || exit 1 + +check_unraid_version_parity() { + local local_version remote_version + + # Read local version + if [[ ! -f /etc/unraid-version ]]; then + warn "Cannot read local /etc/unraid-version — skipping version parity check" + return 0 + fi + local_version=$(grep -oP '(?<=version=")[^"]+' /etc/unraid-version 2>/dev/null) + if [[ -z "$local_version" ]]; then + warn "Cannot parse local unRAID version — skipping parity check" + return 0 + fi + + # Read remote version via SSH + remote_version=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" "grep -oP '(?<=version=")[^"]+' /etc/unraid-version 2>/dev/null" 2>/dev/null) + if [[ -z "$remote_version" ]]; then + warn "Cannot read remote unRAID version from $REMOTE_SERVER_NAME — skipping parity check" + return 0 + fi + + log "Version parity: local=$local_version remote=$remote_version" + + if [[ "$local_version" == "$remote_version" ]]; then + log "unRAID versions match: $local_version ✅" + return 0 + fi + + # Parse major versions + local local_major remote_major + local_major="${local_version%%.*}" + remote_major="${remote_version%%.*}" + + if [[ "$local_major" != "$remote_major" ]]; then + error "unRAID MAJOR version mismatch — local: $local_version remote: $remote_version" + error "Major version differences may break remote APIs, commands, and behaviours" + error "Update both servers to the same major version before running remote operations" + notify "unRAID major version mismatch on $(hostname) — local: $local_version remote: $remote_version — remote operations aborted" "Version Parity" "warning" + return 1 + fi + + # Minor/patch mismatch — action depends on config + local action="${UNRAID_VERSION_MISMATCH_ACTION:-warn}" + warn "unRAID version mismatch — local: $local_version remote: $remote_version" + + if [[ "$action" == "abort" ]]; then + error "UNRAID_VERSION_MISMATCH_ACTION=abort — refusing to continue" + notify "unRAID version mismatch on $(hostname) — local: $local_version remote: $remote_version — aborted" "Version Parity" "warning" + return 1 + fi + + warn "UNRAID_VERSION_MISMATCH_ACTION=warn — continuing despite mismatch (monitor for issues)" + return 0 +} + +# ============================================================================================== +# ── REMOTE DOCKER DAEMON CHECK ──────────────────────────────────────────────────────────────── +# ============================================================================================== +# Verifies the remote Docker daemon is responsive before issuing any remote container commands. +# A hung remote daemon causes docker commands to hang or silently fail — checking first +# prevents cascading failures and gives clear diagnostic output. +# +# Strike system: +# Each consecutive failed check adds a strike to REMOTE_DOCKER_STRIKES +# Below REMOTE_DOCKER_STRIKE_LIMIT → warn and return 1 (caller skips operation) +# At limit → notify critical and return 1 (caller should exit or escalate) +# Strikes reset when daemon recovers +# +# Usage: check_remote_docker_daemon || return 1 + +REMOTE_DOCKER_STRIKES=0 + +check_remote_docker_daemon() { + local timeout="${1:-10}" + + if ssh -i "$SSH_KEY" -o ConnectTimeout="$timeout" root@"$REMOTE_SERVER" "timeout $timeout docker info" >/dev/null 2>&1; then + # Daemon healthy — clear strikes if we had issues + if [[ "$REMOTE_DOCKER_STRIKES" -gt 0 ]]; then + info "Remote Docker daemon on $REMOTE_SERVER_NAME recovered — clearing strikes" + REMOTE_DOCKER_STRIKES=0 + fi + log "Remote Docker daemon healthy on $REMOTE_SERVER_NAME" + return 0 + fi + + REMOTE_DOCKER_STRIKES=$(( REMOTE_DOCKER_STRIKES + 1 )) + local limit="${REMOTE_DOCKER_STRIKE_LIMIT:-3}" + warn "Remote Docker daemon not responding on $REMOTE_SERVER_NAME (strike $REMOTE_DOCKER_STRIKES/$limit)" + + if [[ "$REMOTE_DOCKER_STRIKES" -ge "$limit" ]]; then + error "Remote Docker daemon unresponsive on $REMOTE_SERVER_NAME after $limit consecutive checks" + error "Skipping all remote container operations — check Docker on $REMOTE_SERVER_NAME" + notify "Remote Docker daemon unresponsive on $REMOTE_SERVER_NAME — remote operations skipped on $(hostname)" "Remote Docker Daemon" "warning" + fi + + return 1 +} + +# ============================================================================================== +# ── UNRAID COMMAND VALIDATION ───────────────────────────────────────────────────────────────── +# ============================================================================================== +# Verifies a unRAID-specific command exists and produces expected output before use. +# Protects against commands moving, changing format, or disappearing after upgrades. +# +# If validation fails — notifies and returns 1. Caller exits its own section only. +# Other scripts that pass validation are unaffected. +# +# Usage: +# validate_unraid_cmd # "/usr/local/emhttp/plugins/dynamix/scripts/notify" # "--help" # "Usage" # "unRAID notify script" +# +# Arguments: +# $1 = full path to command +# $2 = test argument to pass (use "" for no argument) +# $3 = expected string in output (use "" to skip output check) +# $4 = human readable name for error messages + +validate_unraid_cmd() { + local cmd_path="$1" + local test_arg="$2" + local expected_output="$3" + local cmd_name="${4:-$1}" + + # Check command exists and is executable + if [[ ! -x "$cmd_path" ]]; then + error "unRAID command not found or not executable: $cmd_path" + error "$cmd_name may have moved or been removed — check after recent unRAID upgrade" + notify "$cmd_name not found on $(hostname) at $cmd_path — check after unRAID upgrade" "Command Validation" "warning" + return 1 + fi + + # Check output matches expected pattern if provided + if [[ -n "$expected_output" ]]; then + local actual_output + actual_output=$(timeout 10 "$cmd_path" $test_arg 2>&1 || true) + if ! echo "$actual_output" | grep -q "$expected_output"; then + error "$cmd_name output format changed — expected '$expected_output' not found" + error "Command may have changed after unRAID upgrade — review $cmd_path" + notify "$cmd_name output format changed on $(hostname) — may need script update after unRAID upgrade" "Command Validation" "warning" + return 1 + fi + fi + + log "$cmd_name validated: $cmd_path ✅" + return 0 +} + +# ============================================================================================== +# ── STATUS DISPLAY ──────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Prints current runtime configuration — triggered by --status flag in any script. +# Useful for verifying detect_hosts() resolved correctly and profile loaded as expected. + show_status() { + local local_ver + local_ver=$(grep -oP '(?<=version=")[^"]+' /etc/unraid-version 2>/dev/null || echo "unknown") echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "Local: $LOCAL_SERVER_NAME" - echo "Remote: $REMOTE_SERVER_NAME" - echo "IP: $REMOTE_SERVER" - echo "Profile: ${PROFILE_NAME:-n/a}" - echo "DryRun: $DRY_RUN" - echo "Logging: $ENABLE_LOGGING" - echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]:-n/a}" - echo "Delayed: ${DELAYED_CONTAINERS[*]:-n/a}" - echo "Excludes: ${EXCLUDE_DIRS[*]:-n/a}" + echo "Local: $LOCAL_SERVER_NAME" + echo "Remote: $REMOTE_SERVER_NAME" + echo "IP: ${REMOTE_SERVER:-not resolved}" + echo "My ID: ${MY_ID:-not set}" + echo "Remote ID: ${REMOTE_ID:-not set}" + echo "unRAID ver: $local_ver" + echo "Profile: ${PROFILE_NAME:-n/a}" + echo "Silent mode: ${SILENT_MODE:-true}" + echo "DryRun: ${DRY_RUN:-false}" + echo "Logging: ${ENABLE_LOGGING:-false}" + echo "SSH Key: ${SSH_KEY:-not set}" + echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]:-n/a}" + echo "Delayed: ${DELAYED_CONTAINERS[*]:-n/a}" + echo "Excludes: ${EXCLUDE_DIRS[*]:-n/a}" echo "━━━━━━━━━━━━━━━━━━━━━━━" } \ No newline at end of file diff --git a/git_pull_execute.sh b/git_pull_execute.sh index 74fb9da..943f42d 100644 --- a/git_pull_execute.sh +++ b/git_pull_execute.sh @@ -1,30 +1,60 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Git Pull & Execute Script ----------------------------------- -# ----------------------------------------------------------------------------------------------- -# Pulls latest scripts from Gitea repo via SSH and sets executable permissions. -# Lives at the repo root — sources Master.conf and common.sh from the same directory. -# Uses GITEA_SSH_KEY, SSH_PORT, GITEA_CONTAINER and GITEA_REPO_PATH from Master.conf. +# ============================================================================================== +# ================================= Git Pull & Execute ========================================= +# ============================================================================================== +# Pulls the latest scripts from the Gitea repository via SSH. +# Lives at the repo root — sources load_config.sh from the same directory. # -# Detects where the Gitea container is running at runtime: -# Gitea running locally → connects via local IP -# Gitea running remotely → connects via remote server's Tailscale IP -# Works correctly through failover — no hardcoded assumptions about which server hosts Gitea. +# ── WHAT THIS SCRIPT DOES ───────────────────────────────────────────────────────────────────── +# 1. Detects which server it's running on via detect_hosts() (MY_ID) +# 2. Configures sparse checkout to exclude other servers' credential files +# Each server only pulls its own master_host*.conf — never sees peer credentials +# 3. Pulls or clones latest scripts from Gitea +# 4. Sets executable permissions on all .sh files # -# Supports --dry-run, --log, --status flags via common.sh parse_args. -# ----------------------------------------------------------------------------------------------- +# ── SPARSE CHECKOUT ─────────────────────────────────────────────────────────────────────────── +# Sparse checkout ensures each server only receives its own host conf: +# HOST1 pulls: master.conf + master_host1.conf + all scripts +# HOST1 skips: master_host2.conf, master_host3.conf etc. +# HOST2 pulls: master.conf + master_host2.conf + all scripts +# HOST2 skips: master_host1.conf, master_host3.conf etc. +# +# Adding a new server: +# Create master_host3.conf in the repo +# All existing servers automatically exclude it on next pull +# New server gets only its own conf ✅ +# +# ── GITEA LOCATION DETECTION ────────────────────────────────────────────────────────────────── +# Detects where Gitea is running at runtime — works through failover: +# Gitea local → connects via local IP +# Gitea remote → connects via Tailscale IP +# Both fail → falls back to GITEA_DOMAIN if configured +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# GITEA_CONTAINER — Docker container name for Gitea +# GITEA_REPO_PATH — repo path on Gitea (e.g. FailedProxy/Unraid_Scripts.git) +# GITEA_DOMAIN — public domain fallback (optional) +# TARGET_DIR — local path to clone/pull into +# GITEA_SSH_KEY — SSH key for Gitea authentication +# SSH_PORT — Gitea SSH port (often 221 or 222) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# git_pull_execute.sh — normal pull +# git_pull_execute.sh --dry-run — preview without making changes +# git_pull_execute.sh --log — verbose output +# git_pull_execute.sh --status — show config and exit +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# Root level script — sources from same directory, not parent -source "$SCRIPT_DIR/Master.conf" -source "$SCRIPT_DIR/common.sh" +# Root level script — load_config.sh is in the same directory +source "$SCRIPT_DIR/load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" @@ -32,27 +62,30 @@ if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi - success "Running as root" acquire_lock -# Detect which server we're on and where Gitea is running +# detect_hosts() sets MY_ID — needed for sparse checkout configuration detect_hosts -# Check if Gitea container is running locally +# ============================================================================================== +# ━━━ Locate Gitea ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_CONTAINERS Locate Gitea ━━━" + if docker ps --format "{{.Names}}" 2>/dev/null | grep -q "^${GITEA_CONTAINER}$"; then # Gitea is running on this server — use local IP GITEA_IP=$(hostname -I | awk '{print $1}') - info "$ICON_CONTAINERS Gitea running locally — connecting via $GITEA_IP" + log "Gitea running locally — connecting via $GITEA_IP" else - # Gitea is not running locally — find it on the remote server via Tailscale - info "$ICON_CONTAINERS Gitea not running locally — checking remote server" + # Gitea not running locally — find it on the remote server via Tailscale + log "Gitea not running locally — checking remote server" GITEA_IP=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) if [[ -n "$GITEA_IP" ]]; then info "$ICON_NET Gitea on $REMOTE_SERVER_NAME — connecting via Tailscale $GITEA_IP" - elif [[ -n "$GITEA_DOMAIN" ]]; then - # Tailscale failed — fall back to public domain + elif [[ -n "${GITEA_DOMAIN:-}" ]]; then warn "Tailscale resolution failed — falling back to $GITEA_DOMAIN" GITEA_IP="$GITEA_DOMAIN" else @@ -69,27 +102,85 @@ require_var TARGET_DIR require_var GITEA_SSH_KEY require_var SSH_PORT -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_NET Repo: $REPO_SSH" - echo "$ICON_GEAR Target: $TARGET_DIR" - echo "$ICON_GEAR SSH Key: $GITEA_SSH_KEY" - echo "$ICON_GEAR SSH Port: $SSH_PORT" - echo "$ICON_NOTIFY Notifications: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_NET Repo: $REPO_SSH" + echo "$ICON_GEAR Target: $TARGET_DIR" + echo "$ICON_GEAR SSH Key: $GITEA_SSH_KEY" + echo "$ICON_GEAR SSH Port: $SSH_PORT" + echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_HOST Remote ID: $REMOTE_ID ($REMOTE_SERVER_NAME)" + echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)" + echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SYNC Git Sync ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Sparse Checkout Configuration ━━━ +# ============================================================================================== +# Build the list of master_host*.conf files that belong to OTHER servers. +# This server pulls everything EXCEPT those files. +# MY_ID is set by detect_hosts() — e.g. "HOST1" + +configure_sparse_checkout() { + local repo_dir="$1" + + log "Configuring sparse checkout for $MY_ID..." + + # Enable sparse checkout + git -C "$repo_dir" config core.sparseCheckout true 2>/dev/null + + # Build exclusion list — all master_host*.conf files except MY_ID's + local sparse_file="$repo_dir/.git/info/sparse-checkout" + mkdir -p "$(dirname "$sparse_file")" + + # Start with: pull everything + echo "/*" > "$sparse_file" + + # Exclude each other server's conf file + # Find all master_host*.conf files present in the repo + local excluded=0 + for conf_file in "$repo_dir"/master_host*.conf; do + [[ -f "$conf_file" ]] || continue + local conf_name + conf_name=$(basename "$conf_file") + + # Determine which HOST ID owns this conf by grepping its hostname var + # Pattern: HOST1="unRAID-..." or HOST2="unRAID-..." + local conf_host_id + conf_host_id=$(grep -m1 -oP '^\s+HOST[0-9]+(?==)' "$conf_file" 2>/dev/null | tr -d ' ') + + if [[ -z "$conf_host_id" ]]; then + log "Cannot determine HOST ID for $conf_name — including in pull (safe default)" + continue + fi + + if [[ "$conf_host_id" != "$MY_ID" ]]; then + echo "!$conf_name" >> "$sparse_file" + log "Sparse checkout: excluding $conf_name (belongs to $conf_host_id)" + ((excluded++)) + else + log "Sparse checkout: including $conf_name (belongs to $MY_ID — this server)" + fi + done + + if [[ "$excluded" -gt 0 ]]; then + info "$ICON_LOCK Sparse checkout: excluding $excluded peer conf file(s) — credentials protected" + else + log "Sparse checkout: no peer conf files to exclude (single server or first run)" + fi +} + +# ============================================================================================== +# ━━━ Git Sync ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_SYNC Git Sync ━━━" echo "$ICON_NET Repo: $REPO_SSH" @@ -101,13 +192,21 @@ SYNC_SUCCESS=false if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would sync $REPO_SSH → $TARGET_DIR" + warn "DRY RUN — would configure sparse checkout for $MY_ID" + warn "DRY RUN — would exclude peer master_host*.conf files" SYNC_SUCCESS=true else mkdir -p "$TARGET_DIR" cd "$TARGET_DIR" || { error "Cannot cd into $TARGET_DIR"; exit 1; } if [[ -d ".git" ]]; then + # ── Existing repository ────────────────────────────────────────────── info "$ICON_SYNC Existing repository detected — updating" + + # Configure sparse checkout BEFORE pull + # Uses conf files already present from last pull to determine exclusions + configure_sparse_checkout "$TARGET_DIR" + log "git reset --hard" git reset --hard @@ -125,9 +224,22 @@ else fi else + # ── Fresh clone ────────────────────────────────────────────────────── info "$ICON_SYNC No repository found — cloning" + + # Clone first — need the repo to exist before configuring sparse checkout if GIT_SSH_COMMAND="ssh -i $GITEA_SSH_KEY -p $SSH_PORT" git clone "$REPO_SSH" .; then success "Clone successful" + + # Configure sparse checkout after clone + # Now all master_host*.conf files are present — can detect exclusions + configure_sparse_checkout "$TARGET_DIR" + + # Apply sparse checkout — removes excluded files from working tree + info "Applying sparse checkout..." + git read-tree -mu HEAD + success "Sparse checkout applied — peer credentials removed from working tree" + SYNC_SUCCESS=true else error "Clone failed" @@ -136,25 +248,25 @@ else fi fi - # ----------------------------------------------------------------------------------------------- - # ━━━ $ICON_GEAR Permissions ━━━ - # ----------------------------------------------------------------------------------------------- + # ── Permissions ────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_GEAR Permissions ━━━" - info "Setting executable permissions on all .sh files..." + log "Setting executable permissions on all .sh files..." find "$TARGET_DIR" -type f -name "*.sh" -exec chmod +x {} \; - success "Permissions applied" + log "Permissions applied" fi END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY GIT SYNC SUMMARY ━━━━━" echo "$ICON_NET Repo: $REPO_SSH" echo "$ICON_GEAR Target: $TARGET_DIR" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_LOCK Excluded: peer master_host*.conf files" echo "$ICON_TIME Duration: $(format_duration $((END - START)))" if [[ "$DRY_RUN" == true ]]; then echo "$ICON_WARN Status: DRY RUN — no changes made" diff --git a/load_config.sh b/load_config.sh new file mode 100644 index 0000000..ad8d57f --- /dev/null +++ b/load_config.sh @@ -0,0 +1,97 @@ +#!/bin/bash +# ============================================================================================== +# ================================= CONFIGURATION LOADER ======================================= +# ============================================================================================== +# Single entry point for all configuration sourcing across the ecosystem. +# Every script sources this file instead of sourcing master.conf files directly. +# +# ── HOW IT WORKS ────────────────────────────────────────────────────────────────────────────── +# 1. Sources master.conf (shared config — thresholds, toggles, profiles, job lists) +# 2. Auto-discovers and sources all master_host*.conf files in the same directory +# Each host conf extends the shared profile arrays and adds host-specific identity +# 3. Sources common.sh (shared functions — detect_hosts, logging, notifications etc.) +# +# ── WHY THIS EXISTS ─────────────────────────────────────────────────────────────────────────── +# Without this loader every script had to explicitly source each conf file: +# source master.conf +# source master_host1.conf +# source master_host2.conf +# source common.sh +# +# Adding a new server meant updating every script. +# With this loader — add master_host3.conf to the git repo and every server +# auto-discovers it on next git pull. Zero script changes required. Ever. +# +# ── ADDING A NEW SERVER ─────────────────────────────────────────────────────────────────────── +# 1. Create master_host3.conf following the same structure as HOST1/HOST2 +# 2. Commit and push to git repo +# 3. All servers pull it automatically — no other changes needed +# +# ── USAGE IN SCRIPTS ────────────────────────────────────────────────────────────────────────── +# Replace the three source lines at the top of every script with: +# +# SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# source "$SCRIPT_DIR/../load_config.sh" +# +# Scripts in subdirectories (Rsync/, Docker_Essentials/ etc.) use ../ to reach root. +# Scripts in root directory use ./ instead: +# +# SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# source "$SCRIPT_DIR/load_config.sh" +# +# ── SPARSE CHECKOUT NOTE ────────────────────────────────────────────────────────────────────── +# Sparse checkout controls which master_host*.conf files each server receives. +# HOST1 only pulls master_host1.conf — never HOST2's credentials. +# HOST2 only pulls master_host2.conf — never HOST1's credentials. +# This loader sources whatever conf files ARE present — sparse checkout handles the rest. +# Both servers pull all non-credential conf files (master.conf, common.sh, load_config.sh). +# +# ============================================================================================== + +# ━━━ Locate config root ━━━ +# load_config.sh always lives in the repo root. +# Scripts call it from subdirectories using ../ — resolve to the actual root. + LOAD_CONFIG_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# ━━━ Source shared config ━━━ +# master.conf must be sourced first — it declares the shared PROFILE_* arrays +# that Host confs extend. Sourcing host confs before master.conf would fail. + if [[ ! -f "$LOAD_CONFIG_DIR/master.conf" ]]; then + echo "[FATAL] master.conf not found at $LOAD_CONFIG_DIR/master.conf" >&2 + echo "[FATAL] Check TARGET_DIR and git pull status" >&2 + exit 1 + fi + source "$LOAD_CONFIG_DIR/master.conf" + +# ━━━ Auto-discover and source all master_host*.conf files ━━━ +# Sorted for consistent load order — HOST1 before HOST2 before HOST3 etc. +# Each host conf extends the shared PROFILE_* arrays and adds host-specific vars. +# Missing files are silently skipped — sparse checkout intentionally withholds some. +# At least one host conf must be present or the ecosystem has no identity to work with. + _host_confs_loaded=0 + + for _conf in $(ls "$LOAD_CONFIG_DIR"/master_host*.conf 2>/dev/null | sort); do + if [[ -f "$_conf" ]]; then + source "$_conf" + (( _host_confs_loaded++ )) + [[ "${ENABLE_LOGGING:-false}" == "true" ]] && \ + echo "[LOG] Loaded host config: $(basename "$_conf")" >&2 + fi + done + + if [[ "$_host_confs_loaded" -eq 0 ]]; then + echo "[FATAL] No master_host*.conf files found in $LOAD_CONFIG_DIR" >&2 + echo "[FATAL] At least one host conf required — check git pull and sparse checkout" >&2 + exit 1 + fi + +# ━━━ Source shared functions ━━━ +# common.sh sourced last — it calls detect_hosts() which needs HOST* vars to be set. + if [[ ! -f "$LOAD_CONFIG_DIR/common.sh" ]]; then + echo "[FATAL] common.sh not found at $LOAD_CONFIG_DIR/common.sh" >&2 + exit 1 + fi + source "$LOAD_CONFIG_DIR/common.sh" + +# ━━━ Cleanup ━━━ + unset _conf _host_confs_loaded LOAD_CONFIG_DIR \ No newline at end of file diff --git a/Master.conf b/master.conf similarity index 62% rename from Master.conf rename to master.conf index 2198506..5a8c601 100644 --- a/Master.conf +++ b/master.conf @@ -2,11 +2,24 @@ # ============================================================================================== # ================================= MASTER CONFIGURATION ======================================= # ============================================================================================== -# All user-facing variables for the unRAID script ecosystem. -# Scripts source this file — edit here, changes apply everywhere on next git pull. +# Shared configuration for the unRAID script ecosystem. +# Contains all settings that apply to every server — thresholds, toggles, profiles, job lists. +# +# ── HOW THE THREE-FILE SYSTEM WORKS ────────────────────────────────────────────────────────── +# Scripts source all three files at startup: +# source master.conf ← shared config (this file) +# source master_host1.conf ← HOST1 credentials, shares, container lists +# source master_host2.conf ← HOST2 credentials, shares, container lists +# +# Sparse checkout (git) ensures each server only pulls its own master_host*.conf. +# HOST2 never sees HOST1 credentials. HOST1 never sees HOST2 credentials. +# +# What belongs here: thresholds, toggles, intervals, profiles, job lists +# What belongs in HOST*: hostnames, SSH keys, API keys, passwords, share paths, +# container names, failover lists, watchdog containers # # ── HOW THIS FILE WORKS ─────────────────────────────────────────────────────────────────────── -# Every script sources Master.conf and common.sh at startup. +# Every script sources all three conf files and common.sh at startup. # Change a value here and it affects all scripts that use it — no hunting through files. # To disable something: comment it out with # rather than deleting it. # To add a new rsync profile: add a key to each PROFILE_* array in the RSYNC section. @@ -16,7 +29,7 @@ # # Section Description # ─────────────────────────────────────────────────────────────────────────────────────────── -# HOST CONFIGURATION Server hostnames, SSH keys, Emby connection details, DATA_DIR +# SHARED HOST CONFIGURATION DATA_DIR and shared connection settings # PARTNERSHIP Mirror relationship lifecycle — onboard/offboard/transfer # LOGGING Enable or disable verbose logging # NOTIFICATIONS unRAID native and Discord webhook settings @@ -35,20 +48,15 @@ # RSYNC PROFILE SYSTEM Per-profile overrides for appdata syncs # # ── FAILOVER ─────────────────────────────────────────────────────────────────────────────── -# FAILOVER Mutual container failover between two servers +# FAILOVER Mutual container failover shared settings # FAILOVER TEST Simulated outage settings for failover_test.sh -# DDNS Script-controlled DDNS — absolute rules -# INTERNET LOSS Containers to stop when internet is lost -# TIERED CONTAINER LISTS What each server runs for the other per tier -# TIER DELAY SETTINGS How long before each tier activates (minutes) -# RSYNC WRITEBACK JOBS Appdata synced back to primary on handback # # ── DOCKER ESSENTIALS ────────────────────────────────────────────────────────────────────── -# DOWNLOADERS RESET Daily reset for slskd, SABnzbd, qBittorrent +# DOWNLOADERS RESET Retention and thresholds for slskd, SABnzbd, qBittorrent # DOCKER DAILY RESTART Containers restarted daily # DOCKER WEEKLY RESTART Containers restarted weekly -# DOCKER WATCHDOG Continuous two-tier self-healing container monitoring -# DOCKER NETWORK CONNECT Ensure networks exist + connect containers on array start +# DOCKER WATCHDOG Thresholds and toggles for container monitoring +# DOCKER NETWORK CONNECT Networks to ensure + containers to connect # # ── UNRAID ESSENTIALS ────────────────────────────────────────────────────────────────────── # INOTIFY TUNING inotify limits — raised at array start by inotify_tuning.sh @@ -63,8 +71,8 @@ # ── MEDIA ────────────────────────────────────────────────────────────────────────────────── # MEDIA PERMISSIONS Share list, mode and owner for permissions script # MEDIA CLEANER Anime and media folder lists and file patterns -# ARR CLEANUP Lidarr, Sonarr, Radarr orphan file cleanup -# ARR FAILED/STALLED RECOVERY Auto blocklist + re-search failed imports and stalled downloads +# ARR CLEANUP Lidarr, Sonarr, Radarr orphan file cleanup settings +# ARR FAILED/STALLED RECOVERY Auto blocklist + re-search settings # # ── TRANSCODES ───────────────────────────────────────────────────────────────────────────── # TRANSCODE MANAGER Ramdisk and SSD fallback transcode management @@ -85,38 +93,28 @@ # ============================================================================================== # ============================================================================================== -# ── HOST CONFIGURATION ──────────────────────────────────────────────────────────────────────── +# ── SHARED HOST CONFIGURATION ───────────────────────────────────────────────────────────────── # ============================================================================================== - -# ━━━ Hosts ━━━ -# Hostnames must match exact Docker/unRAID hostnames AND Tailscale device names — case sensitive. -# Used by detect_hosts() in common.sh to determine which server is local and which is remote. -# Both servers run identical scripts — host detection makes them bidirectional. - HOST1="unRAID-Gmer4Lfe" - HOST2="unRAID-Jayred365" - -# Data directory — persistent script state and statistics files. +# Hostnames defined in master_host*.conf — detect_hosts() reads them at runtime. +# DATA_DIR is the same path on all servers — persistent script state and statistics. # Array share — survives reboots, no flash drive wear. # Created automatically if it doesn't exist. # Only truly critical files (failover state, watchdog reboot log) stay on /boot/config. DATA_DIR="/mnt/user/appdata/unraid_scripts/data" -# SSH keys for server-to-server rsync and failover container operations. -# Both keys must be in /root/.ssh/ and authorised in the remote server's authorized_keys. - HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" - HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" +# ── Version Parity ── +# Controls behaviour when local and remote unRAID versions differ. +# Major version mismatch always aborts regardless of this setting. +# "warn" — log warning and continue (safe for minor/patch differences) +# "abort" — refuse to continue (strict — ensures both sides always match) + UNRAID_VERSION_MISMATCH_ACTION="warn" -# ━━━ Emby ━━━ -# Defined once here — referenced by transcode_manager.sh, emby_session_report.sh, -# emby_database_repair.sh, weekly_sync_maintenance.sh, and TRANSCODE_SERVERS array. -# API key: Emby Dashboard → API Keys → + New Key - HOST1_EMBY_CONTAINER="Emby" - HOST1_EMBY_URL="http://localhost:8096" - HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829" - - HOST2_EMBY_CONTAINER="Emby-Jayred365" - HOST2_EMBY_URL="http://localhost:8096" - HOST2_EMBY_API_KEY="your-host2-emby-api-key" +# ── Remote Docker Daemon ── +# Strike system for remote Docker daemon health checks. +# Scripts that issue remote container commands check the remote daemon first. +# Below limit → skip operation this run | At limit → notify critical + exit + REMOTE_DOCKER_STRIKE_LIMIT=3 # consecutive failures before critical notify + REMOTE_DOCKER_RETRY_WAIT=30 # seconds to wait before retrying after failure # ============================================================================================== # ── PARTNERSHIP ─────────────────────────────────────────────────────────────────────────────── @@ -124,7 +122,7 @@ # Manages the relationship lifecycle between two unRAID servers. # HOST1 is always the owner (source of truth) — HOST2 is always the mirror. # PARTNERSHIP_OWNER_HOST flips to "HOST2" after a --transfer operation. -# All vars derive from HOST1/HOST2 — no duplicate IP or key configuration needed. +# All identity vars (hostnames, SSH keys) live in master_host*.conf. # Hostnames already match Tailscale device names — IP resolution is automatic. # # State files on /boot/config — survives reboots, available before array starts: @@ -143,45 +141,39 @@ PARTNERSHIP_ENABLED=false PARTNERSHIP_OWNER_HOST="HOST1" # "HOST1" or "HOST2" — flips on --transfer -# Auth containers reconfigured on onboard/offboard +# Auth containers reconfigured on onboard/offboard — defined per host in master_host*.conf. # Format: "ContainerName|WebUIPort" -# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's) +# HOST1_PARTNERSHIP_AUTH_WEBUIS / HOST2_PARTNERSHIP_AUTH_WEBUIS +# On onboard → WebUI pointed at owner's Tailscale IP # On offboard → WebUI pointed back at localhost - PARTNERSHIP_AUTH_WEBUIS=( - "NginxProxyManager|81" - "Lldap-Gmer4Lfe|17170" - "Authelia|9091" - "Authelia-Secondary|9092" - ) -# Paths on HOST1 that HOST2 should collect during the grace window after offboard -# Notified on offboard — no auto-deletion, manual collection - PARTNERSHIP_MIRROR_BACKUPS=( - # "/mnt/user/appdata-Failover/Jayred365-Emby" - ) +# Paths to collect during the grace window after offboard — defined per host in master_host*.conf. +# HOST1_PARTNERSHIP_MIRROR_BACKUPS / HOST2_PARTNERSHIP_MIRROR_BACKUPS +# Notified on offboard — no auto-deletion, manual collection. -# Timing — single var controls both Tailscale removal and backup access expiry -# Both expire at the same time — keeping backups accessible beyond Tailscale removal is pointless +# Timing — single var controls both Tailscale removal and backup access expiry. +# Both expire at the same time — keeping backups accessible beyond Tailscale removal is pointless. PARTNERSHIP_GRACE_HOURS=6 # hours after offboard before Tailscale removal # backup access expires at the same time PARTNERSHIP_OFFLINE_THRESHOLD=30 # days either server unreachable before auto-offboard # works both directions independently -# Tailscale removal on offboard +# Tailscale removal on offboard. PARTNERSHIP_REMOVE_TAILSCALE=true # remove mirror from Tailscale tailnet on offboard # false = skip removal (manual or testing) -# Tailscale API — required when PARTNERSHIP_REMOVE_TAILSCALE=true +# Tailscale API — required when PARTNERSHIP_REMOVE_TAILSCALE=true. +# Stays in shared conf — only owner uses it, and owner is always running this script. # API key: https://login.tailscale.com/admin/settings/keys → Devices write scope TAILSCALE_API_KEY="" # tskey-api-... TAILSCALE_TAILNET="" # your tailnet name (e.g. yourname.github) -# Transfer safety +# Transfer safety. PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership" PARTNERSHIP_TRANSFER_STRIKES=3 # consecutive health checks required PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20 # max health check attempts before giving up -# Onboard settings +# Onboard settings. PARTNERSHIP_ONBOARD_VERIFY=true # verify WebUI reachable after reconfiguration PARTNERSHIP_ONBOARD_NOTIFY=true # notify both servers on completion PARTNERSHIP_SYNC_INTERVAL=15 # minutes — informational, actual schedule in cron @@ -190,10 +182,17 @@ # ── LOGGING ─────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== +# Silent-by-default output model — ecosystem only speaks when something is wrong. +# true = only warn() and error() produce output (default — reduces notification spam) +# false = all output visible — use for monitor scripts or debugging +# Override per-run: script --log sets ENABLE_LOGGING=true for [LOG] detail +# Monitor scripts (coffee_report, health_digest etc.) set SILENT_MODE=false themselves + SILENT_MODE=true + # Controls verbose [LOG] output across all scripts. # true = show detailed [LOG] lines — useful for debugging or first-time setup # false = show only user-facing output — cleaner for scheduled runs - ENABLE_LOGGING=true + ENABLE_LOGGING=false # ============================================================================================== # ── NOTIFICATIONS ───────────────────────────────────────────────────────────────────────────── @@ -203,8 +202,9 @@ # normal = job completed successfully / warning = something failed or needs attention NOTIFY_UNRAID=true -# Discord webhook URL — leave blank to disable - DISCORD_WEBHOOK="" +# Discord webhook URL — defined per host in master_host*.conf. +# HOST1_DISCORD_WEBHOOK / HOST2_DISCORD_WEBHOOK +# Allows different webhooks per server, or only one server notifying. # ============================================================================================== # ── GIT / REPO ──────────────────────────────────────────────────────────────────────────────── @@ -232,7 +232,7 @@ # One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally. # Continuous scripts (watchdogs, failover) run until array stops. ARRAY_START_SCRIPTS=( - "git_pull_execute.sh" + "git_pull_execute.sh" # pull latest scripts before anything starts "Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts "unRAID_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill "unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning @@ -245,11 +245,11 @@ # ━━━ Daily Sync Maintenance ━━━ # daily_sync_maintenance.sh runs media share sync first, then iterates -# DAILY_MAINTENANCE_SCRIPTS for all pre and post sync jobs. +# DAILY_MAINTENANCE_SCRIPTS for all jobs. # Schedule: 0 1 * * * (1am daily) DAILY_MAINTENANCE_SCRIPTS=( "git_pull_execute.sh" # pull latest scripts — always runs first - "Media/media_shares_permissions.sh" # apply permissions + "Media/media_shares_permissions.sh" # apply permissions — runs before cleaners "Media/media_cleaner.sh anime" # remove junk from anime shares "Media/media_cleaner.sh media" # remove junk from media shares #"Media/lidarr_cleanup.sh" # remove orphaned music files — enable when ready @@ -259,42 +259,16 @@ ) # Media shares synced daily by daily_sync_maintenance.sh. +# Defined per-host in master_host*.conf — HOST1_DAILY_SYNC_SHARES and HOST2_DAILY_SYNC_SHARES. # Each server syncs only the shares it owns — direction is automatic. # HOST1 pushes its shares to HOST2. HOST2 pushes its shares to HOST1. # Never both pushing the same share — one server is always the truth holder. # These shares use DEFAULT_RSYNC_OPTS — no profile entry needed. # For shares needing custom options or container stops — create a profile in RSYNC section. - HOST1_DAILY_SYNC_SHARES=( - /mnt/user/Books - /mnt/user/Intros - /mnt/user/Kids_Movies - /mnt/user/Kids_Tv_Shows - /mnt/user/Movies - /mnt/user/Music - /mnt/user/Music_Videos - /mnt/user/Nextcloud - /mnt/user/stand-up_comedy - /mnt/user/Sports - /mnt/user/Tv_Shows - /mnt/user/Anime_Shows-Old - /mnt/user/Anime_Movies-Old - ) - HOST2_DAILY_SYNC_SHARES=( - /mnt/user/Anime_Movies - /mnt/user/Anime_Shows - ) - -# Personal encrypted shares — synced for offsite backup, independent of media shares. +# Personal encrypted shares defined per-host in master_host*.conf. # ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content. # See README-Rsync_Setup.md for ZFS encryption setup before uncommenting. - HOST1_PERSONAL_SHARES=( - # /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset - ) - - HOST2_PERSONAL_SHARES=( - # /mnt/user/HOST2-Personal # uncomment after creating encrypted dataset - ) # ━━━ Weekly Sync Maintenance ━━━ # weekly_sync_maintenance.sh stops containers both sides → pulls updates → @@ -304,14 +278,11 @@ "Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync ) -# Shares synced during the weekly maintenance window. +# Shares synced during the weekly maintenance window — defined per host in master_host*.conf. +# HOST1_WEEKLY_SYNC_SHARES / HOST2_WEEKLY_SYNC_SHARES # Containers stopped both sides before sync — full clean state guaranteed. # Profiles drive container stops, excludes, and options — configure in RSYNC section. # Order matters — Emby first (larger), then Critical-Data (auth stack). - WEEKLY_SYNC_SHARES=( - "/mnt/user/Media_Server/Emby" # emby profile — full clean mirror - "/mnt/user/appdata-Failover/Critical-Data" # critical-data profile — auth stack - ) # Container update toggles for the weekly sync window. # Containers already stopped for sync — updates pull at no extra downtime. @@ -323,7 +294,6 @@ # critical_sync_maintenance.sh runs every 15 minutes. # Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check # partnership --check always runs last regardless of rsync gate. -# Format for shares: "/path/to/share" or "/path/to/share|profile-name" # Jobs run every 15 minutes before the rsync shares. # Comment out to disable without removing. @@ -331,11 +301,10 @@ "Docker_Essentials/downloaders_reset.sh" # clear stuck download states every 15min ) -# Shares synced during the critical sync window. - CRITICAL_SYNC_SHARES=( - "/mnt/user/appdata-Failover/Critical-Data|critical-failover" - "/mnt/user/Media_Server/Emby|emby-failover" -) +# Shares synced every 15 minutes — defined per host in master_host*.conf. +# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES +# Format: "/path/to/share" or "/path/to/share|profile-name" +# Order matters — Critical-Data first (auth stack), then Emby dirty sync. # ============================================================================================== # ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── @@ -363,7 +332,7 @@ RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below DAILY_RSYNC_ENABLED=false # Tier 2 — HOST2 rebuild in progress, re-enable when ready WEEKLY_RSYNC_ENABLED=true # Tier 2 — weekly_sync_maintenance.sh rsync section - CRITICAL_RSYNC_ENABLED=false # Tier 2 — critical_sync_maintenance.sh (every 15min) + CRITICAL_RSYNC_ENABLED=false # Tier 2 — disabled during HOST2 rebuild, re-enable when ready FAILOVER_RSYNC_ENABLED=true # Tier 2 — failover.sh writeback jobs on handback # ━━━ Rsync Defaults ━━━ @@ -397,60 +366,56 @@ # List ALL desired options explicitly when defining a profile. # # Current profiles: -# arrs_stack — arr databases — lower bandwidth, containers stopped for consistency -# critical-data — auth stack — containers stopped both sides, Authelia delayed start -# gmer4lfe — server-specific appdata — no container stops needed -# important-data — NextCloud + Postgres — NextCloud delayed start after Postgres -# emby — weekly clean sync — both Emby stopped, full mirror -# called by weekly_sync_maintenance.sh only — do NOT schedule separately -# emby-failover — frequent dirty sync — Emby stays running, WAL excluded -# also used for failover writeback on handback -# critical-failover — frequent dirty sync — auth stays running, WAL excluded +# arrs_stack — arr databases — lower bandwidth, containers stopped for consistency +# critical-data — auth stack — full stop both sides, Authelia delayed start +# called by weekly_sync_maintenance.sh — full clean sync weekly +# critical-failover — dirty sync — auth stays running both sides, WAL excluded +# called by critical_sync_maintenance.sh every 15min +# host1-appdata — HOST1 server-specific appdata — defined in master_host1.conf +# host2-appdata — HOST2 server-specific appdata — defined in master_host2.conf +# important-data — NextCloud + Postgres — NextCloud delayed start after Postgres +# emby — weekly clean sync — both Emby stopped, full mirror +# called by weekly_sync_maintenance.sh only — do NOT schedule separately +# emby-failover — dirty sync — Emby stays running, WAL excluded # called by critical_sync_maintenance.sh every 15min -# full clean sync done by weekly (critical-data profile) declare -A PROFILE_RSYNC_OPTS=( [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete" - [gmer4lfe]="-av --info=progress2 --bwlimit=$BW_LIMIT" + [critical-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" [important-data]="-av --human-readable --bwlimit=$BW_LIMIT" [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" [emby-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" - [critical-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" ) # Per-profile bandwidth limits in KB/s declare -A PROFILE_BW_LIMIT=( [arrs_stack]=5000 # lower — runs alongside other syncs [critical-data]=9500 # high — small dataset, sync fast - [gmer4lfe]=8000 + [critical-failover]=9500 # high — small dataset, sync fast [important-data]=9500 # high — database sync [emby]=8000 # medium — large full mirror [emby-failover]=9500 # high — small critical dataset - [critical-failover]=9500 # high — small dataset, sync fast - ) # Retry attempts per profile declare -A PROFILE_RETRY_COUNT=( [arrs_stack]=3 [critical-data]=3 - [gmer4lfe]=3 + [critical-failover]=3 [important-data]=3 [emby]=3 [emby-failover]=3 - [critical-failover]=3 ) # Seconds between retry attempts declare -A PROFILE_SLEEP=( [arrs_stack]=300 [critical-data]=300 - [gmer4lfe]=300 + [critical-failover]=120 # shorter — frequent dirty sync, retry faster [important-data]=300 [emby]=300 [emby-failover]=120 # shorter — frequent dirty sync, retry faster - [critical-failover]=120 ) # Containers stopped on BOTH LOCAL and REMOTE before rsync. @@ -460,11 +425,10 @@ declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary" - [gmer4lfe]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe" + [critical-failover]="" # dirty sync — auth stays running both sides [important-data]="Postgres-NextCloud NextCloud" [emby]="Emby" [emby-failover]="" # dirty sync — Emby stays running both sides - [critical-failover]="" # dirty sync — auth stays running both sides ) # Containers needing a delay after rsync before starting. @@ -472,38 +436,44 @@ declare -A PROFILE_DELAYED_CONTAINERS=( [arrs_stack]="" [critical-data]="Authelia Authelia-Secondary" # wait for Mariadb + Redis - [gmer4lfe]="" + [critical-failover]="" [important-data]="NextCloud" # wait for Postgres [emby]="" [emby-failover]="" - [critical-failover]="" ) # Seconds before starting delayed containers declare -A PROFILE_CONTAINER_DELAY=( [arrs_stack]=5 [critical-data]=15 # Mariadb + Redis need time to accept connections - [gmer4lfe]=5 + [critical-failover]=5 [important-data]=10 # Postgres needs time before NextCloud [emby]=5 [emby-failover]=5 - [critical-failover]=5 ) # Directories excluded from rsync per profile. # SPACE-SEPARATED STRINGS — converted to array at runtime declare -A PROFILE_EXCLUDE_DIRS=( [arrs_stack]="logs *.tmp" - [gmer4lfe]="logs *.tmp" - [important-data]="logs *.tmp" [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt" + [critical-failover]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt *.db-wal *.db-shm" + [important-data]="logs *.tmp" [emby]="logs transcodes cache crash*" [emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root" - [critical-failover]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt *.db-wal *.db-shm" + ) + +# Remote restart after dirty sync — restart these on remote IF they were running before sync. +# Same logic as stop/start — was stopped = stays stopped, was running = gets restarted. +# Used by dirty sync profiles (critical-failover, emby-failover) so remote picks up changes. +# SPACE-SEPARATED STRINGS — converted to array at runtime + declare -A PROFILE_REMOTE_RESTART_CONTAINERS=( + [critical-failover]="NginxProxyManager Authelia Authelia-Secondary Lldap-Gmer4Lfe Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary" + [emby-failover]="Emby" ) # Note: disk check is auto-detected from disks.ini — no PROFILE_SKIP_DISK_CHECK needed. -# check_remote_disks() reads fsType per disk and handles XFS and ZFS automatically. +# check_remote_disks() reads fsType per disk and handles XFS, ZFS, and cache pools automatically. # ============================================================================================== # ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── @@ -519,157 +489,20 @@ # Failover → start remote DDNS first (Tier 1) # Handback → stop remote DDNS → rsync → start containers → start local DDNS last # -# Tiers: -# Tier 1 — Immediate — vital services + Live TV -# Tier 2 — configurable delay — productivity services -# Tier 3 — configurable delay — secondary services -# Tier 4 — configurable delay — arrs + downloaders +# Per-host container lists and tier delays live in master_host*.conf. +# Shared settings (intervals, state file, thresholds) live here. EXTERNAL_IP="8.8.8.8" - FAILOVER_CHECK_INTERVAL=120 - FAILOVER_HANDBACK_STRIKES=2 + FAILOVER_CHECK_INTERVAL=120 # seconds between failover state checks + FAILOVER_HANDBACK_STRIKES=2 # consecutive healthy checks before initiating handback FAILOVER_STATE_FILE="/boot/config/failover_state.db" FAILOVER_ENABLED=false # HOST2 being rebuilt — set true when back online and tested # false = suppresses "not running" warnings in status scripts # ━━━ Failover Test ━━━ - FAILOVER_TEST_BLOCK_WAIT=150 - FAILOVER_TEST_HANDBACK_WAIT=360 - -# ━━━ DDNS ━━━ - HOST1_DDNS_CONTAINERS=( - "Gmer4Lfe.com" - ) - - HOST2_DDNS_CONTAINERS=( - "Gmer4Lfe.us" - ) - -# ━━━ Internet Loss ━━━ - FAILOVER_HOST1_STOP_ON_NO_NET=( - "Gmer4Lfe.com" - ) - - FAILOVER_HOST2_STOP_ON_NO_NET=( - "Gmer4Lfe.us" - ) - -# ━━━ Tiered Container Lists ━━━ - -# HOST1 runs for HOST2 (HOST2 is down) - FAILOVER_HOST1_RUNS_FOR_HOST2_IMMEDIATE=( - "Gmer4Lfe.us" - "VaultWarden-Jayred365" - ) - - FAILOVER_HOST1_RUNS_FOR_HOST2_2HR=( - # "container-placeholder" - ) - - FAILOVER_HOST1_RUNS_FOR_HOST2_6HR=( - # "container-placeholder" - ) - - FAILOVER_HOST1_RUNS_FOR_HOST2_18HR=( - # "container-placeholder" - ) - -# HOST2 runs for HOST1 (HOST1 is down) - FAILOVER_HOST2_RUNS_FOR_HOST1_IMMEDIATE=( - "Gmer4Lfe.com" - "Emby" - "VaultWarden-Gmer4Lfe" - "Dispatcharr" - "Dispatcharr-Basic" - "Dispatcharr-Iptv-Users" - "ErsatzTV-Emby" - ) - - FAILOVER_HOST2_RUNS_FOR_HOST1_2HR=( - "Postgres-NextCloud" - "NextCloud" - "PostgreSQL_Immich" - "Immich-Gmer4Lfe" - ) - - FAILOVER_HOST2_RUNS_FOR_HOST1_6HR=( - "Gitea" - ) - - FAILOVER_HOST2_RUNS_FOR_HOST1_18HR=( - "Sonarr" - "Radarr" - "Lidarr" - "Readarr" - "Prowlarr" - "Bazarr" - "SABnzbd-Gmer4Lfe" - "Qbittorrent-Gmer4Lfe" - "LidaTube" - "Pinchflat" - "ChannelTube" - ) - -# ━━━ Tier Delay Settings ━━━ -# How long the primary must be down before each tier activates — in minutes. -# Tier 1 is always immediate. Set independently per host. - -# HOST1's containers running on HOST2 (HOST1 is down) - HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich - HOST1_TIER3_DELAY=720 # 12 hours — secondary services - HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders - -# HOST2's containers running on HOST1 (HOST2 is down) - HOST2_TIER2_DELAY=240 - HOST2_TIER3_DELAY=720 - HOST2_TIER4_DELAY=1440 - -# ━━━ Rsync Writeback Jobs ━━━ -# Syncs critical appdata BACK to primary during handback after failover. -# Containers stopped before writeback runs — clean source, no competing writes. -# -# HOST*_TIER1_WRITEBACK_DELAY: -# Short outages skip Tier 1 writeback — primary state is more reliable than dirty sync data -# Only writeback if outage lasted longer than this many minutes -# -# Tier 4 writeback automatically syncs HOST*_DAILY_SYNC_SHARES — no need to list those here. -# Only add paths NOT in DAILY_SYNC_SHARES that need writeback after extended outage. - HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr - HOST2_TIER1_WRITEBACK_DELAY=60 - -# HOST1 writeback — run by HOST2 during HOST1 handback - FAILOVER_HOST1_WRITEBACK_TIER1=( - "/mnt/user/Media_Server/Emby" # watch states built up during outage - ) - - FAILOVER_HOST1_WRITEBACK_TIER2=( - "/mnt/user/appdata-Failover/Important-Data" # NextCloud + Postgres - ) - - FAILOVER_HOST1_WRITEBACK_TIER3=( - # "location-placeholder" - ) - - FAILOVER_HOST1_WRITEBACK_TIER4=( - "/mnt/user/appdata-Failover/Arrs_Stack" # arr databases — downloads queued during outage - ) - -# HOST2 writeback — run by HOST1 during HOST2 handback - FAILOVER_HOST2_WRITEBACK_TIER1=( - # "/mnt/user/appdata-Failover/Jayred365-Emby" - ) - - FAILOVER_HOST2_WRITEBACK_TIER2=( - # "/mnt/user/appdata-Failover/Jayred365-Important" - ) - - FAILOVER_HOST2_WRITEBACK_TIER3=( - # "location-placeholder" - ) - - FAILOVER_HOST2_WRITEBACK_TIER4=( - "/mnt/user/appdata-Failover/Arrs_Stack" - ) +# Controlled simulation of a failover event — run manually via failover_test.sh. + FAILOVER_TEST_BLOCK_WAIT=150 # seconds to wait after blocking connectivity + FAILOVER_TEST_HANDBACK_WAIT=360 # seconds to wait before initiating handback # ============================================================================================== # ── DOCKER ESSENTIALS ───────────────────────────────────────────────────────────────────────── @@ -677,57 +510,33 @@ # ━━━ Downloaders Reset ━━━ # Runs every 15 minutes via CRITICAL_MAINTENANCE_SCRIPTS. -# Clears stuck states, purges old history, prepares each client for a clean cycle. - DOWNLOADER_RETENTION_DAYS=7 +# Clears stuck states, purges old history, prepares each download client for a clean cycle. +# Per-host URLs and API keys live in master_host*.conf. + DOWNLOADER_RETENTION_DAYS=7 # days — purge history older than this -# ── slskd ── -# Clears stuck searches, dead transfers, purges expired failed imports. -# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected. - HOST1_SLSKD_URL="http://localhost:8980" - HOST1_SLSKD_API_KEY="4bF9kL2mNpQrT7vWxYz1A3dEgHjKoRsU" - HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports" - -# ── SABnzbd ── - HOST1_SABNZBD_URL="http://localhost:8180" - HOST1_SABNZBD_API_KEY="8bfefe41d83b4d50883e32859b55ca9a" - -# ── qBittorrent ── -# deleteFiles=false — removes torrent from qBit but leaves files on disk. -# Radarr/Sonarr manage actual files independently. - HOST1_QBIT_URL="http://localhost:8080" - HOST1_QBIT_USERNAME="root" - HOST1_QBIT_PASSWORD="Stay0utD!ck" - QBIT_FAILSAFE_MIN_DAYS=180 - QBIT_FAILSAFE_MIN_RATIO=0 # 0 = age only, no ratio requirement +# qBittorrent failsafe — removes torrents older than threshold regardless of ratio +# deleteFiles=false — removes from qBit but leaves files on disk (arr manages files) + QBIT_FAILSAFE_MIN_DAYS=180 # days — minimum age before failsafe deletion + QBIT_FAILSAFE_MIN_RATIO=0 # 0 = age only, no ratio requirement # ━━━ Docker Daily Restart ━━━ -# Containers restarted every day via daily_sync_maintenance.sh. -# Dispatcharr degrades over time without restart — daily is intentional. - DAILY_RESTART_CONTAINERS=( - "NginxProxyManager" - "Lldap-Gmer4Lfe" - "Authelia" - "Authelia-Secondary" - "Dispatcharr-Iptv-Users" - "Dispatcharr" # Live TV scheduler — degrades without daily restart - "Dispatcharr-Basic" - "ErsatzTV-Emby" - ) +# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS. +# Per-host lists live in master_host*.conf: +# HOST1_DAILY_RESTART_CONTAINERS +# HOST2_DAILY_RESTART_CONTAINERS +# detect_hosts() sets DAILY_RESTART_CONTAINERS to the correct host array at runtime. # ━━━ Docker Weekly Restart ━━━ -# Less critical services restarted weekly via weekly_sync_maintenance.sh (Sunday 2:30am). +# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am). # Containers already stopped for weekly sync — restart adds zero extra downtime. - WEEKLY_RESTART_CONTAINERS=( - "NextCloud" - "Organizrv2-Gmer4Lfe" - "AdGuard-Home" - "Immich-Gmer4Lfe" - ) +# Per-host lists live in master_host*.conf: +# HOST1_WEEKLY_RESTART_CONTAINERS +# HOST2_WEEKLY_RESTART_CONTAINERS # ━━━ Docker Watchdog ━━━ # Continuous two-tier self-healing container monitoring. # Started by array_start.sh — runs until array stops. -# Re-sources Master.conf each cycle — add/remove containers without restarting watchdog. +# Re-sources all three conf files each cycle — add/remove containers without restarting watchdog. # # Tier 1 — strict monitoring of explicitly configured containers: # Memory hard limits — immediate restart if exceeded @@ -741,49 +550,20 @@ # Crash loop detection — RestartCount climbing → notify, critical above limit # Dead containers — remove and restart # Unexpected exits — non-zero exit code → restart - -# Memory hard limits in MB — immediate restart if exceeded -# 20GB=20480 16GB=16384 12GB=12288 10GB=10240 8GB=8192 4GB=4096 2GB=2048 1GB=1024 - declare -A WATCHDOG_CONTAINERS=( - ["Emby"]=18432 - ["LidaTube"]=6144 - ["Tdarr"]=6144 - ["Code-Server"]=1024 - ) - -# HTTP health check URLs — checked every cycle, strike system before restart - declare -A HOST1_WATCHDOG_CONTAINER_URLS=( - ["Emby"]="http://localhost:8096" - ) - - declare -A HOST2_WATCHDOG_CONTAINER_URLS=( - ["Emby"]="http://localhost:8096" - ) - -# Required containers — must always be running -# Strike system before restart, skip list after repeated failures, auto-clears on recovery - HOST1_WATCHDOG_REQUIRED_CONTAINERS=( - "NginxProxyManager" - "Lldap-Gmer4Lfe" - "Authelia" - "Authelia-Secondary" - "Mariadb-Authelia" - "Mariadb-Authelia-Secondary" - "Redis-Authelia" - "Redis-Authelia-Secondary" - ) - - HOST2_WATCHDOG_REQUIRED_CONTAINERS=( - "NginxProxyManager" - # add HOST2 required containers here - ) +# +# All per-host container lists live in master_host*.conf: +# HOST*_WATCHDOG_CONTAINERS — memory hard limits per container +# HOST*_WATCHDOG_CONTAINER_URLS — HTTP health check URLs +# HOST*_WATCHDOG_REQUIRED_CONTAINERS — must always be running +# HOST*_WATCHDOG_SCAN_IGNORE — skip in Tier 2 scan +# HOST*_WATCHDOG_DEPENDENCIES — dependency ordering for restart decisions # Strike state file — /tmp resets on reboot (correct — no stale strikes after reboot) WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db" # CPU thresholds — normalised against total core count at runtime - SOFT_CPU_THRESHOLD=80 - HARD_CPU_THRESHOLD=85 + SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU + HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU CPU_FAIL_LIMIT=2 # consecutive hard CPU strikes before container restart # Memory soft threshold — warn when container reaches this % of its hard limit @@ -791,28 +571,18 @@ # HTTP responsiveness RESP_FAIL_LIMIT=2 # consecutive failed checks before restart - CURL_TIMEOUT=5 # seconds per check + CURL_TIMEOUT=5 # seconds per check before timeout # Watchdog cycle interval - DOCKER_WATCHDOG_INTERVAL=900 # seconds between cycles (15 minutes) + DOCKER_WATCHDOG_INTERVAL=900 # seconds between watchdog cycles (15 minutes) -# Heartbeat +# Heartbeat — proof of life logged periodically even when everything is healthy DOCKER_WATCHDOG_HEARTBEAT=true DOCKER_WATCHDOG_HEARTBEAT_HOURS=1 # Tier 2 master toggle WATCHDOG_SCAN_ALL=true # false = only WATCHDOG_CONTAINERS + required containers -# Containers to skip in Tier 2 scan — legitimately stopped or frequently restarting - WATCHDOG_SCAN_IGNORE=( - "DashGate" - "PIA-WG-Config-Generator" - "Aperture" - "Aperture-Kids" - "pgvector-18-Apeture-Kids" - "Pgvector18-Aperture" - ) - # Individual Tier 2 check toggles WATCHDOG_RESTART_UNHEALTHY=true WATCHDOG_RESTART_DEAD=true @@ -821,12 +591,12 @@ WATCHDOG_NOTIFY_CRASHLOOP=true # Crash loop threshold - WATCHDOG_CRASH_LIMIT=5 + WATCHDOG_CRASH_LIMIT=5 # RestartCount above this = critical crash loop # Startup grace period — skip restarts while system is still booting - WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts + WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures -# Restart loop protection +# Restart loop protection — prevents watchdog from endlessly restarting a broken container WATCHDOG_CONTAINER_RESTART_LIMIT=3 WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db" @@ -834,23 +604,12 @@ # Notification batching — one summary per cycle instead of one ping per event WATCHDOG_BATCH_NOTIFY=true -# Dependency ordering — skip restarting a container if its dependency is also down - declare -A WATCHDOG_DEPENDENCIES=( - ["Authelia"]="Mariadb-Authelia Redis-Authelia" - ["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary" - ["NextCloud"]="Postgres-NextCloud" - ) - # ━━━ Docker Network Connect ━━━ # Ensures custom networks exist and connects containers at array start. - NETWORK_CONNECT_CONTAINERS=( - "memcached" - "Npm-CrowdSec" - ) - - NETWORK_CONNECT_NETWORKS=( - "high-availability" - ) +# Runs once via ARRAY_START_SCRIPTS — idempotent, safe to re-run. +# Container and network lists are host-specific — defined in master_host*.conf: +# HOST1_NETWORK_CONNECT_CONTAINERS / HOST2_NETWORK_CONNECT_CONTAINERS +# HOST1_NETWORK_CONNECT_NETWORKS / HOST2_NETWORK_CONNECT_NETWORKS # ============================================================================================== # ── UNRAID ESSENTIALS ───────────────────────────────────────────────────────────────────────── @@ -860,9 +619,17 @@ # Linux inotify limits — applied at every array start by inotify_tuning.sh. # Default unRAID values are very low — with many Docker containers watching files # you can silently exhaust the limit causing containers to miss file events. +# Symptoms: containers miss file events, downloads not detected, library not updated. +# Live TV stutter is a known symptom of inotify exhaustion. # These settings are lost on reboot — reapplied automatically at array start. - INOTIFY_MAX_INSTANCES=1024 # default: 128 — max inotify instances per user - INOTIFY_MAX_WATCHES=524288 # default: 8192 — max files watched per instance + INOTIFY_MAX_INSTANCES=1024 # default: 128 — max inotify instances per user + # 1024 handles ~20-30 containers watching files simultaneously + INOTIFY_MAX_WATCHES=1048576 # default: 8192 — SHARED budget across ALL users/containers + # 524288 (512K) was previous value — raised to 1M (1048576) + # VSCode/Code-Server alone needs ~50K-200K for large workspaces + # with node_modules. All arr containers + Emby + VSCode share + # this budget. 1M is safe on 128GB RAM (~128MB kernel memory) + # If VSCode shows "unable to watch for file changes" → too low INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — max events queued before dropping # ━━━ System Tuning Monitor ━━━ @@ -876,7 +643,7 @@ # ━━━ Reboot ━━━ # Seconds of warning broadcast to logged-in users before server_reboot.sh reboots. -# Gives users time to save work — 300s = 5 minutes +# Gives users time to save work — 300s = 5 minutes. REBOOT_SLEEP=300 # ━━━ Mover ━━━ @@ -900,13 +667,21 @@ # ━━━ Clear Logs ━━━ # System log files cleared weekly to prevent rootfs fill over time. # These grow continuously — without clearing they eventually consume all rootfs space. +# clear_logs.sh runs in WEEKLY_MAINTENANCE_SCRIPTS — Sunday 2:30am. +# +# Size threshold approach — only clear if log exceeds threshold. +# Avoids destroying useful recent diagnostic context when logs are small. +# LOG_MIN_SIZE_MB: skip clearing if log is under this size (not worth clearing) +# LOG_DOCKER_MAX_MB: clear a container log only if it exceeds this size +# Docker logs grow fastest on active containers (Emby, SABnzbd, Sonarr) +# 100MB per container × 30 containers = 3GB before clearing kicks in LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) + LOG_MIN_SIZE_MB=10 # skip system log if under this size (already small) + LOG_DOCKER_MAX_MB=100 # clear Docker container log only if over this size (MB) # ━━━ WebGUI Watchdog ━━━ # Monitors unRAID WebGUI responsiveness — escalates through nginx restart → emhttp restart. # Separate from docker_watchdog — this monitors the unRAID UI itself, not containers. -# WEBGUI_NGINX_WAIT = seconds after nginx restart before rechecking -# WEBGUI_EMHTTP_WAIT = seconds after emhttp restart before rechecking WEBGUI_URL="http://localhost" WEBGUI_TIMEOUT=5 # seconds before curl gives up on WebGUI check WEBGUI_NGINX_WAIT=15 # seconds after nginx restart before rechecking @@ -919,99 +694,129 @@ # ━━━ Media Permissions ━━━ # Applied recursively by media_shares_permissions.sh. # Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership. - PERMISSIONS_MODE="777" +# +# Why split directory vs file permissions: +# Directories need execute bit to enter — 755 allows owner+group+others to traverse +# Files should NOT be executable — 664 allows owner+group read/write, others read +# This is POSIX best practice for media servers, not a blanket 777 band-aid +# +# Why nobody:users ownership: +# All ecosystem containers run as PUID=99 (nobody) PGID=100 (users) on unRAID +# Files owned by nobody:users are accessible to all containers without world-write +# If this script fixes many files each run → a container has wrong PUID/PGID set +# Fix: add PUID=99 PGID=100 to the container's environment variables +# +# This script runs daily as a failsafe — even with correct container config: +# rsync may bring files with source ownership if not run with --chown +# Manual admin copies create root:root files +# New containers may not have PUID/PGID set yet + PERMISSIONS_DIR_MODE="755" # directories — traverse + list, no world-write + PERMISSIONS_FILE_MODE="664" # files — group read/write, no execute PERMISSIONS_OWNER="nobody:users" - MEDIA_PERMISSION_SHARES=( - /mnt/user/Anime_Movies - /mnt/user/Anime_Movies-Old - /mnt/user/Anime_Shows - /mnt/user/Anime_Shows-Old - /mnt/user/appcache - /mnt/user/Books - /mnt/user/Downloads - /mnt/user/Games - /mnt/user/Intros - /mnt/user/Kids_Movies - /mnt/user/Kids_Tv_Shows - /mnt/user/Movie_Recordings - /mnt/user/Movies - /mnt/user/Music - /mnt/user/Music_Videos - /mnt/user/Photo - /mnt/user/Sports - /mnt/user/stand-up_comedy - /mnt/user/Temp_Storage - /mnt/user/Tv_Recordings - /mnt/user/Tv_Shows - /mnt/user/YouTube - ) +# Share list defined per host in master_host*.conf: +# HOST1_MEDIA_PERMISSION_SHARES / HOST2_MEDIA_PERMISSION_SHARES # ━━━ Media Cleaner ━━━ # Removes junk files from media shares — two profiles: anime and media. # Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media - ANIME_CLEAN_FOLDERS=( - /mnt/user/Anime_Movies - /mnt/user/Anime_Movies-Old - /mnt/user/Anime_Shows - /mnt/user/Anime_Shows-Old - ) - - MEDIA_CLEAN_FOLDERS=( - /mnt/user/Kids_Movies - /mnt/user/Kids_Tv_Shows - /mnt/user/Movies - /mnt/user/Music - /mnt/user/Sports - /mnt/user/stand-up_comedy - /mnt/user/Tv_Shows - ) +# Folder lists defined per host in master_host*.conf: +# HOST1_ANIME_CLEAN_FOLDERS / HOST2_ANIME_CLEAN_FOLDERS +# HOST1_MEDIA_CLEAN_FOLDERS / HOST2_MEDIA_CLEAN_FOLDERS ANIME_FILE_PATTERNS=( - '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' - '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' - '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp' - '*.log' '*.json' + # ── Checksums ──────────────────────────────────────────────────────────────────────── + '*.sfv' '*.md5' '*.sha1' # verification files — useless post-download + + # ── Scene / download metadata ───────────────────────────────────────────────────────── + '*.url' '*.lnk' # scene links + '*.nfo' # scene info files (arrs regenerate their own) + '*.info' '*.diz' # scene description files + '*.nzb' # usenet download files + '*.torrent' # torrent files left by download clients + + # ── Archives and segments ───────────────────────────────────────────────────────────── + '*.rar' '*.zip' '*.7z' '*.ace' # archives — source files not needed after extract + '*.r00' '*.r01' '*.r02' '*.r03' # multi-part rar segments + '*.r04' '*.r05' '*.r06' '*.r07' + '*.r08' '*.r09' '*.srr' # scene repair files + '*.001' '*.002' '*.003' # split archive parts + '*.gz' '*.tar' '*.bz2' # linux archives + + # ── Scene samples and proofs ────────────────────────────────────────────────────────── + '*.sample*' '*.proof*' # scene samples — never needed in library + + # ── Executables and scripts ─────────────────────────────────────────────────────────── + '*.exe' '*.scr' '*.com' # Windows executables + '*.bat' '*.cmd' '*.vbs' '*.ps1' # Windows scripts + '*.msi' '*.dll' '*.sys' # Windows installers and system files + '*.sh' # shell scripts in media folders = suspicious + + # ── Incomplete downloads ────────────────────────────────────────────────────────────── + '*.!ut' '*.!qB' # uTorrent / qBittorrent incomplete markers + '*.crdownload' '*.opdownload' # Chrome / Opera incomplete downloads + '*.part' # partial download files + + # ── Sync conflicts ──────────────────────────────────────────────────────────────────── + '*sync-conflict*' # Syncthing conflict copies ) MEDIA_FILE_PATTERNS=( - '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' - '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' - '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp' - '*.log' '*.json' '*.iso' '*.lrc' + # ── Checksums ──────────────────────────────────────────────────────────────────────── + '*.sfv' '*.md5' '*.sha1' # verification files — useless post-download + + # ── Scene / download metadata ───────────────────────────────────────────────────────── + '*.url' '*.lnk' # scene links + '*.nfo' # scene info files (arrs regenerate their own) + '*.info' '*.diz' # scene description files + '*.nzb' # usenet download files + '*.torrent' # torrent files left by download clients + + # ── Archives and segments ───────────────────────────────────────────────────────────── + '*.rar' '*.zip' '*.7z' '*.ace' # archives — source files not needed after extract + '*.r00' '*.r01' '*.r02' '*.r03' # multi-part rar segments + '*.r04' '*.r05' '*.r06' '*.r07' + '*.r08' '*.r09' '*.srr' # scene repair files + '*.001' '*.002' '*.003' # split archive parts + '*.gz' '*.tar' '*.bz2' # linux archives + + # ── Scene samples and proofs ────────────────────────────────────────────────────────── + '*.sample*' '*.proof*' # scene samples — never needed in library + + # ── Executables and scripts ─────────────────────────────────────────────────────────── + '*.exe' '*.scr' '*.com' # Windows executables + '*.bat' '*.cmd' '*.vbs' '*.ps1' # Windows scripts + '*.msi' '*.dll' '*.sys' # Windows installers and system files + '*.sh' # shell scripts in media folders = suspicious + + # ── Incomplete downloads ────────────────────────────────────────────────────────────── + '*.!ut' '*.!qB' # uTorrent / qBittorrent incomplete markers + '*.crdownload' '*.opdownload' # Chrome / Opera incomplete downloads + '*.part' # partial download files + + # ── Sync conflicts ──────────────────────────────────────────────────────────────────── + '*sync-conflict*' # Syncthing conflict copies + + # ── Media-specific extras ───────────────────────────────────────────────────────────── + '*.iso' # disc images — extracted content is in library + '*.lrc' # lyrics files — handled by music apps not Emby ) # ━━━ Arr Cleanup ━━━ # Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs. -# detect_hosts() selects correct URL, API key, and root path at runtime. +# Per-host URLs, API keys, and path maps live in master_host*.conf. +# detect_hosts() selects correct host vars at runtime. # -# API versions: +# API versions — update MAJOR version here when script is updated for a new arr version: # Sonarr v4 → /api/v3/series → /api/v3/episodefile?seriesId=X # Radarr v6 → /api/v3/movie → /api/v3/moviefile?movieId=X # Lidarr v3 → /api/v1/artist → /api/v1/trackFile?artistId=X -# -# Version checking — scripts verify major version matches before running. -# Update the MAJOR version here when script is updated to support a new arr version. SONARR_VERSION_MAJOR=4 RADARR_VERSION_MAJOR=6 LIDARR_VERSION_MAJOR=3 -# ── Lidarr ──────────────────────────────────────────────────────────────────────────────────── - HOST1_LIDARR_URL="http://localhost:8686" - HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc" - HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New" +# Lidarr shared settings LIDARR_LOCK_WARN_AGE=3600 # 1hr — large libraries take time, not stuck - -# Container path → host path translation -# Lidarr stores file paths using container paths — script scans host paths -# Add one entry per root folder configured in Lidarr Settings → Media Management → Root Folders - declare -A HOST1_LIDARR_PATH_MAP=( - ["/ext-music"]="/mnt/user/Music-New" - ) - declare -A HOST2_LIDARR_PATH_MAP=( - # HOST2 does not run Lidarr — fill in if that changes - ) - LIDARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion LIDARR_MAX_DELETE_GB=5 # require --i-know-what-im-doing if deletion exceeds this LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run @@ -1030,32 +835,9 @@ "*.lrc" ) -# ── Sonarr ──────────────────────────────────────────────────────────────────────────────────── - HOST1_SONARR_URL="http://localhost:8989" - HOST1_SONARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" - HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows" - -# Container path → host path translation -# Add one entry per root folder configured in Sonarr Settings → Media Management → Root Folders -# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder - declare -A HOST1_SONARR_PATH_MAP=( - ["/tv"]="/mnt/user/Tv_Shows" - ["/ext-standup-comedy"]="/mnt/user/stand-up_comedy" - ["/kids tv"]="/mnt/user/Kids_Tv_Shows" - ["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old" - ) - - HOST2_SONARR_URL="http://localhost:8989" - HOST2_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f" - HOST2_SONARR_TV_ROOT="/mnt/user/Anime_Shows" - - declare -A HOST2_SONARR_PATH_MAP=( - # fill in when HOST2 is back online - # ["/tv"]="/mnt/user/Anime_Shows" - ) - +# Sonarr shared settings SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion - SONARR_MAX_DELETE_GB=10 # require --i-know-what-im-doing if deletion exceeds this + SONARR_MAX_DELETE_GB=10 # require --i-know-what-im-doing if deletion exceeds this SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") SONARR_PROTECTED_PATTERNS=( # Subtitles @@ -1070,36 +852,13 @@ "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" "*-clip.*" "*-other.*" - # Theme songs — stored in show folder, not tracked + # Theme songs — stored in show folder, not tracked by arr "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" ) -# ── Radarr ──────────────────────────────────────────────────────────────────────────────────── - HOST1_RADARR_URL="http://localhost:7878" - HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" - HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies" - -# Container path → host path translation -# Add one entry per root folder configured in Radarr Settings → Media Management → Root Folders -# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder - declare -A HOST1_RADARR_PATH_MAP=( - ["/movies"]="/mnt/user/Movies" - ["/kids movies"]="/mnt/user/Kids_Movies" - ["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy" - ["/anime-movies"]="/mnt/user/Anime_Movies-Old" - ) - - HOST2_RADARR_URL="http://localhost:7878" - HOST2_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" - HOST2_RADARR_MOVIES_ROOT="/mnt/user/Anime_Movies" - - declare -A HOST2_RADARR_PATH_MAP=( - # fill in when HOST2 is back online - # ["/anime-movies"]="/mnt/user/Anime_Movies" - ) - +# Radarr shared settings RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion - RADARR_MAX_DELETE_GB=15 # require --i-know-what-im-doing if deletion exceeds this + RADARR_MAX_DELETE_GB=15 # require --i-know-what-im-doing if deletion exceeds this RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") RADARR_PROTECTED_PATTERNS=( # Subtitles @@ -1114,7 +873,7 @@ "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" "*-clip.*" "*-other.*" - # Theme songs — stored in movie folder, not tracked + # Theme songs — stored in movie folder, not tracked by arr "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" ) @@ -1129,76 +888,69 @@ # stalled — download stuck with no connections or progress # # Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first. -# Lidarr runs on HOST1 only — exits cleanly on HOST2. +# Per-host recovery toggles (HOST1_SONARR_RECOVERY etc.) live in master_host*.conf. ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this # matches cron interval — items eligible after one missed cycle - HOST1_SONARR_RECOVERY=true - HOST1_RADARR_RECOVERY=true - HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2 - HOST2_SONARR_RECOVERY=true - HOST2_RADARR_RECOVERY=true - # ============================================================================================== # ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Session-based storage allocator using filesystem symlink indirection. # ffmpeg resolves the symlink ONCE at session start — existing sessions never affected. +# Symlink flips between ramdisk and SSD are transparent to active streams. # -# ⚠️ Docker mount — must use shared propagation: +# ⚠️ Docker mount — must use shared propagation so symlink flips work inside container: # --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared +# Standard path mappings use rprivate — symlink changes are NOT visible inside container. # ━━━ Transcode Manager ━━━ -# tmpfs mount point — created at array start by ramdisk_setup.sh -# Must exist before Emby starts so the symlink resolves correctly +# tmpfs mount point — created at array start by ramdisk_setup.sh. +# Must exist before Emby starts so the symlink resolves correctly. RAMDISK_PATH="/mnt/ramdisk_transcodes" -# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront -# Real-world: 9 streams peaked at ~5.5GB — 8G gives comfortable headroom - RAMDISK_SIZE="8G" +# Ramdisk size and flip thresholds — defined per host in master_host*.conf. +# All three are coupled — if size changes, thresholds must change with it. +# HOST1_RAMDISK_SIZE / HOST2_RAMDISK_SIZE +# HOST1_RAMDISK_WARN_GB / HOST2_RAMDISK_WARN_GB ← flip to SSD at this usage +# HOST1_RAMDISK_LOW_GB / HOST2_RAMDISK_LOW_GB ← flip back to ramdisk at this usage -# Symlink that Emby points at — this path NEVER changes regardless of ramdisk/SSD state -# Emby resolves the symlink once per session at start — symlink flips are transparent -# Must match the container path configured in Emby's Extra Parameters +# Symlink that Emby points at — this path NEVER changes regardless of ramdisk/SSD state. +# Emby resolves the symlink once per session at start — symlink flips are transparent. +# Must match the container path configured in Emby's Extra Parameters. TRANSCODE_LINK="/mnt/ram-transcode" -# SSD fallback location — where transcodes land when ramdisk is too full - TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" +# SSD fallback location — defined per host in master_host*.conf (cache path differs per server): +# HOST1_TRANSCODE_SSD / HOST2_TRANSCODE_SSD -# Usage thresholds in GB — hysteresis gap between WARN and LOW prevents flip-flop -# Gap (6.8 - 5.5 = 1.3GB) means ramdisk must drop 1.3GB before flipping back - RAMDISK_WARN_GB=6.8 # flip to SSD when ramdisk usage reaches this - RAMDISK_LOW_GB=5.5 # flip back to ramdisk when usage drops to this - RAMDISK_SSD_MIN_GB=20 # minimum free GB on SSD before allowing flip to SSD +# Minimum free GB on SSD before allowing flip from ramdisk to SSD. + RAMDISK_SSD_MIN_GB=20 TRANSCODE_MAX_AGE=20 # minutes — HLS segment age before cleanup eligibility - TRANSCODE_ORPHAN_AGE=30 # minutes — files with no matching session at all + TRANSCODE_ORPHAN_AGE=30 # minutes — files with no matching active session TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in one hour TRANSCODE_OWNER="nobody:users" TRANSCODE_CHMOD="755" -# Operating mode — controls symlink direction behaviour +# Operating mode — controls symlink direction behaviour. # smart — auto-flips between ramdisk and SSD based on thresholds (default) # ramdisk — always uses ramdisk, warns if RAMDISK_WARN_GB exceeded but holds # ssd — always uses SSD, never flips to ramdisk - TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd + TRANSCODE_MANAGER_MODE="smart" -# Daily statistics log — read by weekly_health_digest.sh for transcode summary +# Daily statistics log — read by weekly_health_digest.sh for transcode summary. TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db" TRANSCODE_LOG_RETENTION=90 # days before old entries purged + TRANSCODE_CHECK_EMBY=true + # ━━━ Transcode Server Array ━━━ # All media servers sharing the ramdisk transcode space. # Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex # Entries with placeholder API keys are skipped automatically. -# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD. - TRANSCODE_SERVERS=( - "${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby" - # "${HOST2_EMBY_CONTAINER}|${HOST2_EMBY_URL}|${HOST2_EMBY_API_KEY}|emby" - ) - - TRANSCODE_CHECK_EMBY=true +# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk. +# Defined per host in master_host*.conf — Emby container names and keys differ per server: +# HOST1_TRANSCODE_SERVERS / HOST2_TRANSCODE_SERVERS # ============================================================================================== # ── MONITORS ────────────────────────────────────────────────────────────────────────────────── @@ -1207,24 +959,17 @@ # ━━━ Certificate Monitor ━━━ # Checks SSL certificate expiry via direct openssl connection — no NPM dependency. # Checks the actual certificate served by each domain, not what NPM thinks it has. -# CERT_WARN_DAYS = notify this many days before expiry -# CERT_CRIT_DAYS = escalate to critical this many days before expiry -# CERT_TIMEOUT = seconds before giving up on the openssl connection - CERT_MONITOR_DOMAINS=( - "Gmer4Lfe.com" - "Gmer4Lfe.us" - ) +# Domains defined per host in master_host*.conf — each server monitors its own domains: +# HOST1_CERT_MONITOR_DOMAINS / HOST2_CERT_MONITOR_DOMAINS CERT_WARN_DAYS=30 # warn when cert expires within this many days CERT_CRIT_DAYS=7 # critical alert within this many days - CERT_TIMEOUT=10 # seconds per domain check + CERT_TIMEOUT=10 # seconds per domain before giving up # ━━━ Backup Verify ━━━ # Verifies rsync mirror health by comparing random file checksums between servers. # Catches silent corruption or incomplete syncs that rsync itself wouldn't detect. -# Leave BACKUP_VERIFY_SHARES empty to use HOST*_DAILY_SYNC_SHARES automatically. - BACKUP_VERIFY_SHARES=( - # leave empty to use HOST*_DAILY_SYNC_SHARES automatically - ) +# Defined per host in master_host*.conf — leave empty to use HOST*_DAILY_SYNC_SHARES automatically: +# HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES BACKUP_VERIFY_SAMPLE=10 # random files to check per share BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample @@ -1234,29 +979,18 @@ # (hot/max/hotssd/maxssd) — these vars are fallback only if dynamix.cfg not found. SMART_TEMP_WARN=45 # fallback — Celsius warn threshold SMART_TEMP_CRIT=55 # fallback — Celsius critical threshold - SMART_IGNORE_DRIVES=( - "sda" # boot USB — SMART not meaningful on flash drives - ) +# Drives to ignore defined per host in master_host*.conf — hardware is server-specific: +# HOST1_SMART_IGNORE_DRIVES / HOST2_SMART_IGNORE_DRIVES # ━━━ ZFS Memory Snapshot ━━━ # Weekly ZFS pool health and memory diagnostic report — informational only. -# ZFS_REPORT_ARC_WARN_PCT = warn if ARC is using more than this % of its max -# ZFS_REPORT_FREE_WARN_GB = warn if less than this GB free RAM -# ZFS_REPORT_AVAIL_WARN_GB = warn if less than this GB available on ZFS pool -# ZFS_REPORT_DOCKER_TOP = how many top Docker containers to show by memory usage -# ZFS_REPORT_IGNORE_POOLS = individual disk pools to skip (unRAID array disks as ZFS) ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log" - ZFS_REPORT_ARC_WARN_PCT=90 - ZFS_REPORT_FREE_WARN_GB=10 - ZFS_REPORT_AVAIL_WARN_GB=20 - ZFS_REPORT_DOCKER_TOP=10 - ZFS_REPORT_IGNORE_POOLS=( - "disk5" - "disk6" - "disk8" - "disk9" - "disk10" - ) + ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC using more than this % of its max + ZFS_REPORT_FREE_WARN_GB=10 # warn if less than this GB free RAM + ZFS_REPORT_AVAIL_WARN_GB=20 # warn if less than this GB available on ZFS pool + ZFS_REPORT_DOCKER_TOP=10 # how many top Docker containers to show by memory +# Pool ignore list defined per host in master_host*.conf — pool names are server-specific: +# HOST1_ZFS_REPORT_IGNORE_POOLS / HOST2_ZFS_REPORT_IGNORE_POOLS # ━━━ Bandwidth Monitor ━━━ # Called automatically by rsync.sh after each sync — one bounded write per run. @@ -1265,8 +999,8 @@ BANDWIDTH_LOG_RETENTION=90 # days before old entries purged BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report -# Stats files — written by cleanup and recovery scripts, read by coffee report -# All in DATA_DIR — array always running when these are written +# Stats files — written by cleanup and recovery scripts, read by coffee report. +# All in DATA_DIR — array always running when these are written. ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats @@ -1285,88 +1019,145 @@ # ━━━ Emby Session Report ━━━ # Weekly Emby usage statistics via API — no persistent writes, queries fresh each run. -# URL and API key pulled from HOST1/HOST2_EMBY_URL and HOST1/HOST2_EMBY_API_KEY. - EMBY_REPORT_DAYS=7 # days to include in the report period - EMBY_REPORT_TOP_N=10 # number of top content items to show +# URL and API key pulled from HOST*_EMBY_URL and HOST*_EMBY_API_KEY in master_host*.conf. + EMBY_REPORT_DAYS=7 # days to include in the report period + EMBY_REPORT_TOP_N=10 # number of top content items to show # ============================================================================================== # ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Continuous system health monitoring — last line of defense before a crash. # Started by array_start.sh — runs until array stops. -# Re-sources Master.conf each cycle — config changes take effect on next cycle. -# Strike system: sustained threshold hits trigger reboot — single spikes ignored. -# Reboot loop protection: shuts down instead if reboot limit hit in rolling window. +# Re-sources all three conf files each cycle — config changes take effect on next cycle. +# +# ── THREE-TIER RESPONSE SYSTEM ──────────────────────────────────────────────────────────────── +# CRITICAL — bypass ALL strikes, reboot immediately +# Docker daemon down, rootfs 100%, kernel oops, FD exhaustion, /boot read-only +# +# URGENT — bypass strikes only when OOM confirms active crisis +# RAM < MEM_GB AND OOM kills >= OOM_LIMIT in this cycle → reboot NOW +# Without OOM confirmation → normal strike system +# +# STANDARD — strike system (N consecutive failures → reboot) +# RAM tiers, high load, CPU temp, zombies, /var/log, /tmp, containers +# +# ── RAM TIERS ───────────────────────────────────────────────────────────────────────────────── +# MEM_WARN_GB — warn + notify only (informational) +# MEM_SHUTDOWN_GB — stop non-essential containers, recover above MEM_RECOVER_GB +# MEM_GB — strike system → reboot (or bypass with OOM) +# +# ── CONTAINER SHUTDOWN ──────────────────────────────────────────────────────────────────────── +# At MEM_SHUTDOWN_GB: stop all containers NOT in MEM_SHUTDOWN_EXCLUDED list +# Excluded containers stay running — DNS, auth, Emby, Dispatcharr +# Stopped containers stay stopped until RAM recovers above MEM_RECOVER_GB +# Strike list applied — doesn't flip-flop every cycle # ━━━ State Files ━━━ - SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" # /tmp resets on reboot ✅ + SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" # /tmp — resets on reboot ✅ SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db" + SYS_WATCHDOG_OOM_FILE="/tmp/system_watchdog_oom.db" # /tmp — resets on reboot ✅ # ━━━ Strike and Reboot Loop Settings ━━━ -# Strike system: a check must fail this many consecutive cycles before action is taken -# Single spikes (one bad reading) are ignored — sustained problems trigger reboot +# Strike system: a check must fail this many consecutive cycles before action is taken. +# Single spikes (one bad reading) are ignored — sustained problems trigger reboot. SYS_WATCHDOG_STRIKE_LIMIT=2 # consecutive failures before reboot trigger -# How often checks run — 300s = 5 minutes -# At STRIKE_LIMIT=2 and INTERVAL=300: problem must persist 10min before reboot +# How often checks run — 300s = 5 minutes. +# At STRIKE_LIMIT=2 and INTERVAL=300: problem must persist 10min before reboot. SYSTEM_WATCHDOG_INTERVAL=300 -# Reboot loop protection — if system keeps rebooting something is seriously wrong -# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS → shutdown instead of reboot +# Reboot loop protection — if system keeps rebooting something is seriously wrong. +# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS → shutdown instead of reboot. SYS_WATCHDOG_REBOOT_LIMIT=3 SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 -# Heartbeat — proof of life logged periodically even when everything is healthy +# Heartbeat — proof of life logged periodically even when everything is healthy. SYSTEM_WATCHDOG_HEARTBEAT=true SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1 +# ━━━ RAM Tiers ━━━ +# Three-level RAM response — graduated action instead of single threshold. +# HOST1 has 128GB, HOST2 has 64GB — adjust accordingly. +# MEM_WARN_GB > MEM_SHUTDOWN_GB > MEM_GB always + SYS_WATCHDOG_MEM_WARN_GB=10 # warn + notify — informational only + SYS_WATCHDOG_MEM_SHUTDOWN_GB=6 # stop non-essential containers + SYS_WATCHDOG_MEM_GB=4 # strike system → reboot + SYS_WATCHDOG_MEM_RECOVER_GB=30 # RAM must recover above this before restarting containers + +# Containers excluded from RAM emergency shutdown. +# These stay running regardless of RAM pressure. +# DNS and auth must stay up, Emby and Dispatcharr for Live TV continuity. + SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED=( + "NginxProxyManager" # DNS / reverse proxy — internet access + "Authelia" # auth — without this nothing is accessible + "Mariadb" # Authelia dependency + "Redis" # Authelia dependency + "Emby" # media server — Live TV buffering + "Dispatcharr" # Live TV scheduler — loses state if stopped + ) + +# ━━━ OOM Bypass Settings ━━━ +# OOM bypass: if RAM is critically low AND kernel OOM kills exceed this threshold +# in a single cycle → bypass strike system and reboot immediately. +# Rate-based: kills per 5-minute cycle, not absolute count. +# Rationale: 1-2 kills = docker_watchdog handles it ✅ +# 3+ kills while RAM critical = system dying faster than watchdogs can heal ✅ + SYS_WATCHDOG_OOM_LIMIT=3 # OOM kills in one cycle to trigger bypass + # ━━━ Thresholds ━━━ -# Set at "about to become unstable" levels — not "things are a bit high" -# These should be high enough that normal operation never triggers them +# Set at "about to become unstable" — not "things are a bit high". -# rootfs (/) usage — when array is down rsync writes land on rootfs, fills rapidly - SYS_WATCHDOG_ROOTFS_PCT=95 +# rootfs (/) usage — two levels: strike at 95%, critical bypass at 99% + SYS_WATCHDOG_ROOTFS_PCT=95 # standard strike threshold + SYS_WATCHDOG_ROOTFS_CRITICAL_PCT=99 # bypass strikes — truly full, writes failing -# /var/log usage — log spam can fill rootfs, indicates something broken +# /var/log usage — log spam indicates something broken SYS_WATCHDOG_LOG_PCT=95 -# Free RAM in GB — below this is critically low, OOM or swap imminent - SYS_WATCHDOG_MEM_GB=4 +# /tmp usage — tmpfs fills from downloads, lock files fail at 100% + SYS_WATCHDOG_TMP_PCT=90 # warn + attempt clear + SYS_WATCHDOG_TMP_CRITICAL_PCT=98 # bypass strikes if clear failed # ZFS ARC pinned percentage — ARC not releasing after reclaim = memory stuck SYS_WATCHDOG_ARC_PINNED_PCT=98 SYS_WATCHDOG_ARC_RELEASE_PCT=95 # Load average multiplier — threshold = MULTIPLIER × CPU core count -# MULTIPLIER=3 on 16-core = load average of 48 before triggering +# MULTIPLIER=3 on 16-core = load of 48 before triggering SYS_WATCHDOG_LOAD_MULTIPLIER=3 -# Zombie process count — large numbers indicate serious process management failure +# Zombie process count — large numbers = serious process management failure SYS_WATCHDOG_ZOMBIE_LIMIT=50 # CPU temperature — sustained high temp causes throttling or kernel panic SYS_WATCHDOG_CPU_TEMP_MAX=95 +# File descriptor exhaustion — system-wide FD limit near exhaustion +# New connections fail silently, Docker can't spawn processes, SSH fails + SYS_WATCHDOG_FD_CRITICAL_PCT=95 # bypass strikes — critical tier + +# Runaway process — single non-container process consuming excessive CPU +# Multiple strikes before action — single spikes are normal + SYS_WATCHDOG_RUNAWAY_CPU_PCT=90 # % single process must sustain + SYS_WATCHDOG_RUNAWAY_STRIKES=3 # consecutive cycles before warning + +# Array disk errors — accumulating mdstat errors = disk failing NOW + SYS_WATCHDOG_MDSTAT_ERROR_LIMIT=5 # new errors in one cycle before acting + # ━━━ Check Toggles ━━━ -# Disable individual checks without disabling the whole watchdog - SYS_WATCHDOG_CHECK_ROOTFS=true - SYS_WATCHDOG_CHECK_LOG=true - SYS_WATCHDOG_CHECK_RAM=true - SYS_WATCHDOG_CHECK_ARC=true - SYS_WATCHDOG_CHECK_CPU_TEMP=true - SYS_WATCHDOG_CHECK_LOAD=false # disabled — load spikes during transcoding are normal - SYS_WATCHDOG_CHECK_ZOMBIES=true - SYS_WATCHDOG_CHECK_CONTAINERS=true # checks docker_watchdog persistent skip list - SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true +# Per-host — moved to master_host*.conf +# Different servers may have different hardware, NICs, and check requirements +# See HOST*_SYS_WATCHDOG_CHECK_* in master_host*.conf # ━━━ Abort Toggles ━━━ -# Conditions that prevent reboot even when a threshold is hit -# true = abort reboot if this condition is active (conservative — avoid data loss) -# false = reboot anyway (aggressive — a clean reboot beats a hard crash) +# Conditions that prevent reboot even when a threshold is hit. +# CRITICAL tier bypasses these — a truly critical condition reboots regardless. +# true = abort standard reboot if this condition is active +# false = reboot anyway SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # rebooting with bad pool risks data loss - SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity is better than crashing mid-check - SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting move is better than crashing mid-move + SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity better than crashing mid-check + SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting move better than crashing mid-move # ============================================================================================== # ──────────────────────── End Of User Variables ─────────────────────────────────────────────── diff --git a/master_host1.conf b/master_host1.conf new file mode 100644 index 0000000..3d1582e --- /dev/null +++ b/master_host1.conf @@ -0,0 +1,646 @@ +#!/bin/bash +# ============================================================================================== +# ========================== HOST1 CONFIGURATION — unRAID-Gmer4Lfe ============================ +# ============================================================================================== +# HOST1-specific variables — credentials, container names, share paths, failover lists. +# Sourced after master.conf — values here extend shared profile arrays and add HOST1-specific +# identity, credentials, and container configuration. +# +# Sparse checkout (git) ensures HOST2 never receives this file. +# HOST2 never sees HOST1 credentials — clean separation at the file level. +# +# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf. +# DO NOT put HOST2 variables here — they belong in master_host2.conf. +# +# ── INDEX ───────────────────────────────────────────────────────────────────────────────────── +# +# ── IDENTITY & CONNECTIVITY ──────────────────────────────────────────────────────────────── +# IDENTITY hostname, SSH key +# EMBY container name, URL, API key +# NOTIFICATIONS Discord webhook +# PARTNERSHIP auth containers, backup paths +# +# ── RSYNC ────────────────────────────────────────────────────────────────────────────────── +# DAILY SYNC SHARES media shares HOST1 owns and pushes to HOST2 +# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am) +# CRITICAL SYNC SHARES appdata shares synced every 15 minutes +# BACKUP VERIFY shares for checksum verification against remote +# HOST1 RSYNC PROFILE host1-appdata profile for HOST1-specific appdata syncs +# +# ── DOCKER ───────────────────────────────────────────────────────────────────────────────── +# DOCKER DAILY RESTART containers restarted daily +# DOCKER WEEKLY RESTART containers restarted weekly +# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list +# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh +# +# ── FAILOVER ─────────────────────────────────────────────────────────────────────────────── +# DDNS DDNS containers managed by HOST1 +# INTERNET LOSS containers stopped when internet is lost +# FAILOVER TIERS what HOST1 runs for HOST2 per tier +# TIER DELAYS how long HOST1 must be down before each tier activates on HOST2 +# RSYNC WRITEBACK HOST1 appdata synced back on handback +# +# ── MEDIA ────────────────────────────────────────────────────────────────────────────────── +# MEDIA PERMISSIONS share list for media_shares_permissions.sh +# MEDIA CLEANER folder lists for media_cleaner.sh +# +# ── MONITORS ─────────────────────────────────────────────────────────────────────────────── +# CERTIFICATE MONITOR domains checked for SSL expiry +# SMART HEALTH drives to skip in SMART monitoring +# ZFS REPORT pools to exclude from ZFS health report +# +# ── TRANSCODES ───────────────────────────────────────────────────────────────────────────── +# TRANSCODES ramdisk size, thresholds, SSD path, server array +# +# ── ARR STACK ────────────────────────────────────────────────────────────────────────────── +# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs +# LIDARR URL, API key, path map +# SONARR URL, API key, path map +# RADARR URL, API key, path map +# ARR RECOVERY per-arr recovery toggles +# +# ============================================================================================== + +# ============================================================================================== +# ── IDENTITY & CONNECTIVITY ─────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Identity ━━━ +# Hostname must match exact unRAID hostname AND Tailscale device name — case sensitive. +# Used by detect_hosts() in common.sh to identify this server as HOST1. + HOST1="unRAID-Gmer4Lfe" + +# SSH key used for all server-to-server operations — rsync, failover container commands. +# Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys. + HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" + +# ━━━ Emby ━━━ +# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh, +# weekly_sync_maintenance.sh, and HOST1_TRANSCODE_SERVERS below. +# API key: Emby Dashboard → API Keys → + New Key + HOST1_EMBY_CONTAINER="Emby" + HOST1_EMBY_URL="http://localhost:8096" + HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829" + +# ━━━ Notifications ━━━ +# Discord webhook — leave blank to disable. +# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies. + HOST1_DISCORD_WEBHOOK="" + +# ━━━ Partnership ━━━ +# HOST1 is always the owner (source of truth) unless --transfer has been run. +# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs. + +# Auth containers reconfigured on onboard/offboard. +# Format: "ContainerName|WebUIPort" +# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth) +# On offboard → WebUI pointed back at localhost + HOST1_PARTNERSHIP_AUTH_WEBUIS=( + "NginxProxyManager|81" + "Lldap-Gmer4Lfe|17170" + "Authelia|9091" + "Authelia-Secondary|9092" + ) + +# Paths HOST2 should collect during the grace window after offboard. +# Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS. + HOST1_PARTNERSHIP_MIRROR_BACKUPS=( + # "/mnt/user/appdata-Failover/Jayred365-Emby" + ) + +# ============================================================================================== +# ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Daily Sync Shares ━━━ +# Shares HOST1 owns and pushes to HOST2 every night (1am via daily_sync_maintenance.sh). +# HOST1 is the source of truth — HOST2 is the mirror. +# Never push a share both directions — one server always owns it. +# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed. +# For shares needing container stops or custom options — add a profile in master.conf. + HOST1_DAILY_SYNC_SHARES=( + /mnt/user/Books + /mnt/user/Intros + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movies + /mnt/user/Music + /mnt/user/Music_Videos + /mnt/user/Nextcloud + /mnt/user/stand-up_comedy + /mnt/user/Sports + /mnt/user/Tv_Shows + /mnt/user/Anime_Shows-Old + /mnt/user/Anime_Movies-Old + ) + +# Personal encrypted shares — synced for offsite backup, independent of media shares. +# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content. +# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting. + HOST1_PERSONAL_SHARES=( + # /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset + ) + +# ━━━ Weekly Sync Shares ━━━ +# Appdata shares synced during the weekly maintenance window (Sunday 2:30am). +# Containers stopped both sides before sync — full clean state guaranteed. +# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section. +# Order matters — Emby first (larger transfer), then Critical-Data (auth stack). + HOST1_WEEKLY_SYNC_SHARES=( + "/mnt/user/Media_Server/Emby" # emby profile — full clean mirror + "/mnt/user/appdata-Failover/Critical-Data" # critical-data profile — auth stack + ) + +# ━━━ Critical Sync Shares ━━━ +# Appdata shares synced every 15 minutes by critical_sync_maintenance.sh. +# Format: "/path/to/share" or "/path/to/share|profile-name" +# Order matters — Critical-Data first (auth stack), then Emby dirty sync. + HOST1_CRITICAL_SYNC_SHARES=( + "/mnt/user/appdata-Failover/Critical-Data|critical-failover" # auth dirty sync — stays running + "/mnt/user/Media_Server/Emby|emby-failover" # Emby dirty sync — stays running + ) + +# ━━━ Backup Verify ━━━ +# Shares verified by backup_verify.sh — random file checksum comparison against remote. +# Leave empty to use HOST1_DAILY_SYNC_SHARES automatically. +# Sample size and minimum file size defined in master.conf. + HOST1_BACKUP_VERIFY_SHARES=( + # leave empty to use HOST1_DAILY_SYNC_SHARES automatically + ) + +# ━━━ HOST1 Rsync Profile — host1-appdata ━━━ +# HOST1-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf. +# Use for appdata unique to HOST1 (Organizrv2, VaultWarden, UptimeKuma etc.) +# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf. +# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Failover/HOST1-Appdata --profile=host1-appdata + PROFILE_RSYNC_OPTS[host1-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host1-appdata]:-8000}" + PROFILE_BW_LIMIT[host1-appdata]=8000 + PROFILE_RETRY_COUNT[host1-appdata]=3 + PROFILE_SLEEP[host1-appdata]=300 + PROFILE_CRITICAL_CONTAINER_NAMES[host1-appdata]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe" + PROFILE_DELAYED_CONTAINERS[host1-appdata]="" + PROFILE_CONTAINER_DELAY[host1-appdata]=5 + PROFILE_EXCLUDE_DIRS[host1-appdata]="logs *.tmp" + +# ============================================================================================== +# ── DOCKER ──────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Docker Daily Restart ━━━ +# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS. +# Dispatcharr degrades over time without restart — daily is intentional, not just housekeeping. +# Order matters — auth stack first, then media services. + HOST1_DAILY_RESTART_CONTAINERS=( + "NginxProxyManager" + "Lldap-Gmer4Lfe" + "Authelia" + "Authelia-Secondary" + "Dispatcharr-Iptv-Users" + "Dispatcharr" # Live TV scheduler — degrades without daily restart + "Dispatcharr-Basic" + "ErsatzTV-Emby" + ) + +# ━━━ Docker Weekly Restart ━━━ +# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am). +# Containers already stopped for weekly sync — restart adds zero extra downtime. + HOST1_WEEKLY_RESTART_CONTAINERS=( + "NextCloud" + "Organizrv2-Gmer4Lfe" + "AdGuard-Home" + "Immich-Gmer4Lfe" + ) + +# ━━━ Docker Watchdog ━━━ +# Per-HOST1 container configuration for docker_watchdog.sh. +# Shared thresholds and toggles live in master.conf. + +# Memory hard limits in MB — immediate restart if exceeded. +# Set at "container is clearly broken" not "container is busy". +# 20GB=20480 18GB=18432 16GB=16384 12GB=12288 8GB=8192 4GB=4096 2GB=2048 1GB=1024 + declare -A HOST1_WATCHDOG_CONTAINERS=( + ["Emby"]=18432 # 18GB — large library + active transcodes + ["LidaTube"]=6144 # 6GB — memory leak over time + ["Tdarr"]=6144 # 6GB — encoding is memory intensive + ["Code-Server"]=1024 # 1GB — should never need more + ) + +# HTTP health check URLs — checked every cycle, strike system before restart. +# Only add containers with a meaningful web interface to check. + declare -A HOST1_WATCHDOG_CONTAINER_URLS=( + ["Emby"]="http://localhost:8096" + ) + +# Required containers — must always be running on HOST1. +# Strike system before restart — repeated failures go on skip list, auto-clears on recovery. +# Listed in dependency order — dependencies before dependents. + HOST1_WATCHDOG_REQUIRED_CONTAINERS=( + "NginxProxyManager" + "Lldap-Gmer4Lfe" + "Mariadb-Authelia" + "Mariadb-Authelia-Secondary" + "Redis-Authelia" + "Redis-Authelia-Secondary" + "Authelia" + "Authelia-Secondary" + ) + +# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting. +# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking. + HOST1_WATCHDOG_SCAN_IGNORE=( + "DashGate" + "PIA-WG-Config-Generator" + "Aperture" + "Aperture-Kids" + "pgvector-18-Apeture-Kids" + "Pgvector18-Aperture" + ) + +# Dependency ordering — skip restarting a container if its dependency is also down. +# Prevents watchdog from restarting Authelia before Mariadb is back up. +# SPACE-SEPARATED STRINGS — converted to array at runtime. + declare -A HOST1_WATCHDOG_DEPENDENCIES=( + ["Authelia"]="Mariadb-Authelia Redis-Authelia" + ["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary" + ["NextCloud"]="Postgres-NextCloud" + ) + +# ━━━ Docker Network Connect ━━━ +# Containers connected to custom networks at array start by docker_network_connect.sh. +# Networks created if they don't exist — idempotent, safe to re-run. + HOST1_NETWORK_CONNECT_CONTAINERS=( + "memcached" + "Npm-CrowdSec" + ) + + HOST1_NETWORK_CONNECT_NETWORKS=( + "high-availability" + ) + +# ============================================================================================== +# ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ DDNS ━━━ +# DDNS containers HOST1 manages — started/stopped by failover.sh per DDNS absolute rules: +# Internet loss → stop immediately +# Failover → HOST2 starts HOST1's DDNS as Tier 1 (before any other containers) +# Handback → stop HOST1's DDNS on HOST2 → rsync → start containers → start local DDNS last + HOST1_DDNS_CONTAINERS=( + "Gmer4Lfe.com" + ) + +# ━━━ Internet Loss ━━━ +# Containers stopped immediately on HOST1 when internet connection is lost. +# Prevents external-facing services from operating without connectivity. + FAILOVER_HOST1_STOP_ON_NO_NET=( + "Gmer4Lfe.com" + ) + +# ━━━ Failover Tiers — HOST1 Runs for HOST2 ━━━ +# Containers HOST1 starts when HOST2 goes down. +# Tier 1 is always immediate — vital services cannot wait. +# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in master_host2.conf). + FAILOVER_HOST1_RUNS_FOR_HOST2_TIER1=( + "Gmer4Lfe.us" + "VaultWarden-Jayred365" + ) + + FAILOVER_HOST1_RUNS_FOR_HOST2_TIER2=( + # "container-placeholder" + ) + + FAILOVER_HOST1_RUNS_FOR_HOST2_TIER3=( + # "container-placeholder" + ) + + FAILOVER_HOST1_RUNS_FOR_HOST2_TIER4=( + # "container-placeholder" + ) + +# ━━━ Tier Delays — HOST1's Containers on HOST2 ━━━ +# How long HOST1 must be down before each tier activates on HOST2 — in minutes. +# Tier 1 is always immediate — no delay var needed. + HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich + HOST1_TIER3_DELAY=720 # 12 hours — secondary services + HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders + +# ━━━ Rsync Writeback — HOST1 Appdata Back on Handback ━━━ +# Syncs HOST1 appdata BACK to HOST1 when it comes back online after a failover. +# Containers stopped before writeback — clean source, no competing writes. +# +# HOST1_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state +# is more reliable than dirty sync data for brief outages. + HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr + +# Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES — only list paths NOT in that array. + FAILOVER_HOST1_WRITEBACK_TIER1=( + "/mnt/user/Media_Server/Emby" # watch states built up during outage + ) + + FAILOVER_HOST1_WRITEBACK_TIER2=( + "/mnt/user/appdata-Failover/Important-Data" # NextCloud + Postgres — files added during outage + ) + + FAILOVER_HOST1_WRITEBACK_TIER3=( + # "location-placeholder" + ) + + FAILOVER_HOST1_WRITEBACK_TIER4=( + "/mnt/user/appdata-Failover/Arrs_Stack" # arr databases — downloads queued during outage + ) + +# ============================================================================================== +# ── MEDIA ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Media Permissions ━━━ +# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to. +# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership. + HOST1_MEDIA_PERMISSION_SHARES=( + /mnt/user/Anime_Movies + /mnt/user/Anime_Movies-Old + /mnt/user/Anime_Shows + /mnt/user/Anime_Shows-Old + /mnt/user/appcache + /mnt/user/Books + /mnt/user/Downloads + /mnt/user/Games + /mnt/user/Intros + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movie_Recordings + /mnt/user/Movies + /mnt/user/Music + /mnt/user/Music_Videos + /mnt/user/Photo + /mnt/user/Sports + /mnt/user/stand-up_comedy + /mnt/user/Temp_Storage + /mnt/user/Tv_Recordings + /mnt/user/Tv_Shows + /mnt/user/YouTube + ) + +# ━━━ Media Cleaner ━━━ +# Folder lists for media_cleaner.sh — two profiles: anime and media. +# File patterns shared across all servers — defined in master.conf. +# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media + HOST1_ANIME_CLEAN_FOLDERS=( + /mnt/user/Anime_Movies + /mnt/user/Anime_Movies-Old + /mnt/user/Anime_Shows + /mnt/user/Anime_Shows-Old + ) + + HOST1_MEDIA_CLEAN_FOLDERS=( + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movies + /mnt/user/Music + /mnt/user/Sports + /mnt/user/stand-up_comedy + /mnt/user/Tv_Shows + ) + +# ============================================================================================== +# ── MONITORS ────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Certificate Monitor ━━━ +# Domains checked via direct openssl connection — not relying on NPM's certificate state. +# Checks the actual certificate served, not what NPM thinks it has. +# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf. + HOST1_CERT_MONITOR_DOMAINS=( + "Gmer4Lfe.com" + "Gmer4Lfe.us" + ) + +# ━━━ SMART Health ━━━ +# Drives skipped in SMART attribute monitoring — hardware is server-specific. +# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf. + HOST1_SMART_IGNORE_DRIVES=( + "sda" # boot USB — SMART not meaningful on flash drives + ) + +# ━━━ ZFS Report ━━━ +# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools. +# These are individual array disks formatted as ZFS — converting to XFS over time via unBalance. +# Pool health thresholds defined in master.conf. + HOST1_ZFS_REPORT_IGNORE_POOLS=( + "disk5" + "disk6" + "disk8" + "disk9" + "disk10" + ) + +# ============================================================================================== +# ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront. +# Real-world: 9 streams peaked at ~5.5GB — 8G gives comfortable headroom on 128GB RAM. + HOST1_RAMDISK_SIZE="8G" + +# Usage thresholds — coupled to HOST1_RAMDISK_SIZE, adjust all three together if size changes. +# Hysteresis gap (6.8 - 5.5 = 1.3GB) prevents flip-flop between ramdisk and SSD. + HOST1_RAMDISK_WARN_GB=6.8 # flip to SSD when ramdisk usage reaches this + HOST1_RAMDISK_LOW_GB=5.5 # flip back to ramdisk when usage drops to this + +# SSD fallback path — where transcodes land when ramdisk exceeds HOST1_RAMDISK_WARN_GB. +# Must be on cache pool — array disks too slow for active transcode writes. + HOST1_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" + +# Media servers sharing the ramdisk transcode space on HOST1. +# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex +# Entries with placeholder API keys are skipped automatically. +# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk. + HOST1_TRANSCODE_SERVERS=( + "${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby" + ) + +# ============================================================================================== +# ── ARR STACK ───────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh. +# detect_hosts() selects HOST1 vars when running on HOST1. +# +# PATH MAPS — container path → host path translation. +# Arr stores file paths using container-internal paths — scripts need host paths to scan. +# Add one entry per root folder in arr Settings → Media Management → Root Folders. + +# ━━━ Downloaders ━━━ +# Used by downloaders_reset.sh — runs every 15min via CRITICAL_MAINTENANCE_SCRIPTS. +# Clears stuck states, purges old history, prepares each client for a clean cycle. + +# slskd — clears stuck searches, dead transfers, purges expired failed imports. +# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected. + HOST1_SLSKD_URL="http://localhost:8980" + HOST1_SLSKD_API_KEY="4bF9kL2mNpQrT7vWxYz1A3dEgHjKoRsU" + HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports" + +# SABnzbd + HOST1_SABNZBD_URL="http://localhost:8180" + HOST1_SABNZBD_API_KEY="8bfefe41d83b4d50883e32859b55ca9a" + +# qBittorrent — deleteFiles=false removes torrent from qBit but leaves files on disk. +# Radarr/Sonarr manage actual files independently. + HOST1_QBIT_URL="http://localhost:8080" + HOST1_QBIT_USERNAME="root" + HOST1_QBIT_PASSWORD="Stay0utD!ck" + +# ━━━ Lidarr — HOST1 only ━━━ +# HOST2 does not run Lidarr — HOST1_LIDARR_RECOVERY flag handles the exit cleanly. + HOST1_LIDARR_URL="http://localhost:8686" + HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc" + HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New" + + declare -A HOST1_LIDARR_PATH_MAP=( + ["/ext-music"]="/mnt/user/Music-New" + ) + +# ━━━ Sonarr ━━━ + HOST1_SONARR_URL="http://localhost:8989" + HOST1_SONARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" + HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows" + +# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder + declare -A HOST1_SONARR_PATH_MAP=( + ["/tv"]="/mnt/user/Tv_Shows" + ["/ext-standup-comedy"]="/mnt/user/stand-up_comedy" + ["/kids tv"]="/mnt/user/Kids_Tv_Shows" + ["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old" + ) + +# ━━━ Radarr ━━━ + HOST1_RADARR_URL="http://localhost:7878" + HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" + HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies" + +# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder + declare -A HOST1_RADARR_PATH_MAP=( + ["/movies"]="/mnt/user/Movies" + ["/kids movies"]="/mnt/user/Kids_Movies" + ["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy" + ["/anime-movies"]="/mnt/user/Anime_Movies-Old" + ) + +# ━━━ Arr Recovery Toggles ━━━ +# false = skip that arr on this host — exits cleanly without error + HOST1_SONARR_RECOVERY=true + HOST1_RADARR_RECOVERY=true + HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2 + +# ============================================================================================== +# ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Per-host check toggles and NIC config for system_watchdog.sh. +# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names. +# HOST1: TR1950X 128GB — full media server, active transcoding, ZFS cache pools. +# +# Three-tier response — all critical checks enabled by default on HOST1: +# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot +# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle +# Tier 3 (standard strike system): everything else +# +# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section. + +# ━━━ Primary NIC ━━━ +# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]" +# Common values: eth0, bond0, br0, eno1 + HOST1_SYS_WATCHDOG_NIC="eth0" + +# ━━━ Tier 1 — Critical Checks ━━━ +# These bypass the strike system — a single hit triggers immediate reboot. +# Disabling any of these is not recommended — they protect against acute system failure. + +# Docker daemon unresponsive → try restart, reboot if restart fails. +# Without a working daemon docker_watchdog.sh is blind and containers cannot be managed. + HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true + +# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately. +# At 99% rootfs writes fail silently — logs stop, Docker errors out, SSH may stop working. +# Standard 95% threshold still uses strike system — only 99%+ is critical tier. + HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true + +# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately. +# A kernel oops means the kernel ran with a corrupted state — stability is not guaranteed. + HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true + +# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately. +# At 95% FD: new connections fail, Docker can't spawn processes, SSH drops. + HOST1_SYS_WATCHDOG_CHECK_FD=true + +# /boot read-only detected → reboot immediately. +# Unexpected read-only /boot means state files and config writes are silently failing. +# Failover state, watchdog reboot log, and lock files all go stale silently. + HOST1_SYS_WATCHDOG_CHECK_BOOT=true + +# ━━━ Tier 2 — Urgent OOM Check ━━━ +# Bypass strikes when RAM is critically low AND OOM kill rate confirms active crisis. +# Both must be enabled for Tier 2 bypass to function — disable either to always use strikes. + +# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta. +# Also provides diagnostic context in reboot messages (which processes were killed). + HOST1_SYS_WATCHDOG_CHECK_OOM=true + +# Free RAM check — required for both Tier 2 bypass and RAM tier logic. +# Tiers: MEM_WARN_GB(10) → notify | MEM_SHUTDOWN_GB(6) → stop containers | MEM_GB(4) → strikes + HOST1_SYS_WATCHDOG_CHECK_RAM=true + +# ━━━ Tier 3 — Standard Checks (strike system) ━━━ +# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action is taken. +# Single spikes are ignored — sustained problems trigger reboot. + +# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT. +# Log spam (Docker log storms, syslog loops) fills rootfs — indicates something broken. + HOST1_SYS_WATCHDOG_CHECK_LOG=true + +# ZFS ARC memory pinned above SYS_WATCHDOG_ARC_PINNED_PCT after cache drop. +# Enabled on HOST1 — ZFS cache pools actively used. Disable on hosts without ZFS. + HOST1_SYS_WATCHDOG_CHECK_ARC=true + +# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C). +# Sustained high temp causes kernel throttling or panic. Requires lm-sensors. + HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true + +# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count. +# DISABLED on HOST1 — Tdarr and Emby cause legitimate sustained load spikes during encoding. +# Enable on idle servers or adjust SYS_WATCHDOG_LOAD_MULTIPLIER if load is always high. + HOST1_SYS_WATCHDOG_CHECK_LOAD=false + +# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50). +# Large zombie counts indicate serious process management failure — something is stuck. + HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true + +# Check docker_watchdog.sh persistent skip list — required containers on skip list. +# Cross-watchdog coordination: if docker_watchdog gave up, system_watchdog escalates. +# ENABLED — HOST1 fully built and operational, skip list is meaningful. + HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true + +# /tmp filesystem usage above SYS_WATCHDOG_TMP_PCT with auto-clear attempt. +# Script tries to clear aged /tmp files first — only strikes if clear fails. +# Lock files, rsync temp files, and Docker ops use /tmp — 100% means lock failures. + HOST1_SYS_WATCHDOG_CHECK_TMP=true + +# Array disk error count delta in /proc/mdstat — accumulating errors = disk failing now. +# Triggers on SYS_WATCHDOG_MDSTAT_ERROR_LIMIT new errors in one cycle. + HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true + +# Primary NIC operstate — detects NIC going down (physical or driver failure). +# Uses HOST1_SYS_WATCHDOG_NIC above. Strike system — brief flaps don't trigger reboot. + HOST1_SYS_WATCHDOG_CHECK_NETWORK=true + +# sshd running check — attempts restart before escalating. +# sshd down = no remote access. Script tries rc.sshd start, notifies, strikes on failure. + HOST1_SYS_WATCHDOG_CHECK_SSHD=true + +# Runaway process detection — single process above SYS_WATCHDOG_RUNAWAY_CPU_PCT sustained. +# DISABLED — Tdarr encoding and Emby transcoding legitimately peg CPU for extended periods. +# Enable only if HOST1 has no CPU-intensive workloads. + HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false + +# ============================================================================================== +# ──────────────────────── End Of HOST1 Variables ────────────────────────────────────────────── +# ============================================================================================== \ No newline at end of file diff --git a/master_host2.conf b/master_host2.conf new file mode 100644 index 0000000..3846a0b --- /dev/null +++ b/master_host2.conf @@ -0,0 +1,545 @@ +#!/bin/bash +# ============================================================================================== +# ========================== HOST2 CONFIGURATION — unRAID-Jayred365 =========================== +# ============================================================================================== +# HOST2-specific variables — credentials, container names, share paths, failover lists. +# Sourced after master.conf — values here extend shared profile arrays and add HOST2-specific +# identity, credentials, and container configuration. +# +# Sparse checkout (git) ensures HOST1 never receives this file. +# HOST1 never sees HOST2 credentials — clean separation at the file level. +# +# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf. +# DO NOT put HOST1 variables here — they belong in master_host1.conf. +# +# ── STATUS ──────────────────────────────────────────────────────────────────────────────────── +# HOST2 is currently being rebuilt — most sections scaffolded, fill in when back online. +# When ready: set FAILOVER_ENABLED=true and DAILY_RSYNC_ENABLED=true in master.conf. +# +# ── INDEX ───────────────────────────────────────────────────────────────────────────────────── +# +# ── IDENTITY & CONNECTIVITY ──────────────────────────────────────────────────────────────── +# IDENTITY hostname, SSH key +# EMBY container name, URL, API key +# NOTIFICATIONS Discord webhook +# PARTNERSHIP auth containers, backup paths +# +# ── RSYNC ────────────────────────────────────────────────────────────────────────────────── +# DAILY SYNC SHARES media shares HOST2 owns and pushes to HOST1 +# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am) +# CRITICAL SYNC SHARES appdata shares synced every 15 minutes +# BACKUP VERIFY shares for checksum verification against remote +# HOST2 RSYNC PROFILE host2-appdata profile for HOST2-specific appdata syncs +# +# ── DOCKER ───────────────────────────────────────────────────────────────────────────────── +# DOCKER DAILY RESTART containers restarted daily +# DOCKER WEEKLY RESTART containers restarted weekly +# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list +# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh +# +# ── FAILOVER ─────────────────────────────────────────────────────────────────────────────── +# DDNS DDNS containers managed by HOST2 +# INTERNET LOSS containers stopped when internet is lost +# FAILOVER TIERS what HOST2 runs for HOST1 per tier +# TIER DELAYS how long HOST2 must be down before each tier activates on HOST1 +# RSYNC WRITEBACK HOST2 appdata synced back on handback +# +# ── MEDIA ────────────────────────────────────────────────────────────────────────────────── +# MEDIA PERMISSIONS share list for media_shares_permissions.sh +# MEDIA CLEANER folder lists for media_cleaner.sh +# +# ── MONITORS ─────────────────────────────────────────────────────────────────────────────── +# CERTIFICATE MONITOR domains checked for SSL expiry +# SMART HEALTH drives to skip in SMART monitoring +# ZFS REPORT pools to exclude from ZFS health report +# +# ── TRANSCODES ───────────────────────────────────────────────────────────────────────────── +# TRANSCODES ramdisk size, thresholds, SSD path, server array +# +# ── ARR STACK ────────────────────────────────────────────────────────────────────────────── +# SONARR URL, API key, path map +# RADARR URL, API key, path map +# ARR RECOVERY per-arr recovery toggles (no Lidarr on HOST2) +# +# ============================================================================================== + +# ============================================================================================== +# ── IDENTITY & CONNECTIVITY ─────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Identity ━━━ +# Hostname must match exact unRAID hostname AND Tailscale device name — case sensitive. +# Used by detect_hosts() in common.sh to identify this server as HOST2. + HOST2="unRAID-Jayred365" + +# SSH key used for all server-to-server operations — rsync, failover container commands. +# Must be in /root/.ssh/ and authorised in HOST1's /root/.ssh/authorized_keys. + HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" + +# ━━━ Emby ━━━ +# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh, +# weekly_sync_maintenance.sh, and HOST2_TRANSCODE_SERVERS below. +# API key: Emby Dashboard → API Keys → + New Key + HOST2_EMBY_CONTAINER="Emby-Jayred365" + HOST2_EMBY_URL="http://localhost:8096" + HOST2_EMBY_API_KEY="your-host2-emby-api-key" + +# ━━━ Notifications ━━━ +# Discord webhook — leave blank to disable. +# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies. + HOST2_DISCORD_WEBHOOK="" + +# ━━━ Partnership ━━━ +# HOST2 is the mirror — HOST1 is always the owner unless --transfer has been run. +# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs. + +# Auth containers reconfigured on onboard/offboard. +# Format: "ContainerName|WebUIPort" +# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth) +# On offboard → WebUI pointed back at localhost + HOST2_PARTNERSHIP_AUTH_WEBUIS=( + # fill in when HOST2 is back online + # "NginxProxyManager|81" + ) + +# Paths HOST1 should collect during the grace window after offboard. +# Notified on offboard — no auto-deletion, HOST1 must collect manually within PARTNERSHIP_GRACE_HOURS. + HOST2_PARTNERSHIP_MIRROR_BACKUPS=( + # fill in when HOST2 is back online + ) + +# ============================================================================================== +# ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Daily Sync Shares ━━━ +# Shares HOST2 owns and pushes to HOST1 every night (1am via daily_sync_maintenance.sh). +# HOST2 is the source of truth — HOST1 is the mirror. +# Never push a share both directions — one server always owns it. +# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed. +# For shares needing container stops or custom options — add a profile in master.conf. + HOST2_DAILY_SYNC_SHARES=( + /mnt/user/Anime_Movies + /mnt/user/Anime_Shows + ) + +# Personal encrypted shares — synced for offsite backup, independent of media shares. +# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content. +# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting. + HOST2_PERSONAL_SHARES=( + # /mnt/user/HOST2-Personal # uncomment after creating encrypted dataset + ) + +# ━━━ Weekly Sync Shares ━━━ +# Appdata shares synced during the weekly maintenance window (Sunday 2:30am). +# Containers stopped both sides before sync — full clean state guaranteed. +# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section. + HOST2_WEEKLY_SYNC_SHARES=( + # fill in when HOST2 is back online + # "/mnt/user/Media_Server/Emby" + # "/mnt/user/appdata-Failover/Critical-Data" + ) + +# ━━━ Critical Sync Shares ━━━ +# Appdata shares synced every 15 minutes by critical_sync_maintenance.sh. +# Format: "/path/to/share" or "/path/to/share|profile-name" + HOST2_CRITICAL_SYNC_SHARES=( + # fill in when HOST2 is back online + # "/mnt/user/appdata-Failover/Critical-Data|critical-failover" + # "/mnt/user/Media_Server/Emby|emby-failover" + ) + +# ━━━ Backup Verify ━━━ +# Shares verified by backup_verify.sh — random file checksum comparison against remote. +# Leave empty to use HOST2_DAILY_SYNC_SHARES automatically. +# Sample size and minimum file size defined in master.conf. + HOST2_BACKUP_VERIFY_SHARES=( + # leave empty to use HOST2_DAILY_SYNC_SHARES automatically + ) + +# ━━━ HOST2 Rsync Profile — host2-appdata ━━━ +# HOST2-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf. +# Use for appdata unique to HOST2. +# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf. +# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Failover/HOST2-Appdata --profile=host2-appdata + PROFILE_RSYNC_OPTS[host2-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host2-appdata]:-8000}" + PROFILE_BW_LIMIT[host2-appdata]=8000 + PROFILE_RETRY_COUNT[host2-appdata]=3 + PROFILE_SLEEP[host2-appdata]=300 + PROFILE_CRITICAL_CONTAINER_NAMES[host2-appdata]="" # fill in when HOST2 is back online + PROFILE_DELAYED_CONTAINERS[host2-appdata]="" + PROFILE_CONTAINER_DELAY[host2-appdata]=5 + PROFILE_EXCLUDE_DIRS[host2-appdata]="logs *.tmp" + +# ============================================================================================== +# ── DOCKER ──────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Docker Daily Restart ━━━ +# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS. +# Fill in when HOST2 is back online — add containers that degrade without daily restart. + HOST2_DAILY_RESTART_CONTAINERS=( + "NginxProxyManager" + # add HOST2 daily restart containers here + ) + +# ━━━ Docker Weekly Restart ━━━ +# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am). +# Containers already stopped for weekly sync — restart adds zero extra downtime. + HOST2_WEEKLY_RESTART_CONTAINERS=( + # add HOST2 weekly restart containers here + ) + +# ━━━ Docker Watchdog ━━━ +# Per-HOST2 container configuration for docker_watchdog.sh. +# Shared thresholds and toggles live in master.conf. + +# Memory hard limits in MB — immediate restart if exceeded. +# Set at "container is clearly broken" not "container is busy". +# 20GB=20480 16GB=16384 12GB=12288 10GB=10240 8GB=8192 4GB=4096 2GB=2048 1GB=1024 + declare -A HOST2_WATCHDOG_CONTAINERS=( + ["Emby"]=16384 # fill in correct limit when HOST2 is back online + ) + +# HTTP health check URLs — checked every cycle, strike system before restart. +# Only add containers with a meaningful web interface to check. + declare -A HOST2_WATCHDOG_CONTAINER_URLS=( + ["Emby"]="http://localhost:8096" + ) + +# Required containers — must always be running on HOST2. +# Strike system before restart — repeated failures go on skip list, auto-clears on recovery. +# Listed in dependency order — dependencies before dependents. + HOST2_WATCHDOG_REQUIRED_CONTAINERS=( + "NginxProxyManager" + # add HOST2 required containers here when back online + ) + +# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting. +# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking. + HOST2_WATCHDOG_SCAN_IGNORE=( + # add HOST2 scan ignore containers here when back online + ) + +# Dependency ordering — skip restarting a container if its dependency is also down. +# Prevents watchdog from restarting dependent services before their dependencies are up. +# SPACE-SEPARATED STRINGS — converted to array at runtime. + declare -A HOST2_WATCHDOG_DEPENDENCIES=( + # add HOST2 dependencies here when containers are defined + # ["Authelia"]="Mariadb-Authelia Redis-Authelia" + ) + +# ━━━ Docker Network Connect ━━━ +# Containers connected to custom networks at array start by docker_network_connect.sh. +# Networks created if they don't exist — idempotent, safe to re-run. + HOST2_NETWORK_CONNECT_CONTAINERS=( + # fill in when HOST2 is back online + ) + + HOST2_NETWORK_CONNECT_NETWORKS=( + # fill in when HOST2 is back online + ) + +# ============================================================================================== +# ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ DDNS ━━━ +# DDNS containers HOST2 manages — started/stopped by failover.sh per DDNS absolute rules: +# Internet loss → stop immediately +# Failover → HOST1 starts HOST2's DDNS as Tier 1 (before any other containers) +# Handback → stop HOST2's DDNS on HOST1 → rsync → start containers → start local DDNS last + HOST2_DDNS_CONTAINERS=( + "Gmer4Lfe.us" + ) + +# ━━━ Internet Loss ━━━ +# Containers stopped immediately on HOST2 when internet connection is lost. +# Prevents external-facing services from operating without connectivity. + FAILOVER_HOST2_STOP_ON_NO_NET=( + "Gmer4Lfe.us" + ) + +# ━━━ Failover Tiers — HOST2 Runs for HOST1 ━━━ +# Containers HOST2 starts when HOST1 goes down. +# Tier 1 is always immediate — vital services cannot wait. +# Higher tiers activate after HOST1_TIER*_DELAY minutes (set in master_host1.conf). + FAILOVER_HOST2_RUNS_FOR_HOST1_TIER1=( + "Gmer4Lfe.com" + "Emby" + "VaultWarden-Gmer4Lfe" + "Dispatcharr" + "Dispatcharr-Basic" + "Dispatcharr-Iptv-Users" + "ErsatzTV-Emby" + ) + + FAILOVER_HOST2_RUNS_FOR_HOST1_TIER2=( + "Postgres-NextCloud" + "NextCloud" + "PostgreSQL_Immich" + "Immich-Gmer4Lfe" + ) + + FAILOVER_HOST2_RUNS_FOR_HOST1_TIER3=( + "Gitea" + ) + + FAILOVER_HOST2_RUNS_FOR_HOST1_TIER4=( + "Sonarr" + "Radarr" + "Lidarr" + "Readarr" + "Prowlarr" + "Bazarr" + "SABnzbd-Gmer4Lfe" + "Qbittorrent-Gmer4Lfe" + "LidaTube" + "Pinchflat" + "ChannelTube" + ) + +# ━━━ Tier Delays — HOST2's Containers on HOST1 ━━━ +# How long HOST2 must be down before each tier activates on HOST1 — in minutes. +# Tier 1 is always immediate — no delay var needed. + HOST2_TIER2_DELAY=240 # 4 hours — productivity services + HOST2_TIER3_DELAY=720 # 12 hours — secondary services + HOST2_TIER4_DELAY=1440 # 24 hours — arrs + downloaders + +# ━━━ Rsync Writeback — HOST2 Appdata Back on Handback ━━━ +# Syncs HOST2 appdata BACK to HOST2 when it comes back online after a failover. +# Containers stopped before writeback — clean source, no competing writes. +# +# HOST2_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state +# is more reliable than dirty sync data for brief outages. + HOST2_TIER1_WRITEBACK_DELAY=60 # skip writeback if outage under 1hr + +# Tier 4 automatically syncs HOST2_DAILY_SYNC_SHARES — only list paths NOT in that array. + FAILOVER_HOST2_WRITEBACK_TIER1=( + # "/mnt/user/appdata-Failover/Jayred365-Emby" + ) + + FAILOVER_HOST2_WRITEBACK_TIER2=( + # "/mnt/user/appdata-Failover/Jayred365-Important" + ) + + FAILOVER_HOST2_WRITEBACK_TIER3=( + # "location-placeholder" + ) + + FAILOVER_HOST2_WRITEBACK_TIER4=( + "/mnt/user/appdata-Failover/Arrs_Stack" # arr databases — downloads queued during outage + ) + +# ============================================================================================== +# ── MEDIA ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Media Permissions ━━━ +# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to. +# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership. + HOST2_MEDIA_PERMISSION_SHARES=( + /mnt/user/Anime_Movies + /mnt/user/Anime_Shows + ) + +# ━━━ Media Cleaner ━━━ +# Folder lists for media_cleaner.sh — two profiles: anime and media. +# File patterns shared across all servers — defined in master.conf. +# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media + HOST2_ANIME_CLEAN_FOLDERS=( + /mnt/user/Anime_Movies + /mnt/user/Anime_Shows + ) + + HOST2_MEDIA_CLEAN_FOLDERS=( + # fill in when HOST2 is back online + ) + +# ============================================================================================== +# ── MONITORS ────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Certificate Monitor ━━━ +# Domains checked via direct openssl connection — not relying on NPM's certificate state. +# Checks the actual certificate served, not what NPM thinks it has. +# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf. + HOST2_CERT_MONITOR_DOMAINS=( + # fill in when HOST2 is back online + ) + +# ━━━ SMART Health ━━━ +# Drives skipped in SMART attribute monitoring — hardware is server-specific. +# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf. + HOST2_SMART_IGNORE_DRIVES=( + "sda" # boot USB — SMART not meaningful on flash drives + ) + +# ━━━ ZFS Report ━━━ +# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools. +# Pool health thresholds defined in master.conf. + HOST2_ZFS_REPORT_IGNORE_POOLS=( + # fill in when HOST2 is back online + ) + +# ============================================================================================== +# ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront. +# Adjust HOST2_RAMDISK_WARN_GB and HOST2_RAMDISK_LOW_GB together if this changes. + HOST2_RAMDISK_SIZE="8G" + +# Usage thresholds — coupled to HOST2_RAMDISK_SIZE, adjust all three together if size changes. +# Hysteresis gap (6.8 - 5.5 = 1.3GB) prevents flip-flop between ramdisk and SSD. + HOST2_RAMDISK_WARN_GB=6.8 # flip to SSD when ramdisk usage reaches this + HOST2_RAMDISK_LOW_GB=5.5 # flip back to ramdisk when usage drops to this + +# SSD fallback path — where transcodes land when ramdisk exceeds HOST2_RAMDISK_WARN_GB. +# Must be on cache pool — array disks too slow for active transcode writes. + HOST2_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" + +# Media servers sharing the ramdisk transcode space on HOST2. +# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex +# Entries with placeholder API keys are skipped automatically. +# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk. + HOST2_TRANSCODE_SERVERS=( + "${HOST2_EMBY_CONTAINER}|${HOST2_EMBY_URL}|${HOST2_EMBY_API_KEY}|emby" + ) + +# ============================================================================================== +# ── ARR STACK ───────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh. +# detect_hosts() selects HOST2 vars when running on HOST2. +# Lidarr does not run on HOST2 — HOST1_LIDARR_RECOVERY flag handles the exit cleanly. +# +# PATH MAPS — container path → host path translation. +# Arr stores file paths using container-internal paths — scripts need host paths to scan. +# Add one entry per root folder in arr Settings → Media Management → Root Folders. + +# ━━━ Sonarr ━━━ + HOST2_SONARR_URL="http://localhost:8989" + HOST2_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f" + HOST2_SONARR_TV_ROOT="/mnt/user/Anime_Shows" + + declare -A HOST2_SONARR_PATH_MAP=( + # fill in when HOST2 is back online + # ["/tv"]="/mnt/user/Anime_Shows" + ) + +# ━━━ Radarr ━━━ + HOST2_RADARR_URL="http://localhost:7878" + HOST2_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" + HOST2_RADARR_MOVIES_ROOT="/mnt/user/Anime_Movies" + + declare -A HOST2_RADARR_PATH_MAP=( + # fill in when HOST2 is back online + # ["/anime-movies"]="/mnt/user/Anime_Movies" + ) + +# ━━━ Arr Recovery Toggles ━━━ +# false = skip that arr on this host — exits cleanly without error + HOST2_SONARR_RECOVERY=true + HOST2_RADARR_RECOVERY=true + # HOST2_LIDARR_RECOVERY not set — Lidarr does not run on HOST2 + +# ============================================================================================== +# ──────────────────────── End Of HOST2 Variables ────────────────────────────────────────────── +# ============================================================================================== + +# ============================================================================================== +# ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Per-host check toggles and NIC config for system_watchdog.sh. +# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names. +# HOST2: i5 10th gen 64GB — being rebuilt, lighter workload, no ZFS cache pools. +# +# Conservative defaults during rebuild — re-enable checks as HOST2 stabilises. +# Three-tier response — all critical checks enabled regardless of rebuild state: +# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot +# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle +# Tier 3 (standard strike system): selectively disabled during rebuild +# +# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section. + +# ━━━ Primary NIC ━━━ +# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]" +# Common values: eth0, bond0, br0, eno1 + HOST2_SYS_WATCHDOG_NIC="eth0" + +# ━━━ Tier 1 — Critical Checks ━━━ +# All critical checks always enabled — these protect against acute failure regardless of +# rebuild state. Disabling any is not recommended. + +# Docker daemon unresponsive → try restart, reboot if restart fails. + HOST2_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true + +# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately. + HOST2_SYS_WATCHDOG_CHECK_ROOTFS=true + +# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately. + HOST2_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true + +# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately. + HOST2_SYS_WATCHDOG_CHECK_FD=true + +# /boot read-only detected → reboot immediately. + HOST2_SYS_WATCHDOG_CHECK_BOOT=true + +# ━━━ Tier 2 — Urgent OOM Check ━━━ +# Both must be enabled for Tier 2 bypass to function. + +# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta. + HOST2_SYS_WATCHDOG_CHECK_OOM=true + +# Free RAM check — 64GB RAM on HOST2, tiers adjusted relative to HOST1. +# Update master.conf SYS_WATCHDOG_MEM_* thresholds if HOST2 needs different values. +# Currently inheriting shared master.conf values — may want lower thresholds on 64GB. + HOST2_SYS_WATCHDOG_CHECK_RAM=true + +# ━━━ Tier 3 — Standard Checks (strike system) ━━━ +# Several checks disabled during rebuild — enable progressively as HOST2 stabilises. +# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action. + +# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT. + HOST2_SYS_WATCHDOG_CHECK_LOG=true + +# ZFS ARC memory check. +# DISABLED — HOST2 has no ZFS cache pools. Enable if ZFS pools are added later. + HOST2_SYS_WATCHDOG_CHECK_ARC=false + +# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C). + HOST2_SYS_WATCHDOG_CHECK_CPU_TEMP=true + +# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count. +# DISABLED — rebuild operations cause legitimate load spikes. Enable after rebuild. + HOST2_SYS_WATCHDOG_CHECK_LOAD=false + +# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50). + HOST2_SYS_WATCHDOG_CHECK_ZOMBIES=true + +# docker_watchdog.sh persistent skip list check. +# DISABLED during rebuild — skip list may be unreliable mid-rebuild, avoid false reboots. +# Enable once HOST2 is fully operational and docker_watchdog.sh is running stably. + HOST2_SYS_WATCHDOG_CHECK_CONTAINERS=false + +# /tmp filesystem usage with auto-clear attempt. + HOST2_SYS_WATCHDOG_CHECK_TMP=true + +# Array disk error count delta in /proc/mdstat. + HOST2_SYS_WATCHDOG_CHECK_MDSTAT=true + +# Primary NIC operstate — uses HOST2_SYS_WATCHDOG_NIC above. + HOST2_SYS_WATCHDOG_CHECK_NETWORK=true + +# sshd running check — restart attempt before escalating. + HOST2_SYS_WATCHDOG_CHECK_SSHD=true + +# Runaway process detection. +# DISABLED — rebuild workloads may legitimately peg CPU. Enable after rebuild. + HOST2_SYS_WATCHDOG_CHECK_RUNAWAY=false + +# ============================================================================================== +# ──────────────────────── End Of HOST2 Variables ────────────────────────────────────────────── +# ============================================================================================== \ No newline at end of file diff --git a/safe_master.conf b/safe_master.conf deleted file mode 100644 index 97eafaa..0000000 --- a/safe_master.conf +++ /dev/null @@ -1,1518 +0,0 @@ -#!/bin/bash -# ============================================================================================== -# ================================= MASTER CONFIGURATION ======================================= -# ============================================================================================== -# All user-facing variables for the unRAID script ecosystem. -# Scripts source this file — edit here, changes apply everywhere on next git pull. -# -# ── HOW THIS FILE WORKS ─────────────────────────────────────────────────────────────────────── -# Every script sources Master.conf and common.sh at startup. -# Change a value here and it affects all scripts that use it — no hunting through files. -# To disable something: comment it out with # rather than deleting it. -# To add a new rsync profile: add a key to each PROFILE_* array. -# To add or remove orchestrator jobs: edit the arrays in the ORCHESTRATORS section. -# -# ── INDEX ───────────────────────────────────────────────────────────────────────────────────── -# -# Section Description -# ─────────────────────────────────────────────────────────────────────────────────────────── -# HOST CONFIGURATION Server hostnames, SSH keys, Emby connection details, DATA_DIR -# LOGGING Enable or disable verbose logging -# NOTIFICATIONS unRAID native and Discord webhook settings -# GIT / REPO Gitea repository and SSH settings -# -# ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────── -# ARRAY START Scripts launched at array start (array_start.sh) -# DAILY SYNC MAINTENANCE Job list + media shares (daily_sync_maintenance.sh) -# WEEKLY SYNC MAINTENANCE Job list + sync jobs + sync settings (weekly_sync_maintenance.sh) -# MEDIA MANAGEMENT Job list for media_management.sh -# -# ── RSYNC ────────────────────────────────────────────────────────────────────────────────── -# RSYNC DEFAULTS Global fallback rsync options and limits -# REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort -# RSYNC PROFILE SYSTEM Per-profile overrides for appdata syncs -# -# ── FAILOVER ─────────────────────────────────────────────────────────────────────────────── -# FAILOVER Mutual container failover between two servers -# FAILOVER TEST Simulated outage settings for failover_test.sh -# DDNS Script-controlled DDNS — absolute rules -# INTERNET LOSS Containers to stop when internet is lost -# TIERED CONTAINER LISTS What each server runs for the other per tier -# TIER DELAY SETTINGS How long before each tier activates (minutes) -# RSYNC WRITEBACK JOBS Appdata synced back to primary on handback -# -# ── DOCKER ESSENTIALS ────────────────────────────────────────────────────────────────────── -# DOCKER DAILY RESTART Containers restarted daily -# DOCKER WEEKLY RESTART Containers restarted weekly -# DOCKER WATCHDOG Continuous two-tier self-healing container monitoring -# DOCKER NETWORK CONNECT Connect containers to extra networks on array start -# -# ── UNRAID ESSENTIALS ────────────────────────────────────────────────────────────────────── -# REBOOT User warning delay before scheduled reboot -# MOVER Mover stop timeout -# SYSLOG FILTER Docker veth noise filter file path -# PHP-FPM PHP-FPM max children config -# CLEAR LOGS System log file paths -# WEBGUI WATCHDOG WebGUI nginx + emhttp monitoring and restart -# -# ── MEDIA ────────────────────────────────────────────────────────────────────────────────── -# MEDIA PERMISSIONS Share list, mode and owner for permissions script -# MEDIA CLEANER Anime and media folder lists and file patterns -# ARR CLEANUP Lidarr, Sonarr, Radarr orphan file cleanup -# ARR FAILED/STALLED RECOVERY Auto blocklist + re-search failed imports and stalled downloads -# -# ── TRANSCODES ───────────────────────────────────────────────────────────────────────────── -# TRANSCODE MANAGER Ramdisk and SSD fallback transcode management -# TRANSCODE SERVER ARRAY Multi-server session monitoring (Emby, Jellyfin, Plex) -# -# ── MONITORS ─────────────────────────────────────────────────────────────────────────────── -# CERTIFICATE MONITOR SSL certificate expiry monitoring -# BACKUP VERIFY Random sample checksum verification against remote -# SMART HEALTH Drive SMART attribute monitoring -# ZFS MEMORY SNAPSHOT Weekly ZFS health and memory diagnostic report -# BANDWIDTH MONITOR Daily rsync transfer logging and weekly summary -# HEALTH DIGEST Aggregated system health digest — always/smart/weekly -# EMBY SESSION REPORT Weekly Emby usage statistics via API -# -# ── SYSTEM WATCHDOG ──────────────────────────────────────────────────────────────────────── -# SYSTEM WATCHDOG Continuous system health monitoring — last line of defense -# -# ============================================================================================== - -# ============================================================================================== -# ── HOST CONFIGURATION ──────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ Hosts ━━━ -# Hostnames must match exact Docker/unRAID hostnames — case sensitive. -# Used by detect_hosts() in common.sh to determine which server is local and which is remote. -# Both servers run identical scripts — host detection makes them bidirectional. - HOST1="your-host1-hostname" - HOST2="your-host2-hostname" - -# Data directory — persistent script state and statistics files. -# Array share — survives reboots, no flash drive wear. -# Created automatically if it doesn't exist. -# Only truly critical files (failover state, watchdog reboot log) stay on /boot/config. - DATA_DIR="/mnt/user/appdata/unraid_scripts/data" - -# SSH keys for server-to-server rsync and failover container operations. -# Both keys must be in /root/.ssh/ and authorised in the remote server's authorized_keys. - HOST1_SSH_KEY="/root/.ssh/your_host1_key" - HOST2_SSH_KEY="/root/.ssh/your_host2_key" - -# ━━━ Emby ━━━ -# Defined once here — referenced by transcode_manager.sh, emby_session_report.sh, -# emby_database_repair.sh, weekly_sync_maintenance.sh, and TRANSCODE_SERVERS array. -# API key: Emby Dashboard → API Keys → + New Key - HOST1_EMBY_CONTAINER="Emby" - HOST1_EMBY_URL="http://localhost:8096" - HOST1_EMBY_API_KEY="your-emby-api-key" - - HOST2_EMBY_CONTAINER="Emby-Secondary" - HOST2_EMBY_URL="http://localhost:8096" # same port — different server, different key - HOST2_EMBY_API_KEY="your-emby-api-key" - -# ============================================================================================== -# ── PARTNERSHIP ──────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Manages the relationship lifecycle between two unRAID servers. -# HOST1 is always the owner (source of truth) — HOST2 is always the mirror. -# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a --transfer operation. -# All other vars derive from HOST1/HOST2 — no duplicate IP or key configuration needed. -# -# State files live on /boot/config — survives reboots, available before array starts: -# /boot/config/partnership_HOST1.db ← HOST1 writes only -# /boot/config/partnership_HOST2.db ← HOST2 writes only -# Each server SSHes to write/read the other's file directly — no rsync needed -# -# critical_sync_maintenance.sh runs --check every 15min: -# Reads both state files via SSH -# Detects offboard requests → finalises from owner side -# Increments offline counter → auto-offboards after threshold -# Silent when healthy ✅ - - PARTNERSHIP_ENABLED=false - PARTNERSHIP_OWNER_HOST="HOST1" # "HOST1" or "HOST2" — flips on --transfer - -# Auth containers whose WebUI URL gets reconfigured on onboard/offboard -# Format: "ContainerName|Port" -# These are the containers the mirror operator clicks in his Docker UI -# On onboard → pointed at owner's Tailscale IP -# On offboard → pointed back at localhost -PARTNERSHIP_AUTH_WEBUIS=( - "NginxProxyManager|81" - "Lldap|17170" - "Authelia|9091" - "Authelia-Secondary|9092" -) - -# Paths on HOST1 that HOST2 operator should collect before backup retention expires -# Notified on offboard — no auto-deletion, manual collection -PARTNERSHIP_MIRROR_BACKUPS=( - # "/mnt/user/appdata-Failover/Host2-Emby" - # add any paths HOST2 wants to collect after separation -) - -# Offboard settings - PARTNERSHIP_GRACE_HOURS=3 # hours after offboard before Tailscale removal - # backup access also expires at this time - # collect anything needed before this window closes - PARTNERSHIP_REMOVE_TAILSCALE=true # remove mirror from Tailscale on offboard - # false = skip removal (manual or testing) - -# Tailscale API — used to remove HOST2 from tailnet on offboard -# API key: https://login.tailscale.com/admin/settings/keys -# Tailnet: your tailnet name (e.g. "yourname.github" or "youremail.com") - TAILSCALE_API_KEY="" # tskey-api-... - TAILSCALE_TAILNET="" # your tailnet name - -# Transfer safety - PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership" - PARTNERSHIP_TRANSFER_STRIKES=3 # consecutive health checks required - PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20 # max attempts before giving up - -# Onboard settings - PARTNERSHIP_ONBOARD_VERIFY=true # verify WebUI reachable after reconfiguration - PARTNERSHIP_ONBOARD_NOTIFY=true # notify both servers on completion - PARTNERSHIP_SYNC_INTERVAL=15 # minutes — Critical-Data sync frequency - -# Auto-offboard threshold - PARTNERSHIP_OFFLINE_THRESHOLD=30 # days either server unreachable before auto-offboard - # works both directions — mirror offline 30d → owner offboards - # owner offline 30d → mirror offboards itself - -# ============================================================================================== -# ── LOGGING ─────────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# Controls verbose [LOG] output across all scripts. -# true = show detailed [LOG] lines — useful for debugging or first-time setup -# false = show only user-facing output — cleaner for scheduled runs - ENABLE_LOGGING=true - -# ============================================================================================== -# ── NOTIFICATIONS ───────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# unRAID native notification system — integrates with the bell icon in the WebGUI. -# normal = job completed successfully / warning = something failed or needs attention - NOTIFY_UNRAID=true - -# Discord webhook URL — leave blank to disable - DISCORD_WEBHOOK="" - -# ============================================================================================== -# ── GIT / REPO ──────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# Gitea self-hosted repository — used by git_pull_execute.sh. -# Detects Gitea container location at runtime — works through failover automatically. -# Falls back to GITEA_DOMAIN if local and Tailscale both fail. - GITEA_CONTAINER="Gitea" # exact Docker container name - GITEA_REPO_PATH="youruser/Unraid_Scripts.git" # repo path on Gitea server - GITEA_DOMAIN="" # e.g. git.yourdomain.com — requires NPM + DNS setup - TARGET_DIR="/mnt/user/appdata/unraid_scripts" # where scripts are cloned to - GITEA_SSH_KEY="/root/.ssh/your_gitea_key" # SSH key for authenticating to Gitea - SSH_PORT=221 # Gitea SSH port (default 22, Gitea often uses 222/221) - -# ============================================================================================== -# ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# All orchestrator job lists live here — edit arrays to add/remove scripts. -# No changes to orchestrator scripts needed when adding or removing jobs. - -# ━━━ Array Start ━━━ -# Scripts launched by array_start.sh when the array comes online. -# Launched in order — each as a background process. -# One-shot scripts (ramdisk, syslog, fpm, network) run and exit naturally. -# Continuous scripts (watchdogs, failover) run until array stops. - -ARRAY_START_SCRIPTS=( - "Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts - "unRAID_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill - "unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning - "unRAID_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted - "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers - "unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop - "Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop - "Failover/failover.sh" # mutual failover — continuous loop -) - -# ━━━ Daily Sync Maintenance ━━━ -# daily_sync_maintenance.sh runs the media share sync built into the script first, -# then iterates DAILY_MAINTENANCE_SCRIPTS for additional jobs. -# Schedule: 0 1 * * * (1am daily) - -DAILY_MAINTENANCE_SCRIPTS=( - "git_pull_execute.sh" # pull latest scripts — always runs first - "Docker_Essentials/docker_daily_restart.sh" # daily container restarts -) - -# Media shares synced daily by daily_sync_maintenance.sh. -# Each server syncs only the shares it owns (source of truth) — direction is automatic. -# HOST1 pushes its truth shares to HOST2. HOST2 pushes its truth shares to HOST1. -# Never both pushing the same share — one server is always the truth holder. -# These shares use DEFAULT_RSYNC_OPTS — no profile entry needed. -# For shares needing custom options or container stops — create a profile in the RSYNC section. - -HOST1_DAILY_SYNC_SHARES=( - /mnt/user/Books - /mnt/user/Intros - /mnt/user/Kids_Movies - /mnt/user/Kids_Tv_Shows - /mnt/user/Movies - /mnt/user/Music - /mnt/user/Music_Videos - /mnt/user/Nextcloud - /mnt/user/stand-up_comedy - /mnt/user/Sports - /mnt/user/Tv_Shows - /mnt/user/Anime_Shows-Old - /mnt/user/Anime_Movies-Old -) - -HOST2_DAILY_SYNC_SHARES=( - /mnt/user/Anime_Movies - /mnt/user/Anime_Shows -) - -# Personal encrypted shares — synced for offsite backup, independent of media shares. -# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content. -# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting. -HOST1_PERSONAL_SHARES=( - # /mnt/user/Host1-Personal # uncomment after creating encrypted dataset -) - -HOST2_PERSONAL_SHARES=( - # /mnt/user/Host2-Personal # uncomment after creating encrypted dataset -) - -# ━━━ Weekly Sync Maintenance ━━━ -# weekly_sync_maintenance.sh handles the critical sync built into the script first: -# stop containers both sides → pull updates → sync Emby + Critical-Data → restart -# Then iterates WEEKLY_MAINTENANCE_SCRIPTS for additional jobs. -# Schedule: 30 2 * * 0 (Sunday 2:30am) - -WEEKLY_MAINTENANCE_SCRIPTS=( - "Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync -) - -# Shares synced by weekly_sync_maintenance.sh during the maintenance window. -# Containers are stopped both sides before these sync — full clean state guaranteed. -# Profiles drive container stops, excludes, and options — configure in RSYNC section. -# Order matters — Emby first, then auth stack. -WEEKLY_SYNC_JOBS=( - "/mnt/user/Media_Server/Emby" # emby profile — full clean mirror - "/mnt/user/appdata-Failover/Critical-Data" # critical-data profile — auth stack -) - -# ━━━ Critical Sync Maintenance ━━━ -# Shares synced every 15 minutes by critical_sync_maintenance.sh -# Format: "/path/to/share" or "/path/to/share|profile-name" -# Order matters — Critical-Data first (auth stack), then emby-failover (dirty sync) -CRITICAL_SYNC_SHARES=( - "/mnt/user/appdata-Failover/Critical-Data" # auth stack — full sync - "/mnt/user/Media_Server/Emby|emby-failover" # Emby dirty sync — stays running -) - -# Container update toggles for the weekly sync window. -# Containers are already stopped for the sync — updates pull at no extra downtime. -# Both false → sync only, no updates. -# Toggle false temporarily to skip updates without changing the schedule. - CRITICAL_SYNC_UPDATES=true # pull container updates locally - CRITICAL_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH - -# Orchestrators that use rsync — read by rsync_stop.sh for auto-detection -# Add any new orchestrator that calls rsync.sh here — no script changes needed -# rsync_stop.sh will detect and handle it automatically on stop -RSYNC_ORCHESTRATORS=( - "daily_sync_maintenance" - "weekly_sync_maintenance" -) - -# ━━━ Media Management ━━━ -# Job list run directly by daily_sync_maintenance.sh after the media share sync. -# Runs sequentially — permissions first, then cleaners, then arr cleanup. -# Comment out any job to disable without removing it. -# Each individual script can still be run manually for one-off maintenance. - -MEDIA_MANAGEMENT_JOBS=( - "Media/media_shares_permissions.sh" # apply permissions — runs first - "Media/media_cleaner.sh anime" # remove junk from anime shares - "Media/media_cleaner.sh media" # remove junk from media shares - "Media/lidarr_cleanup.sh" # remove orphaned music files - "Media/sonarr_cleanup.sh" # remove orphaned TV files - "Media/radarr_cleanup.sh" # remove orphaned movie files - "Docker_Essentials/downloaders_reset.sh" # clear stuck states + purge old history -) - -# ============================================================================================== -# ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ Rsync Enable/Disable ━━━ -# Two-tier toggle system — Tier 1 overrides Tier 2. -# -# Tier 1 — Global gate: -# RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions -# Use when: remote server completely offline, major maintenance, disaster recovery -# -# Tier 2 — Per-orchestrator (only applies when Tier 1 is true): -# Fine grained control — disable specific orchestrators while keeping others -# Use when: rebuilding secondary, testing, per-window bandwidth management -# -# Real world example (HOST2 data rebuild — your current situation): -# RSYNC_ENABLED=true ← rsync works, individual scripts run fine -# DAILY_RSYNC_ENABLED=false ← skip daily HDD syncs during rebuild -# WEEKLY_RSYNC_ENABLED=true ← Emby + Critical-Data still sync (NVMe, separate BW) -# FAILOVER_RSYNC_ENABLED=true ← handback writeback still works when needed -# → Run individual: bash Rsync/rsync.sh /mnt/user/Movies (test each share manually) -# → When ready: DAILY_RSYNC_ENABLED=true - - RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below - DAILY_RSYNC_ENABLED=false # Tier 2 — daily_sync_maintenance.sh rsync section - WEEKLY_RSYNC_ENABLED=true # Tier 2 — weekly_sync_maintenance.sh rsync section - FAILOVER_RSYNC_ENABLED=true # Tier 2 — failover.sh writeback jobs on handback - CRITICAL_RSYNC_ENABLED=true # Tier 2 — critical_sync_maintenance.sh (every 15min) - -# ━━━ Rsync Defaults ━━━ -# Global fallback values used when no profile match is found. -# Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed. -# Appdata shares match profiles by directory basename (lowercased). -# If a profile key exists it overrides the global. If missing the global is used. - - BW_LIMIT=12500 # KB/s — 12500 ≈ 100Mbit — network transfer speed cap - RETRY_COUNT=3 # retry attempts if rsync fails before giving up - SLEEP=300 # seconds between retry attempts - CRITICAL_CONTAINER_NAMES=() # containers to stop on REMOTE before rsync — profiles override - DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override - CONTAINER_DELAY=5 # seconds to wait before starting delayed containers - EXCLUDE_DIRS=() # directories to exclude from transfer — profiles override - -# --delete removes files on remote that no longer exist on source (mirror behaviour) -# --inplace writes directly to destination — better for large files, avoids temp copies -# --no-whole-file forces delta transfer even on fast connections — sends only changed blocks - DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file) - -# ━━━ Remote Health Checks ━━━ -# Pre-flight check — aborts if remote rootfs (/) usage is at or above this percentage. -# When remote array is down, rsync writes land on rootfs — fills fast and crashes the server. - ROOTFS_WARN=75 - -# ━━━ Rsync Profile System ━━━ -# Profiles allow per-share rsync behaviour without touching script logic. -# Profile key matched by basename of directory passed to rsync.sh (lowercased). -# Override with --profile=name flag. -# -# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit DEFAULT_RSYNC_OPTS. -# List ALL desired options explicitly when defining a profile. -# -# Current profiles: -# arrs_stack — arr databases — lower bandwidth, containers stopped for consistency -# critical-data — auth stack — containers stopped both sides, Authelia delayed start -# host1 — server-specific appdata — no container stops needed -# important-data — NextCloud + Postgres — NextCloud delayed start after Postgres -# emby — weekly clean sync — both Emby stopped, full mirror, minimal excludes -# called by weekly_sync_maintenance.sh only — do NOT schedule separately -# emby-failover — frequent dirty sync — Emby stays running, WAL excluded, critical data only -# also used for failover writeback on handback - -declare -A PROFILE_RSYNC_OPTS=( - [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" - [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete" - [host1]="-av --info=progress2 --bwlimit=$BW_LIMIT" - [important-data]="-av --human-readable --bwlimit=$BW_LIMIT" - [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" - [emby-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" -) - -# Per-profile bandwidth limits in KB/s — overrides global BW_LIMIT for that profile only -# Lower for shares running alongside other jobs, higher for time-sensitive critical data -declare -A PROFILE_BW_LIMIT=( - [arrs_stack]=5000 # lower — runs alongside other syncs, avoids saturating link - [critical-data]=9500 # high — small dataset, get it synced fast and clean - [host1]=8000 - [important-data]=9500 # high — database sync needs to be fast - [emby]=8000 # medium — large full mirror, steady transfer - [emby-failover]=9500 # high — small critical dataset, sync as fast as possible -) - -# Retry attempts per profile — how many times to retry before giving up on a failed sync -declare -A PROFILE_RETRY_COUNT=( - [arrs_stack]=3 - [critical-data]=3 - [host1]=3 - [important-data]=3 - [emby]=3 - [emby-failover]=3 -) - -# Seconds to wait between retry attempts -# emby-failover shorter — frequent sync, faster retry on transient failures -declare -A PROFILE_SLEEP=( - [arrs_stack]=300 - [critical-data]=300 - [host1]=300 - [important-data]=300 - [emby]=300 - [emby-failover]=120 # shorter — frequent dirty sync, retry faster -) - -# Containers stopped on BOTH LOCAL and REMOTE servers before rsync. -# Local stops first — flushes databases cleanly before pushing data out. -# Remote stops next — prevents writes to destination while receiving. -# Only containers that were running get restarted — stopped containers stay stopped. -# Same container names on both servers — consistent naming is required by this ecosystem. -# If a container is not found on a server it is skipped gracefully, not errored. -# SPACE-SEPARATED STRINGS — converted to array at runtime -declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( - [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" - [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap NginxProxyManager Authelia Authelia-Secondary" - [host1]="Organizrv2 UptimeKuma VaultWarden" - [important-data]="Postgres-NextCloud NextCloud" - [emby]="Emby" # weekly clean sync — both Emby instances stopped, WAL checkpointed - [emby-failover]="" # dirty sync — Emby stays running both sides, WAL excluded from sync -) - -# Containers that need a delay before starting after rsync completes. -# Database containers must be accepting connections before dependent apps start. -# Authelia waits for Mariadb + Redis. NextCloud waits for Postgres. -# SPACE-SEPARATED STRINGS — converted to array at runtime -declare -A PROFILE_DELAYED_CONTAINERS=( - [arrs_stack]="" - [critical-data]="Authelia Authelia-Secondary" # wait for Mariadb + Redis to be ready - [host1]="" - [important-data]="NextCloud" # wait for Postgres to accept connections - [emby]="" - [emby-failover]="" -) - -# Seconds to wait before starting delayed containers -# 15s gives Mariadb, Redis, and LLDAP time to accept connections before Authelia starts -declare -A PROFILE_CONTAINER_DELAY=( - [arrs_stack]=5 - [critical-data]=15 # Mariadb + Redis need time to accept connections - [host1]=5 - [important-data]=10 # Postgres needs time before NextCloud - [emby]=5 - [emby-failover]=5 -) - -# Directories excluded from rsync transfer per profile -# emby-failover excludes WAL files — safe to sync while Emby is running -# emby clean sync only excludes logs, transcodes, cache — full metadata mirror -# SPACE-SEPARATED STRINGS — converted to array at runtime -declare -A PROFILE_EXCLUDE_DIRS=( - [arrs_stack]="logs *.tmp" - [host1]="logs *.tmp" - [important-data]="logs *.tmp" - [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt" - [emby]="logs transcodes cache crash*" - # emby-failover: Emby running, WAL excluded — only safe critical data synced - # users.db, library.db, authentication.db, config/ — everything else excluded - [emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root" -) - -# Skip per-disk space check is no longer needed — check_remote_disks() auto-detects -# XFS and ZFS filesystem types from disks.ini, no manual configuration required - -# ============================================================================================== -# ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Mutual container failover between two unRAID servers. -# Each server runs Failover/failover.sh independently via array_start.sh. -# All decisions based on two pings: remote reachable + internet reachable. -# -# States: NORMAL | FAILOVER | NO_INTERNET | DARK -# -# DDNS rules — absolute: -# Internet loss → stop own DDNS immediately -# Failover → start remote DDNS first (Tier 1) -# Handback → stop remote DDNS → rsync → start containers → start local DDNS last -# -# Tiers: -# Tier 1 — Immediate — vital services + Live TV -# Tier 2 — configurable delay — productivity services -# Tier 3 — configurable delay — secondary services -# Tier 4 — configurable delay — arrs + downloaders - - EXTERNAL_IP="8.8.8.8" - FAILOVER_CHECK_INTERVAL=120 - FAILOVER_HANDBACK_STRIKES=2 - FAILOVER_STATE_FILE="/boot/config/failover_state.db" - FAILOVER_ENABLED=false # set true when HOST2 is back online and tested - # false = suppresses "not running" warnings in status scripts - -# ━━━ Failover Test ━━━ - FAILOVER_TEST_BLOCK_WAIT=150 - FAILOVER_TEST_HANDBACK_WAIT=360 - -# ━━━ DDNS ━━━ -HOST1_DDNS_CONTAINERS=( - "yourdomain.com" -) - -HOST2_DDNS_CONTAINERS=( - "yourdomain.us" -) - -# ━━━ Internet Loss ━━━ -FAILOVER_HOST1_STOP_ON_NO_NET=( - "yourdomain.com" -) - -FAILOVER_HOST2_STOP_ON_NO_NET=( - "yourdomain.us" -) - -# ━━━ Tiered Container Lists ━━━ - -# HOST1 runs for HOST2 -FAILOVER_HOST1_RUNS_FOR_HOST2_IMMEDIATE=( - "yourdomain.us" - "VaultWarden-Secondary" - # "container-placeholder" -) - -FAILOVER_HOST1_RUNS_FOR_HOST2_2HR=( - # "container-placeholder" -) - -FAILOVER_HOST1_RUNS_FOR_HOST2_6HR=( - # "container-placeholder" -) - -FAILOVER_HOST1_RUNS_FOR_HOST2_18HR=( - # "container-placeholder" -) - -# HOST2 runs for HOST1 -FAILOVER_HOST2_RUNS_FOR_HOST1_IMMEDIATE=( - "yourdomain.com" - "Emby" - "VaultWarden" - "Dispatcharr" - "Dispatcharr-Basic" - "Dispatcharr-Iptv-Users" - "ErsatzTV-Emby" -) - -FAILOVER_HOST2_RUNS_FOR_HOST1_2HR=( - "Postgres-NextCloud" - "NextCloud" - "PostgreSQL_Immich" - "Immich" - # "container-placeholder" -) - -FAILOVER_HOST2_RUNS_FOR_HOST1_6HR=( - "Gitea" - # "container-placeholder" -) - -FAILOVER_HOST2_RUNS_FOR_HOST1_18HR=( - "Sonarr" - "Radarr" - "Lidarr" - "Readarr" - "Prowlarr" - "Bazarr" - "SABnzbd" - "Qbittorrent" - "LidaTube" - "Pinchflat" - "ChannelTube" - # "container-placeholder" -) - -# ━━━ Tier Delay Settings ━━━ -# How long the primary server must be down before each tier activates — in minutes. -# Tier 1 is always immediate — Live TV and media can't wait. -# Set independently per host — adjust based on hardware and what's worth starting. -# Longer delays = less resource usage on covering server but slower recovery. -# -# HOST1's containers running on HOST2 (HOST1 is down): -HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich — can wait -HOST1_TIER3_DELAY=720 # 12 hours — secondary services — Gitea etc. -HOST1_TIER4_DELAY=1440 # 24 hours — full workflow — arrs and downloaders - -# HOST2's containers running on HOST1 (HOST2 is down): -HOST2_TIER2_DELAY=240 -HOST2_TIER3_DELAY=720 -HOST2_TIER4_DELAY=1440 - -# ━━━ Rsync Writeback Jobs ━━━ -# Syncs critical appdata BACK to primary server during handback after failover. -# Containers are stopped before writeback runs — clean source, no competing writes. -# Purpose: primary comes back online with the state that built up during its outage -# (watch states, auth changes, library updates that happened on HOST2) -# -# HOST*_TIER1_WRITEBACK_DELAY: -# Short outages skip Tier 1 writeback — primary state is more reliable than dirty sync data -# Only writeback if outage lasted longer than this many minutes -# 60 minutes = if HOST1 was down less than 1hr, don't bother writing back Emby -# -# Tier 4 writeback automatically syncs HOST*_DAILY_SYNC_SHARES back — no need to list those here -# Only add paths that are NOT in DAILY_SYNC_SHARES and need writeback after extended outage - -HOST1_TIER1_WRITEBACK_DELAY=60 # minutes — skip Emby writeback if outage under 1hr -HOST2_TIER1_WRITEBACK_DELAY=60 - -# HOST1 writeback — run by HOST2 during HOST1 handback -FAILOVER_HOST1_WRITEBACK_TIER1=( - "/mnt/user/Media_Server/Emby" # watch states, playstates built up during outage -) - -FAILOVER_HOST1_WRITEBACK_TIER2=( - "/mnt/user/appdata-Failover/Important-Data" # NextCloud + Postgres — files added during outage -) - -FAILOVER_HOST1_WRITEBACK_TIER3=( - # "location-placeholder" -) - -FAILOVER_HOST1_WRITEBACK_TIER4=( - # Edge cases outside HOST1_DAILY_SYNC_SHARES - "/mnt/user/appdata-Failover/Arrs_Stack" # arr databases — downloads queued during outage -) - -# HOST2 writeback — run by HOST1 during HOST2 handback -FAILOVER_HOST2_WRITEBACK_TIER1=( - # "/mnt/user/appdata-Failover/Host2-Emby" -) - -FAILOVER_HOST2_WRITEBACK_TIER2=( - # "/mnt/user/appdata-Failover/Host2-Important" -) - -FAILOVER_HOST2_WRITEBACK_TIER3=( - # "location-placeholder" -) - -FAILOVER_HOST2_WRITEBACK_TIER4=( - # Edge cases outside HOST2_DAILY_SYNC_SHARES - "/mnt/user/appdata-Failover/Arrs_Stack" -) - -# ============================================================================================== -# ── DOCKER ESSENTIALS ───────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ Downloaders Reset ━━━ -# Daily maintenance reset for all download clients. -# Called by daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS before container restarts. -# Clears stuck states, purges old history, prepares each downloader for a clean daily cycle. -# -# Retention period — applies to: slskd failed imports, SABnzbd completed and failed history - DOWNLOADER_RETENTION_DAYS=7 - -# ── slskd ── -# Clears stuck/errored searches, dead transfer records, purges expired failed imports -# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected - HOST1_SLSKD_URL="http://localhost:8980" - HOST1_SLSKD_API_KEY="your-slskd-api-key" - HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports" - -# ── SABnzbd ── -# Clears completed history, failed history, and stalled paused queue items - HOST1_SABNZBD_URL="http://localhost:8180" - HOST1_SABNZBD_API_KEY="your-sabnzbd-api-key" - -# ── qBittorrent ── -# Last chance failsafe — deletes torrents older than QBIT_FAILSAFE_MIN_DAYS -# qBittorrent's own rules handle normal cleanup (ratio >= 1.25 OR 45 days inactive) -# This catches anything missed after extended time -# deleteFiles=false — removes torrent from qBit but leaves files on disk -# Radarr/Sonarr manage actual files independently -# QBIT_FAILSAFE_MIN_RATIO=0 disables ratio gate — age is the only condition - HOST1_QBIT_URL="http://localhost:8080" - HOST1_QBIT_USERNAME="admin" - HOST1_QBIT_PASSWORD="your-qbit-password" - QBIT_FAILSAFE_MIN_DAYS=180 - QBIT_FAILSAFE_MIN_RATIO=0 # 0 = age only, no ratio requirement - -# ━━━ Docker Daily Restart ━━━ -# Containers restarted every day by docker_daily_restart.sh via daily_sync_maintenance.sh. -# These containers run better with a daily restart — not just "keeping things fresh". -# Dispatcharr specifically degrades over time without restart — daily is intentional. -# Schedule is set in daily_sync_maintenance.sh — runs at 1am as part of daily window. -# Case-sensitive — must match exact Docker container names. -DAILY_RESTART_CONTAINERS=( - "NginxProxyManager" - "Authelia" - "Dispatcharr-Iptv-Users" - "Dispatcharr" # Live TV scheduler — degrades without daily restart - "Dispatcharr-Basic" - "ErsatzTV-Emby" -) - -# ━━━ Docker Weekly Restart ━━━ -# Less critical services restarted weekly by docker_weekly_restart.sh. -# Called by weekly_sync_maintenance.sh Sunday 2:30am — containers already stopped -# for the weekly sync window so restart adds zero extra downtime. -# Weekly restarts also catch any pending image updates not applied during weekly sync. -WEEKLY_RESTART_CONTAINERS=( - "NextCloud" - "Organizrv2" - "AdGuard-Home" - "Immich" -) - -# ━━━ Docker Watchdog ━━━ -# Continuous two-tier self-healing container monitoring. -# Started by array_start.sh — runs until array stops. -# Re-sources Master.conf each cycle — add/remove containers without restarting watchdog. -# Silent when all healthy — only logs when something needs attention. -# Heartbeat fires periodically as proof of life even when everything is healthy. -# -# Tier 1 — strict monitoring of explicitly configured containers: -# Memory hard limits — immediate restart if container exceeds limit -# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT sustained strikes -# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT failed checks -# Required containers — must always be running, strike + skip list with auto-clear -# -# Tier 2 — global health scan of ALL running containers: -# Unhealthy status — Docker HEALTHCHECK unhealthy → restart -# OOM killed — kernel killed container → restart + notify -# Crash loop detection — RestartCount climbing → notify, critical above limit -# Dead containers — remove and restart -# Unexpected exits — non-zero exit code → restart - -# Memory hard limits in MB — immediate restart if exceeded -# Container restarted the moment it crosses this line — no strike system -# 20GB=20480 16GB=16384 12GB=12288 10GB=10240 -# 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024 -declare -A WATCHDOG_CONTAINERS=( - ["Emby"]=16384 - ["LidaTube"]=6144 - ["Tdarr"]=6144 - ["Code-Server"]=1024 -) - -# HTTP health check URLs — checked every cycle, strike system before restart -# Container must respond with HTTP 200 within CURL_TIMEOUT seconds -# Per-host — HOST1 and HOST2 may run different containers on different ports -declare -A HOST1_WATCHDOG_CONTAINER_URLS=( - ["Emby"]="http://localhost:8096" -) - -declare -A HOST2_WATCHDOG_CONTAINER_URLS=( - ["Emby"]="http://localhost:8096" -) - -# Required containers — must always be running -# Strike system: SYS_WATCHDOG_STRIKE_LIMIT strikes before restart attempt -# Persistent skip list: added after WATCHDOG_CONTAINER_RESTART_LIMIT restarts in window -# Skip list auto-clears when container recovers — no manual intervention needed -# Per-host — each server has different critical containers -HOST1_WATCHDOG_REQUIRED_CONTAINERS=( - "NginxProxyManager" - "Lldap" - "Authelia" - "Mariadb-Authelia" - "Redis-Authelia" - "Authelia-Secondary" - "Redis-Authelia-Secondary" -) - -HOST2_WATCHDOG_REQUIRED_CONTAINERS=( - "NginxProxyManager" - # add HOST2 required containers here -) - -# Strike state file — /tmp resets on reboot which is correct -# Fresh start after reboot means no stale strikes carrying over - WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db" - -# CPU thresholds — normalised against total core count automatically at runtime -# SOFT = warn only, HARD = strike toward restart -# CPU_FAIL_LIMIT = consecutive HARD strikes before restart - SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU - HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU - CPU_FAIL_LIMIT=2 # consecutive hard CPU strikes before container restart - -# Memory soft threshold — warn when container reaches this % of its WATCHDOG_CONTAINERS hard limit -# Does not trigger restart — informational only - SOFT_MEM_THRESHOLD=80 - -# HTTP responsiveness — consecutive failed checks before restart -# CURL_TIMEOUT = seconds before curl gives up on a single check - RESP_FAIL_LIMIT=2 # consecutive failed checks before restart - CURL_TIMEOUT=5 # seconds per check before timeout - -# How often the watchdog runs its checks -# 900 = 15 minutes — long enough to not be noisy, short enough to catch issues quickly -# Containers have this long to recover before next check - DOCKER_WATCHDOG_INTERVAL=900 # seconds between watchdog cycles - -# Heartbeat — proof of life logged periodically even when everything is healthy -# Useful to confirm the watchdog is still running without flooding logs - DOCKER_WATCHDOG_HEARTBEAT=true # true = log heartbeat / false = completely silent - DOCKER_WATCHDOG_HEARTBEAT_HOURS=1 # how often to log heartbeat (hours) - -# Tier 2 master toggle — set false to disable global container scanning entirely -# When false only WATCHDOG_CONTAINERS and required containers are monitored - WATCHDOG_SCAN_ALL=true - -# Containers to skip in Tier 2 scan entirely -# Useful for containers that legitimately exit/restart frequently -WATCHDOG_SCAN_IGNORE=( - "DashGate" - "PIA-WG-Config-Generator" - "Aperture" - "Aperture-Kids" - "pgvector-18-Apeture-Kids" - "Pgvector18-Aperture" -) - -# Individual Tier 2 check toggles — disable specific checks without disabling Tier 2 - WATCHDOG_RESTART_UNHEALTHY=true # restart containers with Docker HEALTHCHECK = unhealthy - WATCHDOG_RESTART_DEAD=true # restart containers in dead state - WATCHDOG_RESTART_CRASHED=true # restart containers that exited with non-zero code - WATCHDOG_NOTIFY_OOM=true # notify + restart OOM killed containers - WATCHDOG_NOTIFY_CRASHLOOP=true # notify when Docker RestartCount keeps climbing - -# Crash loop threshold — notify critical if Docker has restarted this many times total -# Above this number the notification escalates to critical — manual intervention needed - WATCHDOG_CRASH_LIMIT=5 - -# Startup grace period — skip restarts while system is still booting after array start -# Prevents watchdog from restarting containers that are legitimately still initializing - WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures - -# Restart loop protection — stops hammering a broken container -# If watchdog restarts a container more than LIMIT times in WINDOW hours → skip list -# Skip list auto-clears when container recovers healthy - WATCHDOG_CONTAINER_RESTART_LIMIT=3 # max watchdog restarts allowed in window - WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours - WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db" - # rolling restart history for loop detection - -# Notification batching — one clean summary per cycle instead of one ping per event -# true = batch all events into one notification at end of cycle -# false = send one notification per event (noisy on busy systems) - WATCHDOG_BATCH_NOTIFY=true - -# Dependency ordering — skip restarting a container if its dependency is also down -# Prevents restarting Authelia before its database is ready -# Space-separated list of dependencies per container -declare -A WATCHDOG_DEPENDENCIES=( - ["Authelia"]="Mariadb-Authelia Redis-Authelia" - ["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary" - ["NextCloud"]="Postgres-NextCloud" -) - -# ━━━ Docker Network Connect ━━━ -# Connects containers to extra Docker networks on array start via array_start.sh. -# Useful for containers that need their own custom network but also need to be -# reachable from your main custom bridge network. -# Every container in NETWORK_CONNECT_CONTAINERS is connected to every network in -# NETWORK_CONNECT_NETWORKS — containers not found are skipped gracefully. -NETWORK_CONNECT_CONTAINERS=( - "memcached" - "Npm-CrowdSec" -) - -NETWORK_CONNECT_NETWORKS=( - "high-availability" # must exist before array start — create in Docker settings -) - -# ============================================================================================== -# ── UNRAID ESSENTIALS ───────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ inotify Tuning ━━━ -# Linux inotify limits — applied at every array start by inotify_tuning.sh -# Default unRAID values are very low — with many Docker containers watching files -# (Sonarr, Radarr, Lidarr, NextCloud etc.) you can silently exhaust the limit. -# Symptoms: containers miss file events, downloads not detected, library not updated. -# These settings are lost on reboot — reapplied automatically at array start. - INOTIFY_MAX_INSTANCES=1024 # default: 128 — max inotify instances per user - INOTIFY_MAX_WATCHES=524288 # default: 8192 — max files watched per instance - INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — max events queued before dropping - -# ━━━ System Tuning Monitor ━━━ -# Tracks inotify and php-fpm usage over time — read by sunday_morning_coffee_report.sh -# Snapshot written every 6 hours by system_tuning_monitor.sh -# Log bounded to TUNING_LOG_RETENTION days — auto-purges on each write - INOTIFY_WARN_PCT=80 # warn if inotify instances exceed this % of limit - PHP_FPM_WARN_PCT=80 # warn if php-fpm workers exceed this % of max_children - TUNING_MONITOR_LOG="$DATA_DIR/system_tuning_history.db" - TUNING_LOG_RETENTION=30 # days — enough for monthly trend visibility - -# ━━━ Reboot ━━━ -# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots. -# Gives users time to save work — 300s = 5 minutes - REBOOT_SLEEP=300 - -# ━━━ Mover ━━━ -# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process. -# Gives mover time to finish current file transfer before being interrupted. - MOVER_STOP_TIMEOUT=300 - -# ━━━ Syslog Filter ━━━ -# Path for the rsyslog filter file that suppresses Docker veth interface noise. -# Docker creates a new veth interface for each container — generates hundreds of -# log lines per hour that have no diagnostic value. Filter removes them at source. - FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" - -# ━━━ PHP-FPM ━━━ -# Higher max_children allows more concurrent PHP requests to the unRAID WebGUI. -# Default is very low — increasing it prevents WebGUI slowdowns under load. -# 250 is safe for servers with 32GB+ RAM. - PHP_CONF="/etc/php-fpm.d/www.conf" - PHP_MAX_CHILDREN=250 - -# ━━━ Clear Logs ━━━ -# System log files cleared weekly to prevent rootfs fill over time. -# These grow continuously — without clearing they eventually consume all rootfs space. - LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) - -# ━━━ WebGUI Watchdog ━━━ -# Monitors unRAID WebGUI responsiveness — escalates through nginx restart → emhttp restart. -# Separate from docker_watchdog — this monitors the unRAID UI itself, not containers. -# WEBGUI_NGINX_WAIT = seconds after nginx restart before rechecking -# WEBGUI_EMHTTP_WAIT = seconds after emhttp restart before rechecking - WEBGUI_URL="http://localhost" - WEBGUI_TIMEOUT=5 # seconds before curl gives up on WebGUI check - WEBGUI_NGINX_WAIT=15 # seconds after nginx restart before rechecking - WEBGUI_EMHTTP_WAIT=30 # seconds after emhttp restart before rechecking - -# ============================================================================================== -# ── MEDIA ───────────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ Media Permissions ━━━ -# Applied recursively to all shares in MEDIA_PERMISSION_SHARES by media_shares_permissions.sh. -# Runs first in MEDIA_MANAGEMENT_JOBS — arr cleanup scripts depend on correct ownership. -# 777 mode = read/write/execute for all users — standard for unRAID media shares -# nobody:users = standard unRAID media share ownership - PERMISSIONS_MODE="777" - PERMISSIONS_OWNER="nobody:users" - -MEDIA_PERMISSION_SHARES=( - /mnt/user/Anime_Movies - /mnt/user/Anime_Movies-Old - /mnt/user/Anime_Shows - /mnt/user/Anime_Shows-Old - /mnt/user/appcache - /mnt/user/Books - /mnt/user/Downloads - /mnt/user/Games - /mnt/user/Intros - /mnt/user/Kids_Movies - /mnt/user/Kids_Tv_Shows - /mnt/user/Movie_Recordings - /mnt/user/Movies - /mnt/user/Music - /mnt/user/Music_Videos - /mnt/user/Photo - /mnt/user/Sports - /mnt/user/stand-up_comedy - /mnt/user/Temp_Storage - /mnt/user/Tv_Recordings - /mnt/user/Tv_Shows - /mnt/user/YouTube -) - -# ━━━ Media Cleaner ━━━ -# Removes junk files from media shares — two profiles: anime and media. -# Called via MEDIA_MANAGEMENT_JOBS. Run manually: Media/media_cleaner.sh anime|media - -ANIME_CLEAN_FOLDERS=( - /mnt/user/Anime_Movies - /mnt/user/Anime_Movies-Old - /mnt/user/Anime_Shows - /mnt/user/Anime_Shows-Old -) - -MEDIA_CLEAN_FOLDERS=( - /mnt/user/Kids_Movies - /mnt/user/Kids_Tv_Shows - /mnt/user/Movies - /mnt/user/Music - /mnt/user/Sports - /mnt/user/stand-up_comedy - /mnt/user/Tv_Shows -) - -ANIME_FILE_PATTERNS=( - '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' - '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' - '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp' - '*.log' '*.json' -) - -MEDIA_FILE_PATTERNS=( - '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' - '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' - '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp' - '*.log' '*.json' '*.iso' '*.lrc' -) - -# ━━━ Arr Cleanup ━━━ -# Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs. -# Compares tracked file paths from API against disk — deletes untracked files older than ORPHAN_AGE. -# detect_hosts() selects correct URL, API key, and root path at runtime. -# -# Protected patterns are NEVER deleted — cover art, metadata, subtitles generated by the arr -# are not included in the tracked file API response but must not be deleted. -# -# API versions and endpoint patterns: -# Sonarr v4 → /api/v3/series (get IDs) → /api/v3/episodefile?seriesId=X per series -# Radarr v5 → /api/v3/movie (get IDs) → /api/v3/moviefile?movieId=X per movie -# Lidarr v3 → /api/v1/artist (get IDs) → /api/v1/trackFile?artistId=X per artist -# All require per-ID loops — bulk endpoints removed in newer versions -# -# Version checking — scripts verify the arr major version matches before running -# If the arr updates and breaks the API the script exits safely before touching files -# Update the MAJOR version here when the script is updated to support a new version -# MINOR = 0 means any minor version within that major is accepted - - SONARR_VERSION_MAJOR=4 # tested major version — script exits if major differs - RADARR_VERSION_MAJOR=6 # tested major version — script exits if major differs - LIDARR_VERSION_MAJOR=3 # tested major version — script exits if major differs - -# ── Lidarr ──────────────────────────────────────────────────────────────────────────────────── -HOST1_LIDARR_URL="http://localhost:8686" -HOST1_LIDARR_API_KEY="your-lidarr-api-key" -HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New" -LIDARR_LOCK_WARN_AGE=3600 # 1hr — large libraries take time, not stuck - -# Container path → host path translation -# Lidarr stores file paths using container paths — script scans host paths -# Add one entry per root folder configured in Lidarr Settings → Media Management → Root Folders -declare -A HOST1_LIDARR_PATH_MAP=( - ["/ext-music"]="/mnt/user/Music-New" -) -declare -A HOST2_LIDARR_PATH_MAP=( - # HOST2 does not run Lidarr — fill in if that changes - # ["/ext-music"]="/mnt/user/Music-New" -) - -LIDARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion - # protects files that may still be mid-import or recently downloaded -LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") -LIDARR_PROTECTED_PATTERNS=( - # Metadata - "*.nfo" "*.tbn" - # Images — album art, artist images, Emby artwork - "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" - "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" - "banner.*" "thumb.*" "landscape.*" - "folder.*" "cover.*" "album.*" "artist.*" "disc.*" - # Lyrics - "*.lrc" -) - # NEVER deleted — cover art, metadata, lyrics - # Lidarr generates these but doesn't include them in trackFile API - # Without this protection cleanup would delete all your artwork -LIDARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this -LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run - # protects against API returning partial data on a bad day -LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count" - # persists last known tracked count for percentage comparison - -# ── Sonarr ──────────────────────────────────────────────────────────────────────────────────── -HOST1_SONARR_URL="http://localhost:8989" -HOST1_SONARR_API_KEY="your-sonarr-api-key" -HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows" - -# Container path → host path translation -# Add one entry per root folder configured in Sonarr Settings → Media Management → Root Folders -# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder -declare -A HOST1_SONARR_PATH_MAP=( - ["/tv"]="/mnt/user/Tv_Shows" - ["/ext-standup-comedy"]="/mnt/user/stand-up_comedy" - ["/kids tv"]="/mnt/user/Kids_Tv_Shows" - ["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old" -) - -HOST2_SONARR_URL="http://localhost:8989" -HOST2_SONARR_API_KEY="your-sonarr-api-key" -HOST2_SONARR_TV_ROOT="/mnt/user/Anime_Shows" - -declare -A HOST2_SONARR_PATH_MAP=( - # Fill in when HOST2 is back online - # ["/tv"]="/mnt/user/Anime_Shows" -) - -SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion -SONARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this -SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") -SONARR_PROTECTED_PATTERNS=( - # Subtitles - "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" - # Metadata - "*.nfo" "*.tbn" - # Images — cover art, posters, fanart, Emby artwork - "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" - "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" - "banner.*" "thumb.*" "landscape.*" - # Kodi/Emby extras — not tracked by Sonarr API - "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" - "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" - "*-clip.*" "*-other.*" - # Theme songs — stored in show folder, not tracked - "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" -) - -# ── Radarr ──────────────────────────────────────────────────────────────────────────────────── -HOST1_RADARR_URL="http://localhost:7878" -HOST1_RADARR_API_KEY="your-radarr-api-key" -HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies" - -# Container path → host path translation -# Add one entry per root folder configured in Radarr Settings → Media Management → Root Folders -# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder -declare -A HOST1_RADARR_PATH_MAP=( - ["/movies"]="/mnt/user/Movies" - ["/kids movies"]="/mnt/user/Kids_Movies" - ["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy" - ["/anime-movies"]="/mnt/user/Anime_Movies-Old" -) - -HOST2_RADARR_URL="http://localhost:7878" -HOST2_RADARR_API_KEY="your-radarr-api-key" -HOST2_RADARR_MOVIES_ROOT="/mnt/user/Anime_Movies" - -declare -A HOST2_RADARR_PATH_MAP=( - # Fill in when HOST2 is back online - # ["/anime-movies"]="/mnt/user/Anime_Movies" -) - -RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion -RADARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this -RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") -RADARR_PROTECTED_PATTERNS=( - # Subtitles - "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" - # Metadata - "*.nfo" "*.tbn" - # Images — cover art, posters, fanart, Emby artwork - "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" - "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" - "banner.*" "thumb.*" "landscape.*" - # Kodi/Emby extras — not tracked by Radarr API - "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" - "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" - "*-clip.*" "*-other.*" - # Theme songs — stored in movie folder, not tracked - "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" -) - -# ━━━ Arr Failed/Stalled Recovery ━━━ -# Auto blocklist + re-search failed imports and stalled downloads. -# Runs every 6 hours — schedule: 0 */6 * * * -# -# Targets four problem types: -# importFailed — downloaded but arr couldn't import -# importPending — downloaded, stuck waiting to import (won't self-resolve) -# error status — serious failure not covered above -# stalled — download stuck with no connections or progress -# -# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first. -# API versions: Sonarr /api/v3/ — Radarr /api/v3/ — Lidarr /api/v1/ -# Lidarr runs on HOST1 only — exits cleanly on HOST2. - -ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this - # gives the arr time to retry on its own before we intervene - # matches cron interval — items are eligible after one missed cycle - -# Per-arr enable/disable toggles — set false to temporarily disable without removing from cron -# Useful if an arr is having issues and you want to skip it for a few runs -HOST1_SONARR_RECOVERY=true # Tv_Shows import recovery -HOST1_RADARR_RECOVERY=true # Movies import recovery -HOST1_LIDARR_RECOVERY=true # Music import recovery — HOST1 only, exits cleanly on HOST2 -HOST2_SONARR_RECOVERY=true # Anime_Shows import recovery -HOST2_RADARR_RECOVERY=true # Anime_Movies import recovery - -# ============================================================================================== -# ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Session-based storage allocator using filesystem symlink indirection. -# ffmpeg resolves the symlink ONCE at session start — existing sessions are never affected. -# -# How it works: -# ramdisk_setup.sh — creates tmpfs and symlink at array start via array_start.sh -# transcode_management.sh — every 3min, runs cleanup then manager in correct order -# transcode_cleanup.sh — removes old inactive files -# transcode_manager.sh — manages symlink direction based on usage thresholds -# -# ⚠️ Docker mount — must use shared propagation: -# --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared -# Standard rprivate bind mounts lock the inode — sessions drift to SSD permanently. - -# ━━━ Transcode Manager ━━━ -# tmpfs mount point — created at array start by ramdisk_setup.sh -# Must exist before Emby starts so the symlink resolves correctly - RAMDISK_PATH="/mnt/ramdisk_transcodes" - -# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront -# Set this to a comfortable limit based on your typical concurrent stream count -# Real-world: 9 streams peaked at ~5.5GB — 8G gives comfortable headroom - RAMDISK_SIZE="8G" - -# Symlink that Emby points at — this path NEVER changes regardless of ramdisk/SSD state -# Emby resolves the symlink once per session at start — symlink flips are transparent -# Must match the container path configured in Emby's Extra Parameters - TRANSCODE_LINK="/mnt/ram-transcode" - -# SSD fallback location — where transcodes land when ramdisk is too full -# Must have enough free space to handle peak session load - TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" - -# Usage thresholds in GB — hysteresis gap between WARN and LOW prevents flip-flop -# RAMDISK_WARN_GB: flip symlink to SSD when ramdisk usage reaches this -# RAMDISK_LOW_GB: flip symlink back to ramdisk when usage drops to this -# Gap (6.8 - 5.5 = 1.3GB) means ramdisk must drop 1.3GB before flipping back -# Without hysteresis a session right at the threshold causes rapid flipping - RAMDISK_WARN_GB=6.8 - RAMDISK_LOW_GB=5.5 - -# Minimum free GB on SSD before allowing a flip to SSD -# Prevents flipping to SSD when it's almost full — that would be worse than a full ramdisk - RAMDISK_SSD_MIN_GB=20 - -# File age thresholds in minutes before cleanup eligibility -# TRANSCODE_MAX_AGE: HLS segment files older than this with no active session = clean up -# TRANSCODE_ORPHAN_AGE: files with no matching session at all = clean up - TRANSCODE_MAX_AGE=20 - TRANSCODE_ORPHAN_AGE=30 - -# Notify if symlink flips this many times in one hour -# Frequent flips indicate the ramdisk is too small or thresholds need adjustment - TRANSCODE_FLIP_WARN=3 - -# Permissions applied to ramdisk and SSD transcode directories - TRANSCODE_OWNER="nobody:users" - TRANSCODE_CHMOD="755" - -# Operating mode — controls symlink direction behaviour -# smart — auto-flips between ramdisk and SSD based on RAMDISK_WARN_GB / RAMDISK_LOW_GB -# hysteresis gap prevents flip-flop — default for production -# ramdisk — always uses ramdisk, never flips to SSD -# warns if RAMDISK_WARN_GB exceeded but holds position -# use during SSD maintenance or when SSD space is low -# ssd — always uses SSD, never flips to ramdisk -# use during ramdisk maintenance or after a ramdisk issue - TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd - -# Daily statistics log — read by weekly_health_digest.sh for transcode summary -# Tracks peak usage, flip count, session ratio, files cleaned per day -# Bounded to TRANSCODE_LOG_RETENTION days — auto-purges old entries on each write - TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db" - TRANSCODE_LOG_RETENTION=90 # days before old entries are purged - -# ━━━ Transcode Server Array ━━━ -# All media servers sharing the ramdisk transcode space. -# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex -# Entries with placeholder API keys are skipped automatically. -# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD. - -TRANSCODE_SERVERS=( - "${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby" - # "${HOST2_EMBY_CONTAINER}|${HOST2_EMBY_URL}|${HOST2_EMBY_API_KEY}|emby" - # "Jellyfin|http://localhost:8097|jellyfin-api-key|jellyfin" - # "Plex|http://localhost:32400|plex-token|plex" -) - - TRANSCODE_CHECK_EMBY=true - -# ============================================================================================== -# ── MONITORS ────────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -# ━━━ Certificate Monitor ━━━ -# Checks SSL certificate expiry via direct openssl connection — no NPM dependency. -# Checks the actual certificate served by each domain, not what NPM thinks it has. -# CERT_WARN_DAYS = notify this many days before expiry -# CERT_CRIT_DAYS = escalate to critical this many days before expiry -# CERT_TIMEOUT = seconds before giving up on the openssl connection -CERT_MONITOR_DOMAINS=( - "yourdomain.com" - "yourdomain.us" -) - CERT_WARN_DAYS=30 # warn when cert expires within this many days - CERT_CRIT_DAYS=7 # critical alert within this many days - CERT_TIMEOUT=10 # seconds per domain check - -# ━━━ Backup Verify ━━━ -# Verifies rsync mirror health by comparing random file checksums between servers. -# Catches silent corruption or incomplete syncs that rsync itself wouldn't detect. -# Leave BACKUP_VERIFY_SHARES empty to use HOST*_DAILY_SYNC_SHARES automatically. -# BACKUP_VERIFY_SAMPLE = number of random files to checksum per share -# BACKUP_VERIFY_MIN_SIZE = skip files smaller than this (small files are rarely corrupted) -BACKUP_VERIFY_SHARES=( - # leave empty to use HOST*_DAILY_SYNC_SHARES automatically -) - BACKUP_VERIFY_SAMPLE=10 # random files to check per share - BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample - -# ━━━ SMART Health ━━━ -# Monitors drive SMART attributes — discovers all drives automatically via /dev/sd* and /dev/nvme*. -# Reads live SMART data — no persistent writes. -# SMART_IGNORE_DRIVES = drives to skip (boot USB, drives without meaningful SMART data) - SMART_TEMP_WARN=45 # Celsius — warn above this temperature - SMART_TEMP_CRIT=55 # Celsius — critical above this temperature -SMART_IGNORE_DRIVES=( - "sda" # boot USB — SMART not meaningful on flash drives -) - -# ━━━ ZFS Memory Snapshot ━━━ -# Weekly ZFS pool health and memory diagnostic report — informational only, no action taken. -# ZFS_REPORT_ARC_WARN_PCT = warn if ARC is using more than this % of its max -# ZFS_REPORT_FREE_WARN_GB = warn if less than this GB free RAM -# ZFS_REPORT_AVAIL_WARN_GB = warn if less than this GB available on ZFS pool -# ZFS_REPORT_DOCKER_TOP = how many top Docker containers to show by memory usage -# ZFS_REPORT_IGNORE_POOLS = individual disk pools to skip (unRAID array disks as ZFS) - ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log" - ZFS_REPORT_ARC_WARN_PCT=90 - ZFS_REPORT_FREE_WARN_GB=10 - ZFS_REPORT_AVAIL_WARN_GB=20 - ZFS_REPORT_DOCKER_TOP=10 -ZFS_REPORT_IGNORE_POOLS=( - "disk10" - "disk9" - "disk8" - "disk6" - "disk5" -) - -# ━━━ Bandwidth Monitor ━━━ -# Called automatically by rsync.sh after each sync — one bounded write per run. -# Tracks transfer size, duration and profile per sync for weekly summary reporting. -# BANDWIDTH_LOG_RETENTION = days to keep entries before auto-purging old records -# BANDWIDTH_WARN_GB = flag in weekly summary if a single sync exceeded this size - BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db" - BANDWIDTH_LOG_RETENTION=90 # days before old entries are purged - BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report - -# Stats files — written by cleanup and recovery scripts, read by coffee report -# All in DATA_DIR — array always running when these are written - ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats - ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats - -# ━━━ Health Digest ━━━ -# Aggregated system health summary — reads existing state files, no new writes. -# Three profiles control when the digest email is sent: -# always — sends every run regardless of findings -# smart — sends only when DIGEST_SMART_ON_* conditions are found -# weekly — sends once per week on DIGEST_DAY only -# Smart profile triggers — set true to send digest when finding is detected: - DIGEST_PROFILE="weekly" # always | smart | weekly - DIGEST_DAY="Sunday" # day of week for weekly profile - DIGEST_SMART_ON_WATCHDOG=true # send if any watchdog strikes are active - DIGEST_SMART_ON_FAILOVER=true # send if failover state is not NORMAL - DIGEST_SMART_ON_CERT_WARN=true # send if any cert is under CERT_WARN_DAYS - DIGEST_SMART_ON_BANDWIDTH=true # send if any transfer exceeded BANDWIDTH_WARN_GB - -# ━━━ Emby Session Report ━━━ -# Weekly Emby usage statistics via API — no persistent writes, queries fresh each run. -# Shows top content, most active users, session counts over the report period. -# URL and API key pulled from HOST1/HOST2_EMBY_URL and HOST1/HOST2_EMBY_API_KEY -# defined in Host Configuration at the top of this file — no duplication needed. - EMBY_REPORT_DAYS=7 # days to include in the report period - EMBY_REPORT_TOP_N=10 # number of top content items to show - -# ============================================================================================== -# ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Continuous system health monitoring — last line of defense before a crash. -# Started by array_start.sh — runs until array stops. -# Re-sources Master.conf each cycle — config changes take effect on next cycle. -# Strike system: sustained threshold hits trigger reboot — single spikes ignored. -# Reboot loop protection: shuts down instead if reboot limit hit in rolling window. -# Silent when healthy — logs only when a threshold is triggered. - -# ━━━ State Files ━━━ - SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" # /tmp resets on reboot ✅ - SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" # survives reboots - SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db" # reboot loop detection - -# ━━━ Strike and Reboot Loop Settings ━━━ -# Strike system: a check must fail this many consecutive cycles before action is taken -# Single spikes (one bad reading) are ignored — sustained problems trigger reboot - SYS_WATCHDOG_STRIKE_LIMIT=2 # consecutive failures before reboot trigger - -# How often checks run — 300s = 5 minutes -# At STRIKE_LIMIT=2 and INTERVAL=300: problem must persist 10min before reboot - SYSTEM_WATCHDOG_INTERVAL=300 # seconds between watchdog cycles - -# Reboot loop protection — if system keeps rebooting something is seriously wrong -# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS hours → shutdown instead of reboot -# Prevents infinite reboot loops when the underlying problem can't be fixed by rebooting - SYS_WATCHDOG_REBOOT_LIMIT=3 # max reboots before shutdown instead - SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 # rolling window in hours - -# Heartbeat — proof of life logged periodically even when everything is healthy - SYSTEM_WATCHDOG_HEARTBEAT=true # true = log heartbeat / false = completely silent - SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1 # how often to log heartbeat (hours) - -# ━━━ Thresholds ━━━ -# Set at "about to become unstable" levels — not "things are a bit high" -# These should be high enough that normal operation never triggers them - -# rootfs (/) usage percentage — when array is down rsync writes land on rootfs -# fills rapidly and can crash the server — 95% is almost too late, act fast - SYS_WATCHDOG_ROOTFS_PCT=95 - -# /var/log usage percentage — log spam can fill rootfs, indicates something broken - SYS_WATCHDOG_LOG_PCT=95 - -# Free RAM in GB — below this is critically low, OOM or swap imminent -# Your server has 128GB — 4GB free means something is consuming everything - SYS_WATCHDOG_MEM_GB=4 - -# ZFS ARC pinned percentage — ARC not releasing after reclaim = memory stuck -# SYS_WATCHDOG_ARC_RELEASE_PCT = after reclaim attempt, if still above this → trigger - SYS_WATCHDOG_ARC_PINNED_PCT=98 - SYS_WATCHDOG_ARC_RELEASE_PCT=95 - -# Load average multiplier — threshold = MULTIPLIER × CPU core count -# MULTIPLIER=3 on 16-core = load average of 48 before triggering -# Set high — transcoding causes legitimate high load spikes - SYS_WATCHDOG_LOAD_MULTIPLIER=3 - -# Zombie process count — large numbers indicate serious process management failure -# A few zombies are normal — 50 means something is very wrong - SYS_WATCHDOG_ZOMBIE_LIMIT=50 - -# CPU temperature in Celsius — sustained high temp causes throttling or kernel panic -# 95°C is close to tjmax on most CPUs — triggers before thermal shutdown - SYS_WATCHDOG_CPU_TEMP_MAX=95 - -# ━━━ Check Toggles ━━━ -# Disable individual checks without disabling the whole watchdog -# All enabled by default except load — transcoding causes legitimate load spikes - SYS_WATCHDOG_CHECK_ROOTFS=true - SYS_WATCHDOG_CHECK_LOG=true - SYS_WATCHDOG_CHECK_RAM=true - SYS_WATCHDOG_CHECK_ARC=true - SYS_WATCHDOG_CHECK_CPU_TEMP=true - SYS_WATCHDOG_CHECK_LOAD=false # disabled — load spikes during transcoding are normal - SYS_WATCHDOG_CHECK_ZOMBIES=true - SYS_WATCHDOG_CHECK_CONTAINERS=true # checks docker_watchdog persistent skip list - SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true # checks if Docker daemon is responding - -# ━━━ Abort Toggles ━━━ -# Conditions that prevent reboot even when a threshold is hit -# true = abort reboot if this condition is active (conservative — avoid data loss) -# false = reboot anyway (aggressive — a clean reboot beats a hard crash) -# Philosophy: aborting is safer for data, rebooting is safer for stability - SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # rebooting with bad pool risks data loss - SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity is better than crashing mid-check - SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting move is better than crashing mid-move - -# ============================================================================================== -# ──────────────────────── End Of User Variables ─────────────────────────────────────────────── -# ============================================================================================== - - - - - - - -important_sync_maintenance.sh should be a new orch and follow the same structure in master as our other orchs. it should trigger the important rsync and trigger, arrs_failed_stalled. any thing else it should trigger \ No newline at end of file diff --git a/unRAID_Essentials/README-Unraid_Essentials.md b/unRAID_Essentials/README-Unraid_Essentials.md index d8b830b..7e94d1e 100644 --- a/unRAID_Essentials/README-Unraid_Essentials.md +++ b/unRAID_Essentials/README-Unraid_Essentials.md @@ -1,319 +1,807 @@ -# unRAID Essentials +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# 🖥️ UNRAID ESSENTIALS +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -System-level scripts that act on the unRAID server itself — not containers, not media, not monitoring. These scripts keep the server healthy, respond to problems, and handle graceful shutdown and restart sequences. +**System-level scripts that act on the unRAID server itself — not containers, not +media, not monitoring.** Keeping the server stable under load, recovering a frozen +WebGUI, tuning kernel limits, suppressing log noise, and handling graceful shutdowns +and reboots with proper warning sequences. ``` -Monitors/ — observes and reports -Docker_Essentials/ — acts on containers -unRAID_Essentials/ — acts on the server itself (this folder) +unRAID_Essentials/ ← acts on the server itself (this folder) +Docker_Essentials/ ← acts on containers +Media/ ← acts on the library +Monitors/ ← observes and reports ``` -Most scripts in this folder are either scheduled at array start or run on a weekly maintenance schedule. A few are utilities called manually or by other scripts. All support `--dry-run`. +> **The escalation chain matters here.** Docker_Essentials handles container-level +> problems. unRAID_Essentials handles server-level problems. The watchdogs are +> designed to work together — docker_watchdog.sh heals containers first, +> system_watchdog.sh reboots only when healing has failed. --- -## Scripts +## ━━━ THE PROBLEM THAT BUILT THIS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -### `system_watchdog.sh` +--- -**The last line of defense.** Reboots the system cleanly when it is about to become unstable. Works alongside `docker_watchdog.sh` — containers are the first line of healing, system reboot is the last resort. +### 🔴 Server Getting Into Unstable States With No Recovery Path -```bash -# Scheduled as: */15 * * * * (every 15 minutes) -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/system_watchdog.sh +A container has a severe memory leak. RAM drops to 2GB. The system starts swapping. +Everything slows down. Docker watchdog tries to restart the container — but Docker +itself is barely responding. The restart hangs. The watchdog is stuck. Nothing is +getting better. The server needs a reboot, but nothing in the ecosystem is authorised +to call one. + +Or: rootfs fills to 99%. SSH stops working. Docker can't write log files. The WebGUI +shows nothing useful. The server is functionally dead but still technically running. +Again — needs a reboot, nothing calls one. + +The fix: `system_watchdog.sh` — the last line of defense. Three-tier response: +immediate reboot on critical failures, OOM-confirmed bypass for RAM crises, and a +strike system for sustained threshold breaches. When everything else has failed, +system_watchdog reboots cleanly before a hard crash happens. + +--- + +### 🔴 WebGUI Freezing and Nobody Noticing + +The WebGUI becomes unresponsive. Nginx gets into a bad state. Or PHP-FPM workers are +saturated and new requests are queueing indefinitely. Or emhttp itself has frozen. +From a user perspective: dashboard doesn't load, settings don't save, containers +can't be started or stopped via the UI. + +Nothing in the container stack alerts on a frozen WebGUI — it's not a container +problem, it's a web server problem. The only way to know is if someone tries to use +the UI and notices. By which point it may have been broken for hours. + +The fix: `webgui_restart.sh` — checks every 10 minutes, escalates through nginx → +php-fpm → emhttp. Lightest fix first. Notifies on any restart so you know it happened. +Silent when healthy — 144 runs per day with no output is the correct behaviour. + +--- + +### 🔴 50+ Containers Starting and Filling Syslog With Veth Noise + +Array starts. 50+ containers come up simultaneously. Docker creates a virtual network +interface for each one. Each creation generates multiple syslog entries. In the first +few minutes after array start the syslog is buried under hundreds of lines of: + +``` +kernel: veth2a3b4c5: renamed from eth0 +kernel: docker0: port 1(veth2a3b4c5) entered blocking state +kernel: docker0: port 1(veth2a3b4c5) entered forwarding state ``` -**Relationship with docker_watchdog.sh:** +Real events — a failed mount, a permission error, a service that didn't start — +are invisible in this noise. And on a busy server that restarts containers regularly, +this noise continues throughout the day. + +The fix: `docker_syslog_filter.sh` — creates an rsyslog drop rule before any +container starts. Applied at array start, idempotent, silent when already correct. + +--- + +### 🔴 WebGUI Queueing Requests Under Load Without Explanation + +The WebGUI feels slow. Clicking a button takes 5 seconds. Saving settings seems to +hang. Nothing in the logs explains it. Container starts from the UI timeout. The +server itself is not under load — CPU is fine, RAM is fine. + +The cause: PHP-FPM's `pm.max_children` defaults to 4-8 workers. On a server with +multiple users, active plugins, automated tools polling the API, and 50+ containers +all potentially hitting the WebGUI simultaneously, those 4-8 workers saturate +immediately. New requests queue behind active ones. Everything feels slow. + +The fix: `php_fpm_max_children.sh` — sets `pm.max_children=250` at array start. +250 workers × ~40MB = ~10GB worst case. On 128GB this is trivially small. The +WebGUI becomes responsive immediately. Idempotent — silent when already correct. + +--- + +### 🔴 inotify Exhaustion Producing Unexplained Failures + +Already documented in README-Monitors.md (system_tuning_monitor.sh section). Short +version: when inotify limits are exhausted, containers silently stop receiving file +system events. Downloads complete but arrs don't detect them. The kernel hits the +limit and new watches fail silently. VSCode shows "unable to watch for file changes" +and misses edits. + +The fix: `inotify_tuning.sh` — raises all three inotify limits at array start. +1M watches (raised from 512K — Code-Server with node_modules needs this), 1024 +instances, 32768 queued events. The startup race note: if Code-Server starts before +this runs, it inherits old limits. Restart Code-Server if the VSCode error appears +after limits are applied. + +--- + +### 🔴 Mover Getting Killed Mid-Transfer Leaving Files Inconsistent + +The mover is running — moving a large batch of files from cache to array. Someone +clicks reboot from the UI. Or a script kills the mover process directly. The mover +stops mid-file. The file exists partially on both cache and array simultaneously. +unRAID's deduplication layer is confused. The file is inaccessible. + +The fix: `mover_stop.sh` — warns logged-in users via wall message, waits the +configured timeout, then sends SIGTERM (graceful) and verifies. The mover gets to +finish its current file operation before stopping. SIGKILL is a last resort with a +warning that partial files may exist. + +--- + +## ━━━ WHAT THIS FOLDER DOES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Four distinct server-level roles: + ``` -docker_watchdog.sh — container level, tries to self-heal first - memory limits, CPU, HTTP checks, restarts - -system_watchdog.sh — system level, acts when healing has failed - reboots when the server itself is unstable -``` - -**What it checks** (all individually toggleable in `Master.conf`): - -| Check | Threshold | Why It Matters | -|-------|-----------|----------------| -| rootfs usage | 95% | Fills rapidly when array is down — crash imminent | -| /var/log usage | 95% | Log spam filling rootfs — indicates something broken | -| Free RAM | 4GB | Critically low RAM means OOM kills or swap imminent | -| ZFS ARC pinned | 98% | ARC not releasing after reclaim — memory stuck | -| CPU temperature | 95°C | Sustained tjmax causes throttling or kernel panic | -| Load average | cores × 3 | Sustained high load — something stuck or runaway | -| Zombie processes | 50 | Large zombie count — serious process management failure | -| Docker daemon | responsive | Unresponsive daemon means containers cannot be managed | -| Required containers | running | Stopped containers the watchdog couldn't recover | - -**Strike system:** - -Checks use a strike system — a single spike doesn't trigger a reboot. The threshold must be hit on consecutive cycles. `SYS_WATCHDOG_STRIKE_LIMIT=2` means two consecutive 15-minute cycles above the threshold before acting. Single spikes are ignored. - -**Abort conditions:** - -Some conditions prevent a reboot even if thresholds are hit: - -| Condition | Default | Reason | -|-----------|---------|--------| -| ZFS pool unhealthy | abort | Rebooting with bad pool risks data loss | -| Parity running | reboot anyway | Aborting parity beats crashing mid-check | -| Mover running | reboot anyway | Aborting move beats crashing mid-move | - -Set `true` to abort reboot if condition is active. Set `false` to reboot regardless. Philosophy: a graceful reboot before a crash is always better than a hard crash mid-operation. - -**Reboot loop protection:** - -Tracks reboot timestamps in a persistent log on `/boot/` — survives reboots. If the server reboots `SYS_WATCHDOG_REBOOT_LIMIT` times within `SYS_WATCHDOG_REBOOT_WINDOW_HRS` hours it shuts down instead. A reboot loop means something is fundamentally broken that rebooting is not fixing — shutting down prevents hardware damage and gives you time to investigate. - -**State files:** -``` -/tmp/system_watchdog_state.db — strike counts (resets on reboot) -/boot/config/system_watchdog_failed.db — container skip list (persistent) -/boot/config/system_watchdog_reboots.db — reboot timestamps (persistent) +🛡️ Last-resort stability system_watchdog.sh — reboots before crash +🌐 WebGUI availability webgui_restart.sh — recovers frozen UI +⚙️ Kernel tuning inotify_tuning.sh — file watch limits + php_fpm_max_children.sh — PHP worker count +🔇 Log hygiene docker_syslog_filter.sh — suppress veth noise + clear_logs.sh — weekly log trimming +🔄 Graceful operations mover_stop.sh — clean mover stop + server_reboot.sh — clean reboot with warning + user_scripts_stop.sh — stop running scripts ``` --- -### `webgui_restart.sh` +## ━━━ SCRIPTS AT A GLANCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Monitors the unRAID WebGUI and restarts it automatically if unresponsive. +| Script | Purpose | When | +|--------|---------|------| +| `system_watchdog.sh` | Three-tier last-resort stability watchdog | Continuous background loop | +| `webgui_restart.sh` | WebGUI availability — nginx → php-fpm → emhttp escalation | Every 10 minutes | +| `inotify_tuning.sh` | Raise inotify kernel limits | At array start | +| `php_fpm_max_children.sh` | Set PHP-FPM worker count | At array start | +| `docker_syslog_filter.sh` | Suppress Docker veth log noise | At array start | +| `clear_logs.sh` | Size-threshold weekly log cleanup | Weekly via maintenance window | +| `mover_stop.sh` | Clean mover stop with SIGTERM → SIGKILL | Manual | +| `server_reboot.sh` | Graceful reboot with pre-flight warnings | Manual or called by system_watchdog | +| `user_scripts_stop.sh` | Stop all running User Script processes | Manual or called by server_reboot | + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🛡️ system_watchdog.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +The last line of defense. Reboots the system cleanly before it crashes uncleanly. +Three-tier response system — critical failures bypass everything and reboot immediately, +OOM-confirmed crises bypass the strike system, sustained threshold breaches use strikes. +Runs continuously as a background process started by `array_start.sh`. + +> Full architecture documentation in `README-Docker_Essentials.md` under +> "Relationship to System Watchdog". This section covers the system_watchdog itself. ```bash -# Scheduled as: */10 * * * * (every 10 minutes) -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/webgui_restart.sh -``` - -**Escalation path:** -``` -1. curl WebGUI → unresponsive -2. Restart nginx → wait WEBGUI_NGINX_WAIT seconds → recheck -3. Still unresponsive → restart emhttp → wait WEBGUI_EMHTTP_WAIT seconds → recheck -4. Still unresponsive → notify warning — manual intervention needed -``` - -**Why emhttp is more disruptive:** - -`nginx` is the web server layer — restarting it is fast and clean. `emhttp` is the core unRAID management daemon — it manages the array, Docker, VMs, and everything else. Restarting it takes longer but recovers cleanly. The escalation path tries the less disruptive option first. - -A notification is sent on any restart — nginx or emhttp — so you know what happened and when. - -**Configuration:** -```bash -WEBGUI_URL="http://localhost" # adjust if running non-standard port -WEBGUI_TIMEOUT=5 # seconds before curl gives up -WEBGUI_NGINX_WAIT=15 # wait after nginx restart before recheck -WEBGUI_EMHTTP_WAIT=30 # wait after emhttp restart — takes longer +# Started by: array_start.sh (continuous background process) +# Interval: SYSTEM_WATCHDOG_INTERVAL=300 (5 minutes) ``` --- -### `docker_syslog_filter.sh` - -Suppresses noisy Docker network interface messages from the unRAID syslog. +### ── Three-Tier Response System ────────────────────────────────────────────── ```bash -# Scheduled as: At Startup of Array -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/docker_syslog_filter.sh -``` +# ───────────────────────────────────────────────────────────────────────────── +# TIER 1 — CRITICAL (bypass ALL strikes, reboot immediately) +# These failures are acute — the system is not recoverable by waiting. +# Single detection = immediate reboot. No confirmation window. +# +# Docker daemon unresponsive: +# Attempt /etc/rc.d/rc.docker restart first. +# Wait 15 seconds. Verify daemon responding. +# If still hung → CRITICAL reboot. +# A hung daemon cannot be healed — every subsequent docker command hangs. +# +# rootfs at ROOTFS_CRITICAL_PCT (99%+): +# Writes are failing. SSH may stop. Logs can't be written. +# Nothing can be fixed from this state without a reboot. +# +# Kernel BUG/Oops in dmesg: +# Kernel running with corrupted state. +# Delta-based: new oops since last cycle → reboot. +# +# File descriptor exhaustion (FD_CRITICAL_PCT=95%): +# New connections failing. Docker can't spawn processes. SSH drops. +# +# /boot read-only: +# State files and config writes silently failing. +# Write test on /boot every cycle. -**The problem it solves:** +# TIER 2 — URGENT (bypass strikes when OOM confirms crisis) +# RAM < MEM_GB (4GB) AND OOM kills >= OOM_LIMIT (3) in this cycle +# Without OOM confirmation → standard strike system applies. +# Rationale: 1-2 OOM kills = docker_watchdog.sh handles it. +# 3+ kills while RAM critical = system dying faster than watchdogs heal. +# OOM victims from dmesg included in reboot message (diagnostic context). -Every time Docker starts a container it creates virtual network interfaces (`veth` devices). Every time a container stops, they're removed. Each creation and removal generates syslog entries. On a server with 50+ containers starting at array start this creates hundreds of lines of noise that buries real log messages. - -The filter creates an rsyslog configuration file that suppresses these specific messages and restarts rsyslog to apply it. The filter file is recreated on every array start — rsyslog configuration doesn't survive reboots on unRAID. - -**Configuration:** -```bash -FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" +# TIER 3 — STANDARD (strike system — N consecutive failures → reboot) +# Everything else: RAM tiers, load, CPU temp, zombies, /var/log, +# /tmp, containers, NIC, mdstat, sshd +# ───────────────────────────────────────────────────────────────────────────── ``` --- -### `php_fpm_max_children.sh` - -Sets the PHP-FPM `pm.max_children` value to allow more concurrent requests to the unRAID WebGUI. +### ── RAM Tiers ──────────────────────────────────────────────────────────────── ```bash -# Scheduled as: At Startup of Array -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/php_fpm_max_children.sh -``` - -**Why this is needed:** - -unRAID's WebGUI is served via PHP-FPM. The default `pm.max_children` value is conservative. On a server with many users, plugins, or automated tools hitting the API simultaneously, the default value can cause requests to queue or time out. Increasing it allows more concurrent PHP processes. - -The setting does not survive reboots — PHP-FPM configuration is reset on each boot. Running this at array start ensures it's always applied. - -**Configuration:** -```bash -PHP_CONF="/etc/php-fpm.d/www.conf" -PHP_MAX_CHILDREN=250 # set based on available RAM - # each PHP worker uses ~30-50MB - # 250 workers × 40MB = ~10GB worst case -``` - -Set `PHP_MAX_CHILDREN` based on your available RAM. Higher values allow more concurrency but use more memory if all workers are active simultaneously. - ---- - -### `clear_logs.sh` - -Clears unRAID system log files and Docker container logs. - -```bash -# Scheduled as: 0 5 * * 0 (Sunday 5am weekly) -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/clear_logs.sh -``` - -**Why weekly log clearing:** - -unRAID system logs live in RAM (`/var/log/`) — they don't persist across reboots. However on a stable server that runs for weeks without rebooting, these logs grow continuously. Docker container logs can grow particularly large if a container is verbose. Both can fill rootfs if left unchecked. - -Weekly clearing keeps rootfs clean without being too aggressive. If you need to investigate a problem the week's logs are still available. - -**Configuration:** -```bash -LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) -# Docker container logs are cleared automatically — no configuration needed +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Three-level graduated response — not a single threshold. +# + SYS_WATCHDOG_MEM_WARN_GB=10 # warn + notify once — informational only + SYS_WATCHDOG_MEM_SHUTDOWN_GB=6 # stop non-essential containers, wait for recovery + SYS_WATCHDOG_MEM_GB=4 # strike system → reboot (or bypass if OOM confirms) + SYS_WATCHDOG_MEM_RECOVER_GB=30 # RAM must reach this before restarting containers +# +# Containers excluded from RAM emergency shutdown — these stay running: + SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED=( + "NginxProxyManager" # external access — stop this and users lose everything + "Authelia" # auth — without this nothing is accessible + "Mariadb" # Authelia dependency + "Redis" # Authelia dependency + "Emby" # media server — Live TV buffering + "Dispatcharr" # Live TV scheduler — loses state if stopped + ) +# +# Coordination with docker_watchdog.sh: +# system_watchdog writes mem_shutdown_active=true to SYS_WATCHDOG_STATE_FILE. +# docker_watchdog reads this flag and defers ALL container restart logic. +# Without this: both watchdogs fight — system_watchdog stops containers, +# docker_watchdog restarts them, RAM never recovers. +# With this: docker_watchdog stands down until mem_shutdown_active clears. +# ───────────────────────────────────────────────────────────────────────────── ``` --- -### `mover_stop.sh` - -Safely stops the unRAID mover with a warning delay before terminating. +### ── Per-Host Check Toggles ─────────────────────────────────────────────────── ```bash -# Run manually when needed — not scheduled -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/mover_stop.sh -``` +# master_host1.conf (all checks in master_host*.conf — not master.conf) +# ───────────────────────────────────────────────────────────────────────────── +# Each check is independently toggleable per server. +# HOST1 and HOST2 may have different hardware and different workloads. +# detect_hosts() aliases HOST*_SYS_WATCHDOG_CHECK_* → SYS_WATCHDOG_CHECK_* +# +# Tier 1 — Critical (bypass strikes): + HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true + HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true + HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true + HOST1_SYS_WATCHDOG_CHECK_FD=true + HOST1_SYS_WATCHDOG_CHECK_BOOT=true -**When you need this:** +# Tier 2 — Urgent (OOM bypass): + HOST1_SYS_WATCHDOG_CHECK_OOM=true + HOST1_SYS_WATCHDOG_CHECK_RAM=true -The mover moves files from the cache pool to the array. If you need to stop it mid-run — before an array stop, before maintenance, or because it's been running too long — killing it directly can leave files in an inconsistent state. This script warns first and gives the mover time to finish its current file operation cleanly. - -**Configuration:** -```bash -MOVER_STOP_TIMEOUT=300 # seconds to wait before sending SIGTERM - # gives mover time to finish current file +# Tier 3 — Standard (strike system): + HOST1_SYS_WATCHDOG_CHECK_LOG=true + HOST1_SYS_WATCHDOG_CHECK_ARC=true # HOST1 runs ZFS — enable + HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true + HOST1_SYS_WATCHDOG_CHECK_LOAD=false # disabled — transcoding causes normal spikes + HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true + HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true # monitors docker_watchdog persistent skip list + HOST1_SYS_WATCHDOG_CHECK_TMP=true + HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true + HOST1_SYS_WATCHDOG_CHECK_NETWORK=true + HOST1_SYS_WATCHDOG_CHECK_SSHD=true + HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false # disabled — may false positive during encoding + HOST1_SYS_WATCHDOG_NIC="eth0" # verify: ip link show | grep "^[0-9]" ``` --- -### `rsync_stop.sh` - -Stops all running rsync processes on both local and remote servers. +### ── Abort Conditions ───────────────────────────────────────────────────────── ```bash -# Run manually when needed — not scheduled -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/rsync_stop.sh -``` - -**When you need this:** - -An rsync job may need to be stopped — before an array stop, because it's consuming too much bandwidth, or because it started at the wrong time. Simply killing rsync can leave containers stopped on the remote server (since rsync.sh stops containers before syncing and restarts them after). - -This script: -1. Kills rsync processes locally -2. Kills rsync processes on the remote via SSH -3. Checks all configured profile containers locally -4. Restarts any containers that were left stopped by the interrupted rsync - -The remote is left in whatever container state it was in — the remote server's watchdog handles recovery on its own. - ---- - -### `server_reboot.sh` - -Gracefully reboots the unRAID server with a warning delay. - -```bash -# Run manually when needed — not typically scheduled -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/server_reboot.sh -``` - -**Sequence:** -``` -1. Broadcast warning to all logged-in users -2. Wait REBOOT_SLEEP seconds -3. Stop Docker gracefully -4. Stop VM Manager gracefully -5. Issue reboot -``` - -The warning gives users time to finish what they are doing — saving files in VS Code Server, finishing a download, wrapping up a session. The Docker and VM stop ensures containers and VMs shut down cleanly rather than being hard-killed by the reboot. - -**Configuration:** -```bash -REBOOT_SLEEP=300 # seconds of warning before reboot (default: 5 minutes) -``` - -**Note:** `system_watchdog.sh` calls this script automatically when thresholds are exceeded. You can also call it manually for planned maintenance. - ---- - -### `user_script_stop.sh` - -Stops all running User Scripts plugin jobs. - -```bash -# Run manually when needed — not typically scheduled -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/user_script_stop.sh -``` - -**When you need this:** - -Before stopping the array, before a reboot, or when a script has hung and needs to be cleared. The User Scripts plugin runs scripts in `/tmp/user.scripts/` — this script identifies all running processes with that path signature and terminates them cleanly. - -Useful before `server_reboot.sh` to ensure no scripts are mid-execution when the reboot happens. - ---- - -## Startup Sequence - -The recommended array start sequence for scripts in this folder: - -```bash -# At Startup of Array — in this order -ramdisk_setup.sh # Transcodes/ — creates ramdisk before anything uses it -docker_syslog_filter.sh # suppress veth noise before containers start -php_fpm_max_children.sh # WebGUI performance before anyone accesses it -docker_network_connect.sh # Docker_Essentials/ — connect containers to networks -``` - -These run once at array start. The order matters — ramdisk before containers, filter before logs fill with noise, PHP config before WebGUI requests arrive. - ---- - -## Scheduled Maintenance Summary - -```bash -# At Startup of Array -docker_syslog_filter.sh -php_fpm_max_children.sh - -# Every 10 minutes -*/10 * * * * webgui_restart.sh - -# Every 15 minutes -*/15 * * * * system_watchdog.sh - -# Weekly — Sunday morning -0 5 * * 0 clear_logs.sh +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Conditions that prevent a reboot even when a threshold is hit. +# CRITICAL tier bypasses these — truly critical conditions reboot regardless. +# + SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # rebooting with bad pool risks data loss + SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity is better than crashing + SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting mover is better than crashing +# +# Philosophy: a graceful reboot before a crash is always better than a hard crash. +# The abort conditions protect against the cases where a reboot itself causes harm +# (data loss from bad ZFS pool). Parity and mover can be restarted after reboot. ``` --- -## --dry-run Support - -Every script in this folder supports `--dry-run`. Always test before scheduling: +### ── Reboot Loop Protection ─────────────────────────────────────────────────── ```bash -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/system_watchdog.sh --dry-run -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/webgui_restart.sh --dry-run -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/clear_logs.sh --dry-run -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/server_reboot.sh --dry-run +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# If the server keeps rebooting, something is wrong that rebooting isn't fixing. +# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS → shutdown instead. +# Shutdown prevents: hardware damage, filesystem corruption from repeated reboots, +# infinite loop that never lets you investigate. +# State file: /boot/config/system_watchdog_reboots.db — survives reboots. +# + SYS_WATCHDOG_REBOOT_LIMIT=3 # reboots before shutdown instead + SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 # rolling window in hours ``` -`server_reboot.sh --dry-run` is particularly useful — it walks through the entire shutdown sequence, shows what would be stopped, and exits without rebooting. \ No newline at end of file +--- + +### ── State File Heartbeat ───────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# system_watchdog writes watchdog_cycle=N to SYS_WATCHDOG_STATE_FILE every cycle. +# This keeps the file's modification time current. +# +# docker_watchdog.sh uses the state file mtime as a stale guard — if the file +# is more than 2 hours old while mem_shutdown_active=true is set, system_watchdog +# may have stopped running. docker_watchdog resumes normal operation rather than +# being silenced indefinitely by a stale flag. +# +# Without this heartbeat: if all standard checks pass and no state writes happen +# (e.g. CHECK_KERNEL_OOPS=false AND CHECK_MDSTAT=false), the file mtime could go +# stale even with the watchdog running. +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +system_watchdog.sh # normal (continuous — started by array_start.sh) +system_watchdog.sh --dry-run # trigger detection without rebooting +system_watchdog.sh --status # show all tiers, thresholds, active strikes, RAM state +system_watchdog.sh --log # verbose per-cycle output +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🌐 webgui_restart.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Monitors the unRAID WebGUI availability and recovers it automatically when unresponsive. +Three-step escalating strategy — lightest fix first, heaviest last. Silent when healthy. + +```bash +# Scheduled: */10 * * * * (every 10 minutes) +``` + +--- + +### ── Three-Step Escalation ──────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Step 1 — nginx restart +# Lightest fix — handles most WebGUI failures. +# nginx is the web server layer. Crash, worker stuck, connection timeout. +# Wait WEBGUI_NGINX_WAIT seconds → curl recheck. +# +# Step 2 — php-fpm restart +# Added because WebGUI can appear frozen due to PHP worker exhaustion. +# pm.max_children workers all occupied → new requests queue → dashboard hangs. +# php-fpm restart far less disruptive than emhttp. +# Wait WEBGUI_PHP_WAIT seconds → curl recheck. +# Note: system_tuning_monitor.sh tracks worker saturation over time. +# +# Step 3 — emhttp restart +# Heaviest fix. emhttp is the core unRAID management daemon. +# Array, Docker, shares continue running — only WebGUI management restarts. +# Takes longer to recover — WEBGUI_EMHTTP_WAIT gives it time. +# Wait WEBGUI_EMHTTP_WAIT seconds → curl recheck. +# +# If all three fail → notify warning — manual intervention needed. +# Guidance: pgrep -x nginx emhttp | check journalctl | consider server_reboot.sh +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Configuration ──────────────────────────────────────────────────────────── + +```bash +# master.conf + WEBGUI_URL="http://localhost" # adjust if non-standard port + WEBGUI_TIMEOUT=5 # seconds before curl times out + WEBGUI_NGINX_WAIT=15 # seconds after nginx restart before recheck + WEBGUI_PHP_WAIT=10 # seconds after php-fpm restart before recheck + WEBGUI_EMHTTP_WAIT=30 # seconds after emhttp restart — takes longer +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +webgui_restart.sh # check once — silent if healthy, escalates if not +webgui_restart.sh --dry-run # walk through escalation without restarting anything +webgui_restart.sh --status # show WebGUI state + nginx/php-fpm/emhttp process state +webgui_restart.sh --log # verbose — show each escalation step +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 📡 inotify_tuning.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Raises Linux inotify kernel limits at array start. Idempotent — completely silent +when values are already correct. + +```bash +# Scheduled: At Startup of Array (via array_start.sh — FIRST in ARRAY_START_SCRIPTS) +# Must run before containers start — containers inherit limits at startup +``` + +--- + +### ── Three Limits ───────────────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── + INOTIFY_MAX_INSTANCES=1024 # default: 128 — max inotify instances per user + # 1024 handles ~20-30 containers watching files + + INOTIFY_MAX_WATCHES=1048576 # default: 8192 — SHARED budget across ALL users/containers + # Was 512K — raised to 1M (1048576) + # VSCode/Code-Server alone needs ~50K-200K for large workspaces + # with node_modules. All arr containers + Emby + VSCode share + # this budget. 1M safe on 128GB RAM (~128MB kernel memory) + # If VSCode shows "unable to watch for file changes" → too low + + INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — events buffered before dropping +``` + +--- + +### ── Startup Race Condition ─────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# inotify limits are kernel-wide — they take effect immediately on sysctl write. +# But containers that have ALREADY started inherit the OLD limits at startup. +# Those containers keep their inherited (low) limits until restarted. +# +# This is why inotify_tuning.sh must be FIRST in ARRAY_START_SCRIPTS — before +# any container starts. If Code-Server starts before limits are raised: +# → Code-Server inherits old 8192 watch limit +# → VSCode shows "unable to watch for file changes" +# → Fix: docker restart Code-Server (picks up current kernel limits on start) +# +# The script warns if it changed any values: +# "If Code-Server is running: docker restart Code-Server" +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +inotify_tuning.sh # normal run (idempotent — silent when correct) +inotify_tuning.sh --dry-run # show what would change without changing +inotify_tuning.sh --status # current values vs targets + top inotify consumers +inotify_tuning.sh --log # verbose — show each sysctl write +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ⚙️ php_fpm_max_children.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Raises PHP-FPM `pm.max_children` at array start to prevent WebGUI queueing under +load. Idempotent — completely silent when already correct. No PHP-FPM restart unless +value actually changed. + +```bash +# Scheduled: At Startup of Array (via array_start.sh) +``` + +--- + +### ── Why 250 Workers ────────────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── + PHP_MAX_CHILDREN=250 # default: 4-8 — far too low for a busy server + # Each worker: ~2MB resident memory at idle + # 250 workers × 2MB = ~500MB — trivial on 128GB + # Worst case (all active): ~250 × 40MB = ~10GB + # In practice: rarely all active simultaneously + # On 64GB (HOST2): still appropriate — 250 × 40MB + # = 10GB worst case = 15% of RAM, acceptable +# +# Without this fix: +# 5 users hit the WebGUI simultaneously → 8 workers exhausted → 5 more queue +# Each queued request waits for a worker to free → 5-10 second response times +# Looks like a slow server — it's just a queue +# + PHP_CONF="/etc/php83/php-fpm.d/www.conf" # path may change with PHP version +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +php_fpm_max_children.sh # normal run (idempotent) +php_fpm_max_children.sh --dry-run # show what would change +php_fpm_max_children.sh --status # show current value vs target + worker count +php_fpm_max_children.sh --log # verbose — show config write + restart +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔇 docker_syslog_filter.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Creates an rsyslog drop rule for Docker veth/docker0 interface noise. Idempotent — +silent when filter already correct, only writes + restarts rsyslog when something changed. + +```bash +# Scheduled: At Startup of Array (via array_start.sh — before containers start) +``` + +--- + +### ── What Gets Suppressed ───────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# These kernel messages are generated on every container start and stop: +# +# kernel: veth2a3b4c5: renamed from eth0 +# kernel: docker0: port 1(veth2a3b4c5) entered blocking state +# kernel: docker0: port 1(veth2a3b4c5) entered forwarding state +# kernel: docker0: port 1(veth2a3b4c5) entered disabled state +# +# 50+ containers at array start = 200-400 lines of this in the first minute. +# Containers restart throughout the day = continuous noise. +# Real events buried and invisible in syslog. +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Idempotent Design ──────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# Runs at every array start but only changes something when needed: +# Filter file exists and content is correct → exit 0 silently +# Filter file missing or content changed → write + restart rsyslog +# +# Expected content compared exactly — single source of truth: +EXPECTED_FILTER='if ($msg contains "veth" or $msg contains "docker0") then { + stop +}' +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +docker_syslog_filter.sh # normal run (idempotent — silent when correct) +docker_syslog_filter.sh --dry-run # show what would be written without writing +docker_syslog_filter.sh --status # show current filter file + rsyslog state +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🗑️ clear_logs.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Clears system logs and Docker container logs using size thresholds. Only clears logs +large enough to be worth clearing — preserves recent diagnostic context on small logs. + +```bash +# Called by: weekly_sync_maintenance.sh via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am) +``` + +--- + +### ── Size Threshold Approach ────────────────────────────────────────────────── + +```bash +# master.conf +# ───────────────────────────────────────────────────────────────────────────── +# Why thresholds instead of clearing everything: +# A 2MB syslog contains useful recent history — not worth clearing. +# A 500MB Docker log is consuming rootfs and contains mostly noise — clear it. +# Blind truncation destroys diagnostic context for no benefit. +# + LOG_MIN_SIZE_MB=10 # skip system log if under this size — keep history + LOG_DOCKER_MAX_MB=100 # clear Docker container log only if over this size + # Active containers (Emby, SABnzbd) grow fastest + # 100MB × 30 containers = 3GB before any clearing kicks in + + LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) +# +# Truncation not logrotate: +# unRAID writes to tmpfs (/var/log) — logrotate's compress + archive approach +# would consume more tmpfs space, not less. +# : > file keeps the file descriptor valid while emptying content. +# Safe for running services (syslogd continues writing to the same fd). +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +clear_logs.sh # normal run — silent if all logs under threshold +clear_logs.sh --dry-run # show what would be cleared and sizes +clear_logs.sh --status # show current log sizes vs thresholds +clear_logs.sh --log # verbose — show each file evaluated +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ⏹️ mover_stop.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Stops the unRAID mover cleanly — wall warning, configurable timeout, SIGTERM → verify +→ SIGKILL sequence. Safe to run when mover is not running — exits cleanly with a log. + +--- + +### ── Stop Sequence ──────────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# 1. Check if mover is running — exit cleanly if not +# 2. Wall message: "HOST1 (unRAID-Gmer4Lfe) — mover stopping in 30s" +# MY_ID included — on shared terminal it's clear which server +# 3. Wait MOVER_STOP_TIMEOUT seconds +# 4. SIGTERM — allows mover to finish its current file before stopping +# No partial files — the mover completes what it's working on +# 5. Wait 5 seconds — verify if stopped +# 6. If still running → SIGKILL (forced) +# Warning: partial files possible — same as a hard crash +# 7. Final verify — error if still running +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +mover_stop.sh # stop mover with configured timeout +mover_stop.sh --dry-run # show what would happen +mover_stop.sh --status # show mover state (running, PID, start time) +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🔁 server_reboot.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Graceful reboot with pre-flight warnings, wall message, unRAID notification, VM +graceful shutdown, then Docker and services stop, sync, reboot. + +> Full documentation in `README-Tools.md` — `server_reboot.sh` section. This is a +> quick reference. + +--- + +### ── Shutdown Sequence ──────────────────────────────────────────────────────── + +```bash +# ───────────────────────────────────────────────────────────────────────────── +# 1. Pre-flight warnings (warn not block): +# rsync running, mover running, active Emby sessions +# Warnings show in summary — you chose to reboot, these are for context +# 2. Wall message + unRAID notification — MY_ID included +# 3. Wait REBOOT_SLEEP seconds (default 30) +# 4. virsh shutdown each VM → wait REBOOT_VM_WAIT seconds for graceful exit +# 5. Stop libvirt (VM Manager) +# 6. Stop Docker service +# 7. sync — flush filesystem buffers +# 8. /sbin/reboot +# ───────────────────────────────────────────────────────────────────────────── +``` + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +server_reboot.sh # reboot with 30s warning +server_reboot.sh --dry-run # full sequence walkthrough without rebooting +server_reboot.sh --status # show running processes that would be affected +server_reboot.sh --reason="maintenance" # include reason in wall + notification +server_reboot.sh --log # verbose per-step output +``` + +--- + +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## 🛑 user_scripts_stop.sh +## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Stops all running User Script processes. Identifies by `/tmp/user.scripts` path +signature. Shows script names not PIDs. SIGTERM → verify → SIGKILL with self-exclusion. + +> Full documentation in `README-Tools.md` — `user_scripts_stop.sh` section. + +--- + +### ── Usage ─────────────────────────────────────────────────────────────────── + +```bash +user_scripts_stop.sh # stop all — SIGTERM → SIGKILL if needed +user_scripts_stop.sh --dry-run # show which scripts would be stopped, by name +user_scripts_stop.sh --status # show running scripts with PID and runtime +user_scripts_stop.sh --log # verbose per-process output +``` + +--- + +## ━━━ STARTUP SEQUENCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +```bash +# master.conf — ARRAY_START_SCRIPTS (order matters) +# ───────────────────────────────────────────────────────────────────────────── +ARRAY_START_SCRIPTS=( + # ── One-shot — run and exit ─────────────────────────────────────────────── + "unRAID_Essentials/inotify_tuning.sh" # FIRST — raise limits before + # containers inherit old values + "unRAID_Essentials/docker_syslog_filter.sh" # SECOND — before containers + # create veth interfaces + "unRAID_Essentials/php_fpm_max_children.sh" # before WebGUI serves requests + "Transcodes/ramdisk_setup.sh" # before Emby starts transcoding + "Docker_Essentials/docker_network_connect.sh" # before watchdogs check states + + # ── Continuous — run until array stops ─────────────────────────────────── + "unRAID_Essentials/system_watchdog.sh" # before docker_watchdog — + # writes state file docker_watchdog reads + "Docker_Essentials/docker_watchdog.sh" # before failover — containers + # must be healthy for failover decisions + "Failover/failover.sh" # last — needs everything stable +) +# ───────────────────────────────────────────────────────────────────────────── +# array_start.sh is the ONLY "At Startup of Array" entry in User Scripts. +# It launches everything above in order. +``` + +--- + +## ━━━ FULL SCHEDULE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +```bash +# At Startup of Array — via array_start.sh: +# inotify_tuning.sh +# docker_syslog_filter.sh +# php_fpm_max_children.sh +# (ramdisk_setup.sh — in Transcodes/) +# system_watchdog.sh (continuous) + +# Every 10 minutes: +*/10 * * * * webgui_restart.sh # silent when healthy — escalates when not + +# Weekly — via weekly_sync_maintenance.sh: +# clear_logs.sh # Sunday 2:30am via WEEKLY_MAINTENANCE_SCRIPTS + +# Manual: +# mover_stop.sh — before array ops that need mover stopped +# server_reboot.sh — planned maintenance reboots +# user_scripts_stop.sh — emergency script stop or pre-reboot cleanup +``` \ No newline at end of file diff --git a/unRAID_Essentials/clear_logs.sh b/unRAID_Essentials/clear_logs.sh index df95d41..65dc1c8 100644 --- a/unRAID_Essentials/clear_logs.sh +++ b/unRAID_Essentials/clear_logs.sh @@ -1,134 +1,254 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Clear Logs Script ------------------------------------------ -# ----------------------------------------------------------------------------------------------- -# Clears unRAID system logs and Docker container logs safely. -# Log file paths are configured in Master.conf under LOG_FILES. -# Supports --dry-run to preview what would be cleared without making changes. -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= Clear Logs ================================================= +# ============================================================================================== +# Clears system and Docker container logs to prevent rootfs fill over time. +# Runs weekly via WEEKLY_MAINTENANCE_SCRIPTS — Sunday 2:30am. +# Uses size thresholds — only clears logs that have grown large enough to matter. +# +# ── WHAT IT CLEARS ──────────────────────────────────────────────────────────────────────────── +# System logs — LOG_FILES from master.conf (/var/log/syslog, messages, dmesg) +# Cleared if size exceeds LOG_MIN_SIZE_MB +# These grow continuously — weekly clearing keeps rootfs healthy +# +# Docker logs — /var/lib/docker/containers/**/*-json.log +# Cleared only if individual container log exceeds LOG_DOCKER_MAX_MB +# Active containers (Emby, SABnzbd) grow fastest — 100MB+ easily +# Inactive containers not cleared — their logs are typically small +# +# ── SIZE THRESHOLD APPROACH ─────────────────────────────────────────────────────────────────── +# Truncating everything blindly destroys useful diagnostic context. +# A 2MB log is not worth clearing — it contains useful recent history. +# A 500MB log is consuming rootfs and contains mostly noise — clear it. +# +# LOG_MIN_SIZE_MB — system logs under this size are left alone +# LOG_DOCKER_MAX_MB — Docker logs under this size are left alone +# +# ── WHY NOT LOGROTATE ───────────────────────────────────────────────────────────────────────── +# unRAID writes to tmpfs (/var/log) — logrotate's compress + archive approach +# would consume even more tmpfs space. Truncation (: > file) keeps the file +# descriptor open and valid while emptying content — safe for running services. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock — prevents concurrent runs corrupting logs +# Root check — truncating system logs requires root +# Size thresholds — only clears logs that have grown large enough +# Byte tracking — reports MB freed for weekly digest +# validate_unraid — notify validated before use +# Silent on clean — small logs = nothing to clear = no output ✅ +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# LOG_FILES — system log paths to check and clear +# LOG_MIN_SIZE_MB — minimum system log size before clearing (default 10MB) +# LOG_DOCKER_MAX_MB — clear Docker log only if above this size (default 100MB) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# clear_logs.sh — normal run (threshold-based clearing) +# clear_logs.sh --dry-run — show what would be cleared and sizes +# clear_logs.sh --status — show current log sizes +# clear_logs.sh --log — verbose output per file +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" - -# ROOT CHECK +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root" + error "Must be run as root — truncating system logs requires root" exit 1 fi -success "Running as root" +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +acquire_lock + +detect_hosts + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be cleared" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" - echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_HEALTH System Logs: ${LOG_FILES[*]}" - echo "$ICON_CONTAINERS Docker Logs: /var/lib/docker/containers" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━ $ICON_SUMMARY LOG STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_GEAR Min system log: ${LOG_MIN_SIZE_MB:-10}MB before clearing" + echo "$ICON_GEAR Max Docker log: ${LOG_DOCKER_MAX_MB:-100}MB before clearing" + echo "" + + echo "━━━ System Logs ━━━" + for f in "${LOG_FILES[@]}"; do + if [[ -f "$f" ]]; then + size=$(du -sh "$f" 2>/dev/null | cut -f1) + size_mb=$(du -sm "$f" 2>/dev/null | cut -f1) + threshold="${LOG_MIN_SIZE_MB:-10}" + if [[ "${size_mb:-0}" -ge "$threshold" ]]; then + echo " $ICON_WARN $f — $size (above ${threshold}MB threshold — would clear)" + else + echo " $ICON_SUCCESS $f — $size (under threshold)" + fi + else + echo " $ICON_SKIP $f — not found" + fi + done + + echo "" + echo "━━━ Docker Logs (top 10 by size) ━━━" + if [[ -d /var/lib/docker/containers ]]; then + find /var/lib/docker/containers/ -name "*-json.log" 2>/dev/null | \ + while IFS= read -r logfile; do + size_mb=$(du -sm "$logfile" 2>/dev/null | cut -f1) + container_id=$(basename "$(dirname "$logfile")" | cut -c1-12) + container_name=$(docker inspect --format '{{.Name}}' "$container_id" \ + 2>/dev/null | tr -d '/' || echo "$container_id") + echo "${size_mb:-0} $container_name $logfile" + done | sort -rn | head -10 | \ + while read -r size_mb name logfile; do + threshold="${LOG_DOCKER_MAX_MB:-100}" + if [[ "$size_mb" -ge "$threshold" ]]; then + echo " $ICON_WARN ${size_mb}MB — $name (above ${threshold}MB — would clear)" + else + echo " $ICON_SUCCESS ${size_mb}MB — $name" + fi + done + else + echo " Docker directory not found" + fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -# ----------------------------------------------------------------------------------------------- -# FUNCTIONS -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Clear Logs ━━━ +# ============================================================================================== +START=$(date +%s) +SYS_CLEARED=0 +SYS_SKIPPED=0 +SYS_BYTES=0 +DOCKER_CLEARED=0 +DOCKER_SKIPPED=0 +DOCKER_BYTES=0 +FAILED=() -# Clears a single log file if it exists. -# Skips with a warning if the file is not found. -clear_file() { - local file="$1" +# ── System Logs ─────────────────────────────────────────────────────────────────────────────── +for logfile in "${LOG_FILES[@]}"; do + if [[ ! -f "$logfile" ]]; then + log "$logfile — not found, skipping" + continue + fi - if [[ ! -f "$file" ]]; then - warn "Not found: $file — skipping" - return + size_bytes=$(stat -c%s "$logfile" 2>/dev/null || echo 0) + size_mb=$(( size_bytes / 1048576 )) + size_h=$(du -sh "$logfile" 2>/dev/null | cut -f1) + threshold="${LOG_MIN_SIZE_MB:-10}" + + if [[ "$size_mb" -lt "$threshold" ]]; then + log "$logfile — ${size_h} (under ${threshold}MB — skipping)" + (( SYS_SKIPPED++ )) + continue fi if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would clear: $file" + warn "DRY RUN — would clear: $logfile (${size_h})" + (( SYS_CLEARED++ )) + SYS_BYTES=$(( SYS_BYTES + size_bytes )) + continue + fi + + if : > "$logfile" 2>/dev/null; then + log "Cleared: $logfile (freed ${size_h})" + (( SYS_CLEARED++ )) + SYS_BYTES=$(( SYS_BYTES + size_bytes )) else - : > "$file" - success "Cleared: $file" + error "Failed to clear: $logfile" + FAILED+=("$logfile") fi -} - -# Finds and clears all Docker container json log files. -# Skips gracefully if Docker directory or log files are not found. -clear_docker_logs() { - if [[ ! -d /var/lib/docker/containers ]]; then - warn "$ICON_CONTAINERS Docker directory not found — skipping" - return - fi - - local files - files=$(find /var/lib/docker/containers/ -name "*-json.log" 2>/dev/null || true) - - if [[ -z "$files" ]]; then - warn "$ICON_CONTAINERS No Docker logs found — skipping" - return - fi - - while IFS= read -r file; do - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would clear Docker log: $file" - else - : > "$file" - success "Cleared Docker log: $(basename "$(dirname "$file")")" - fi - done <<< "$files" -} - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_HEALTH Clear Logs ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_HEALTH Clear Logs ━━━" -echo "$ICON_HEALTH System Logs: ${LOG_FILES[*]}" -echo "$ICON_CONTAINERS Docker Logs: enabled" -echo "$ICON_GEAR Dry Run: $DRY_RUN" -echo "" - -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" - -START=$(date +%s) -CLEAR_FAILED=false - -for logfile in "${LOG_FILES[@]}"; do - clear_file "$logfile" || CLEAR_FAILED=true done -echo "" -echo "━━━ $ICON_CONTAINERS Docker ━━━" -clear_docker_logs +# ── Docker Logs ─────────────────────────────────────────────────────────────────────────────── +if [[ ! -d /var/lib/docker/containers ]]; then + log "Docker containers directory not found — skipping Docker log clear" +else + while IFS= read -r logfile; do + [[ -z "$logfile" ]] && continue + + size_bytes=$(stat -c%s "$logfile" 2>/dev/null || echo 0) + size_mb=$(( size_bytes / 1048576 )) + size_h=$(du -sh "$logfile" 2>/dev/null | cut -f1) + threshold="${LOG_DOCKER_MAX_MB:-100}" + + # Get container name for display + container_id=$(basename "$(dirname "$logfile")" | cut -c1-12) + container_name=$(docker inspect --format '{{.Name}}' "$container_id" \ + 2>/dev/null | tr -d '/' || echo "$container_id") + + if [[ "$size_mb" -lt "$threshold" ]]; then + log "Docker $container_name — ${size_h} (under ${threshold}MB — skipping)" + (( DOCKER_SKIPPED++ )) + continue + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would clear Docker log: $container_name (${size_h})" + (( DOCKER_CLEARED++ )) + DOCKER_BYTES=$(( DOCKER_BYTES + size_bytes )) + continue + fi + + if : > "$logfile" 2>/dev/null; then + log "Cleared Docker log: $container_name (freed ${size_h})" + (( DOCKER_CLEARED++ )) + DOCKER_BYTES=$(( DOCKER_BYTES + size_bytes )) + else + error "Failed to clear Docker log: $container_name" + FAILED+=("docker:$container_name") + fi + done < <(find /var/lib/docker/containers/ -name "*-json.log" 2>/dev/null) +fi END=$(date +%s) +TOTAL_BYTES=$(( SYS_BYTES + DOCKER_BYTES )) +TOTAL_FREED_H=$(awk "BEGIN {printf \"%.1fMB\", $TOTAL_BYTES / 1048576}") +TOTAL_CLEARED=$(( SYS_CLEARED + DOCKER_CLEARED )) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━━━ $ICON_SUMMARY LOG CLEANER SUMMARY ━━━━━" -echo "$ICON_HEALTH System Logs: ${LOG_FILES[*]}" -echo "$ICON_CONTAINERS Docker Logs: $([[ "$DRY_RUN" == true ]] && echo "skipped (dry run)" || echo "cleared")" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no changes made" -elif [[ "$CLEAR_FAILED" == true ]]; then - echo "$ICON_ERROR Status: $ICON_ERROR SOME LOGS FAILED TO CLEAR" - notify "Log clear completed with errors on $(hostname)" "Clear Logs" "warning" +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== +if [[ "$TOTAL_CLEARED" -gt 0 || ${#FAILED[@]} -gt 0 ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY LOG CLEANER SUMMARY ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_HEALTH System cleared: $SYS_CLEARED file(s)" + echo "$ICON_CONTAINERS Docker cleared: $DOCKER_CLEARED file(s)" + echo "$ICON_HEALTH Total freed: $TOTAL_FREED_H" + echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" + [[ "$SYS_SKIPPED" -gt 0 || "$DOCKER_SKIPPED" -gt 0 ]] && \ + log "Skipped: ${SYS_SKIPPED} system + ${DOCKER_SKIPPED} Docker (under threshold)" + echo "" + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — no files cleared" + elif [[ ${#FAILED[@]} -gt 0 ]]; then + echo "$ICON_ERROR Status: SOME FILES FAILED — ${FAILED[*]}" + notify "Log clear failed on $(hostname) ($MY_ID) — ${FAILED[*]}" \ + "Clear Logs" "warning" + else + log "$ICON_DONE Status: done — ${TOTAL_FREED_H} freed" + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" else - echo "$ICON_DONE Status: $ICON_SUCCESS DONE" - notify "Logs cleared successfully on $(hostname)" "Clear Logs" "normal" + # All logs under threshold — completely silent + log "All logs under threshold — nothing to clear" fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file + +[[ ${#FAILED[@]} -gt 0 ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/unRAID_Essentials/docker_syslog_filter.sh b/unRAID_Essentials/docker_syslog_filter.sh index 650fd01..2b04f0f 100644 --- a/unRAID_Essentials/docker_syslog_filter.sh +++ b/unRAID_Essentials/docker_syslog_filter.sh @@ -1,125 +1,204 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Docker Syslog Filter --------------------------------------- -# ----------------------------------------------------------------------------------------------- -# Suppresses noisy Docker veth/docker0 syslog messages on unRAID boot. -# Creates an rsyslog filter file and restarts the rsyslog service. -# Filter file path is configured in Master.conf under FILTER_FILE. -# Supports --dry-run to preview what would be done without making changes. -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ============================= Docker Syslog Filter =========================================== +# ============================================================================================== +# Suppresses noisy Docker veth/docker0 interface messages from syslog. +# Run once at array start via ARRAY_START_SCRIPTS in master.conf. +# Idempotent — completely silent when filter is already correct. +# +# ── WHY THIS EXISTS ─────────────────────────────────────────────────────────────────────────── +# Every time Docker creates or destroys a container network interface it logs messages like: +# kernel: veth2a3b4c5: renamed from eth0 +# kernel: docker0: port 1(veth2a3b4c5) entered blocking state +# kernel: docker0: port 1(veth2a3b4c5) entered forwarding state +# +# On a busy server creating and restarting many containers these fill syslog rapidly — +# hundreds of entries per minute on container restarts, completely masking real events. +# The filter tells rsyslog to drop these before they reach the log file. +# +# ── HOW IT WORKS ────────────────────────────────────────────────────────────────────────────── +# Creates /etc/rsyslog.d/ignore-docker-veth.conf (FILTER_FILE in master.conf). +# rsyslog processes .conf files in /etc/rsyslog.d/ automatically on startup. +# Filter uses rsyslog's RainerScript to match messages containing "veth" or "docker0" +# and calls stop — the message is dropped before reaching any output target. +# +# ── IDEMPOTENT DESIGN ───────────────────────────────────────────────────────────────────────── +# On every array start: checks if filter file already exists with correct content. +# If already correct → completely silent — no rsyslog restart, no output. +# Only writes + restarts rsyslog if filter is missing or content has changed. +# This prevents unnecessary rsyslog restarts on every boot. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# Root check — writing to /etc/rsyslog.d/ requires root +# acquire_lock — prevents concurrent runs at array start +# Idempotent check — only restarts rsyslog when filter actually changed +# Directory creation — mkdir -p /etc/rsyslog.d/ before writing +# rsyslog verify — checks rsyslog running after restart +# validate_unraid — notify validated before use +# Silent on success — runs every boot, no noise when already correct +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# FILTER_FILE — path for rsyslog drop filter (default /etc/rsyslog.d/ignore-docker-veth.conf) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# docker_syslog_filter.sh — normal run (idempotent) +# docker_syslog_filter.sh --dry-run — show what would change +# docker_syslog_filter.sh --status — show filter file state and rsyslog status +# docker_syslog_filter.sh --log — verbose output +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" +# Expected filter content — used for idempotent check +EXPECTED_FILTER='if ($msg contains "veth" or $msg contains "docker0") then { + stop +}' -# ROOT CHECK +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root" + error "Must be run as root — writing to /etc/rsyslog.d/ requires root" exit 1 fi -success "Running as root" +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +acquire_lock + +detect_hosts + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" - echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_HEALTH Filter File: $FILTER_FILE" - echo "$ICON_CONTAINERS Targets: veth, docker0" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━ $ICON_SUMMARY SYSLOG FILTER STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_HEALTH Filter file: $FILTER_FILE" + echo "" + + if [[ -f "$FILTER_FILE" ]]; then + echo " Filter file: EXISTS" + echo "" + echo " Current content:" + while IFS= read -r line; do + echo " $line" + done < "$FILTER_FILE" + echo "" + + if [[ "$(cat "$FILTER_FILE" 2>/dev/null)" == "$EXPECTED_FILTER" ]]; then + echo " $ICON_SUCCESS Content: correct ✅" + else + echo " $ICON_WARN Content: differs from expected — would be rewritten" + fi + else + echo " Filter file: NOT FOUND — would be created" + fi + + echo "" + echo "━━━ rsyslog Status ━━━" + if pgrep -x rsyslogd >/dev/null 2>&1; then + echo " rsyslogd: running ✅" + else + echo " rsyslogd: NOT running" + fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -# ----------------------------------------------------------------------------------------------- -# FUNCTIONS -# ----------------------------------------------------------------------------------------------- - -# Creates the rsyslog filter file that suppresses veth and docker0 noise. -# Filter is written to FILTER_FILE defined in Master.conf. -create_filter() { - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would create filter file: $FILTER_FILE" - return - fi - - info "Writing rsyslog filter file: $FILTER_FILE" - - cat <<'EOF' > "$FILTER_FILE" -if ($msg contains "veth" or $msg contains "docker0") then { - stop -} -EOF - - success "Filter file written" -} - -# Restarts the rsyslog service to apply the new filter. -# Uses unRAID's native rc.rsyslogd script. -restart_rsyslog() { - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would restart rsyslog service" - return 0 - fi - - info "Restarting rsyslog..." - - if /etc/rc.d/rc.rsyslogd restart; then - success "rsyslog restarted" - return 0 - else - error "Failed to restart rsyslog" - return 1 - fi -} - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_HEALTH Syslog Filter ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_HEALTH Syslog Filter ━━━" -echo "$ICON_HEALTH Filter File: $FILTER_FILE" -echo "$ICON_CONTAINERS Targets: veth / docker0" -echo "$ICON_GEAR Dry Run: $DRY_RUN" -echo "" - -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" +# ============================================================================================== +# ━━━ Idempotent Check ━━━ +# ============================================================================================== +# Already correct — completely silent +if [[ -f "$FILTER_FILE" ]] && \ + [[ "$(cat "$FILTER_FILE" 2>/dev/null)" == "$EXPECTED_FILTER" ]]; then + log "Filter already correct — no changes needed" + exit 0 +fi +# ============================================================================================== +# ━━━ Apply Filter ━━━ +# ============================================================================================== START=$(date +%s) -create_filter +log "Filter file missing or outdated — applying..." -RSYSLOG_OK=true -restart_rsyslog || RSYSLOG_OK=false +# Ensure rsyslog.d directory exists +if [[ "$DRY_RUN" == false ]]; then + mkdir -p "$(dirname "$FILTER_FILE")" || { + error "Failed to create directory: $(dirname "$FILTER_FILE")" + exit 1 + } +fi + +# Write filter file +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would write filter to: $FILTER_FILE" + warn "Content:" + echo "$EXPECTED_FILTER" | while IFS= read -r line; do + echo " $line" + done +else + echo "$EXPECTED_FILTER" > "$FILTER_FILE" || { + error "Failed to write filter file: $FILTER_FILE" + notify "Syslog filter write failed on $(hostname) ($MY_ID)" \ + "Syslog Filter" "warning" + exit 1 + } + log "Filter file written: $FILTER_FILE" +fi + +# Restart rsyslog to apply +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would restart rsyslog" +else + log "Restarting rsyslog..." + if /etc/rc.d/rc.rsyslogd restart >/dev/null 2>&1; then + sleep 2 + # Verify rsyslog actually running after restart + if pgrep -x rsyslogd >/dev/null 2>&1; then + log "rsyslog restarted and running ✅" + else + error "rsyslog not running after restart" + notify "rsyslog failed to start after filter update on $(hostname) ($MY_ID)" \ + "Syslog Filter" "warning" + exit 1 + fi + else + error "rsyslog restart command failed" + notify "rsyslog restart failed on $(hostname) ($MY_ID) — filter may not be active" \ + "Syslog Filter" "warning" + exit 1 + fi +fi END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY SYSLOG FILTER SUMMARY ━━━━━" -echo "$ICON_HEALTH Filter File: $FILTER_FILE" -echo "$ICON_CONTAINERS Targets: veth / docker0" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_HEALTH Filter file: $FILTER_FILE" +echo "$ICON_HEALTH Targets: veth / docker0" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" +echo "" if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no changes made" -elif [[ "$RSYSLOG_OK" == false ]]; then - echo "$ICON_ERROR Status: $ICON_ERROR RSYSLOG RESTART FAILED" - notify "rsyslog restart failed on $(hostname) — syslog filter may not be active" "Syslog Filter" "warning" + warn "DRY RUN — no changes made" else - echo "$ICON_DONE Status: $ICON_SUCCESS DONE" - notify "Syslog filter applied on $(hostname)" "Syslog Filter" "normal" + log "$ICON_DONE Status: done — Docker veth noise suppressed ✅" fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +exit 0 \ No newline at end of file diff --git a/unRAID_Essentials/inotify_tuning.sh b/unRAID_Essentials/inotify_tuning.sh index 5d24b74..5945d61 100644 --- a/unRAID_Essentials/inotify_tuning.sh +++ b/unRAID_Essentials/inotify_tuning.sh @@ -1,100 +1,177 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- inotify Tuning -------------------------------------------- -# ----------------------------------------------------------------------------------------------- -# Increases Linux inotify limits to prevent "too many open files" and inotify exhaustion. -# Run once at array start via ARRAY_START_SCRIPTS in Master.conf. +# ============================================================================================== +# ================================= inotify Tuning ============================================ +# ============================================================================================== +# Raises Linux inotify limits at array start to prevent exhaustion across the container stack. +# Run once at array start via ARRAY_START_SCRIPTS in master.conf. +# Settings are lost on reboot — this script reapplies them on every array start. # -# Why this matters: -# Each Docker container that watches files (Sonarr, Radarr, Lidarr, NextCloud etc.) -# consumes inotify instances and watches. unRAID defaults are very low — with many -# containers running you can exhaust the limit silently, causing containers to miss -# file events (new downloads not detected, library not updated etc.) +# ── THREE INOTIFY LIMITS ────────────────────────────────────────────────────────────────────── +# max_user_instances — max number of independent inotify file descriptor objects per user +# Each container that calls inotify_init() consumes one instance +# Default 128 — exhausted quickly with 20+ active containers # -# max_user_instances = max number of inotify instances per user (default: 128) -# max_user_watches = max number of files/dirs watched per instance (default: 8192) -# max_queued_events = max events queued before dropping (default: 16384) +# max_user_watches — SHARED budget across ALL users and containers on the system +# Each watched file or directory costs one watch from this pool +# Default 8192 — VSCode alone can need 50K-200K for large workspaces # -# These settings are lost on reboot — this script reapplies them at every array start. -# All values configurable in Master.conf under unRAID Essentials. -# ----------------------------------------------------------------------------------------------- +# max_queued_events — max events buffered before kernel starts dropping them +# Low value = events silently lost during high-activity periods +# Default 16384 — sufficient for most setups +# +# ── WHY VSCODE THROWS "UNABLE TO WATCH FOR FILE CHANGES" ───────────────────────────────────── +# VSCode (and Code-Server in Docker) opens one inotify watch per file in the workspace. +# A typical project with node_modules can easily have 100K-200K files. +# All containers on the host share max_user_watches — the combined usage of: +# Sonarr, Radarr, Lidarr, Emby, Nextcloud, Code-Server, AdGuard, all other arrs +# easily exceeds 524288 (512K) watches on a busy server. +# Raising to 1048576 (1M) gives sufficient headroom — safe on 128GB RAM (~128MB kernel use). +# +# ── STARTUP ORDER MATTERS ───────────────────────────────────────────────────────────────────── +# inotify_tuning.sh must run BEFORE containers that watch files start. +# In ARRAY_START_SCRIPTS order: inotify_tuning.sh first, then container-starting scripts. +# If Code-Server starts before limits are raised it inherits the old (low) limits. +# Code-Server restart fixes this: limits are kernel-wide, not process-bound at start. +# So if Code-Server is already running: docker restart Code-Server after this script runs. +# +# ── CONSUMERS ON THIS STACK ─────────────────────────────────────────────────────────────────── +# Emby — watches all media library paths (1 watch per folder) +# Sonarr — watches TV_Shows folder tree +# Radarr — watches Movies folder tree +# Lidarr — watches Music folder tree +# Nextcloud — watches data directory for changes +# Code-Server — watches entire workspace (can be 50K-200K with node_modules) +# AdGuard Home — watches config directory +# + all other containers using inotify internally +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock — prevents duplicate runs at array start +# Root check — sysctl writes require root +# validate_unraid — notify validated before use +# Silent on success — runs every boot, no noise when already correct +# Only warns on changes or failures +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# INOTIFY_MAX_INSTANCES — default 1024 +# INOTIFY_MAX_WATCHES — default 1048576 (1M) +# INOTIFY_MAX_QUEUED_EVENTS — default 32768 +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# inotify_tuning.sh — normal run (apply settings) +# inotify_tuning.sh --dry-run — show what would change +# inotify_tuning.sh --status — show current vs target values and top consumers +# inotify_tuning.sh --log — verbose output +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR inotify Tuning ━━━" - +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root" + error "Must be run as root — sysctl writes require root" exit 1 fi -success "Running as root" +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" -# ----------------------------------------------------------------------------------------------- -# ━━━ Current Values ━━━ -# ----------------------------------------------------------------------------------------------- -CURRENT_INSTANCES=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo "?") -CURRENT_WATCHES=$(sysctl -n fs.inotify.max_user_watches 2>/dev/null || echo "?") -CURRENT_EVENTS=$(sysctl -n fs.inotify.max_queued_events 2>/dev/null || echo "?") +acquire_lock -info "Current: instances=$CURRENT_INSTANCES watches=$CURRENT_WATCHES queued=$CURRENT_EVENTS" -info "Target: instances=${INOTIFY_MAX_INSTANCES} watches=${INOTIFY_MAX_WATCHES} queued=${INOTIFY_MAX_QUEUED_EVENTS}" +detect_hosts +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY INOTIFY STATUS ━━━━━" - echo " max_user_instances: $CURRENT_INSTANCES (target: ${INOTIFY_MAX_INSTANCES})" - echo " max_user_watches: $CURRENT_WATCHES (target: ${INOTIFY_MAX_WATCHES})" - echo " max_queued_events: $CURRENT_EVENTS (target: ${INOTIFY_MAX_QUEUED_EVENTS})" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "" - echo " Active inotify instances in use:" - find /proc/*/fd -lname "anon_inode:inotify" 2>/dev/null | \ - awk -F/ '{print $3}' | sort -u | while read -r pid; do - cmd=$(cat /proc/$pid/comm 2>/dev/null || echo "?") - echo " PID $pid ($cmd)" - done | head -20 + + echo "━━━ Kernel Limits ━━━" + CURRENT_INSTANCES=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo "?") + CURRENT_WATCHES=$(sysctl -n fs.inotify.max_user_watches 2>/dev/null || echo "?") + CURRENT_EVENTS=$(sysctl -n fs.inotify.max_queued_events 2>/dev/null || echo "?") + + for label in "max_user_instances current=$CURRENT_INSTANCES target=$INOTIFY_MAX_INSTANCES" \ + "max_user_watches current=$CURRENT_WATCHES target=$INOTIFY_MAX_WATCHES" \ + "max_queued_events current=$CURRENT_EVENTS target=$INOTIFY_MAX_QUEUED_EVENTS"; do + echo " $label" + done + + echo "" + echo "━━━ Active Instances ━━━" + USED_INSTANCES=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l) + USED_INSTANCES="${USED_INSTANCES//[^0-9]/}" + echo " Instances in use: ${USED_INSTANCES:-0} / $CURRENT_INSTANCES" + if [[ "$CURRENT_INSTANCES" -gt 0 ]]; then + PCT=$(( ${USED_INSTANCES:-0} * 100 / CURRENT_INSTANCES )) + echo " Utilisation: ${PCT}%" + fi + + echo "" + echo "━━━ Top Consumers ━━━" + find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | \ + awk -F/ '{print $3}' | sort | uniq -c | sort -rn | head -10 | \ + while read -r count pid; do + cmd=$(cat /proc/"$pid"/comm 2>/dev/null || echo "?") + cgroup=$(cat /proc/"$pid"/cgroup 2>/dev/null | \ + grep docker | grep -o '[a-f0-9]\{12\}' | head -1 || echo "") + if [[ -n "$cgroup" ]]; then + label="[docker:${cgroup}] $cmd" + else + label="[host] $cmd" + fi + echo " ${count} instances — $label (PID $pid)" + done | head -10 + + echo "" + echo "━━━ VSCode / Code-Server ━━━" + echo " If VSCode shows 'unable to watch for file changes':" + echo " 1. Verify max_user_watches target is set high enough" + echo " 2. Check total watches used: cat /proc/sys/fs/inotify/max_user_watches" + echo " 3. After any limit change: docker restart Code-Server" + echo " (running containers inherit limits at start, not dynamically)" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== # ━━━ Apply Settings ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" +# ============================================================================================== CHANGED=0 FAILED=0 apply_sysctl() { - local key="$1" - local value="$2" + local key="$1" value="$2" local current current=$(sysctl -n "$key" 2>/dev/null || echo 0) if [[ "$current" -eq "$value" ]]; then - success "$key = $value (already set)" - return + log "$key = $value (already correct)" + return 0 fi if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would set $key = $value (currently $current)" - return + return 0 fi if sysctl -w "${key}=${value}" >/dev/null 2>&1; then - success "$key = $value (was $current)" - ((CHANGED++)) + warn "Set $key = $value (was $current)" + (( CHANGED++ )) else error "Failed to set $key = $value" - ((FAILED++)) + (( FAILED++ )) fi } @@ -102,25 +179,35 @@ apply_sysctl "fs.inotify.max_user_instances" "$INOTIFY_MAX_INSTANCES" apply_sysctl "fs.inotify.max_user_watches" "$INOTIFY_MAX_WATCHES" apply_sysctl "fs.inotify.max_queued_events" "$INOTIFY_MAX_QUEUED_EVENTS" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━━━ $ICON_SUMMARY INOTIFY TUNING SUMMARY ━━━━━" -echo " max_user_instances: $(sysctl -n fs.inotify.max_user_instances 2>/dev/null)" -echo " max_user_watches: $(sysctl -n fs.inotify.max_user_watches 2>/dev/null)" -echo " max_queued_events: $(sysctl -n fs.inotify.max_queued_events 2>/dev/null)" -echo "" - -if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no changes made" -elif [[ "$FAILED" -gt 0 ]]; then - echo "$ICON_ERROR Status: $FAILED setting(s) failed" - notify "inotify tuning failed on $(hostname) — $FAILED setting(s) could not be applied" "inotify Tuning" "warning" +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== +if [[ "$FAILED" -gt 0 ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY INOTIFY TUNING SUMMARY ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_ERROR $FAILED setting(s) failed to apply" + notify "inotify tuning failed on $(hostname) ($MY_ID) — $FAILED setting(s) could not be applied" \ + "inotify Tuning" "warning" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" exit 1 elif [[ "$CHANGED" -gt 0 ]]; then - echo "$ICON_DONE Status: $ICON_SUCCESS $CHANGED setting(s) applied" + echo "" + echo "━━━━━ $ICON_SUMMARY INOTIFY TUNING SUMMARY ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo " max_user_instances: $(sysctl -n fs.inotify.max_user_instances 2>/dev/null)" + echo " max_user_watches: $(sysctl -n fs.inotify.max_user_watches 2>/dev/null)" + echo " max_queued_events: $(sysctl -n fs.inotify.max_queued_events 2>/dev/null)" + echo "" + warn "$CHANGED setting(s) updated" + if [[ "$CHANGED" -gt 0 ]]; then + warn "If Code-Server is running: docker restart Code-Server" + warn "Running containers inherit limits at start — restart picks up new values" + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" else - echo "$ICON_DONE Status: $ICON_SUCCESS All settings already correct" + # Already correct — completely silent (runs every boot) + log "inotify limits already correct — no changes needed" fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file + +exit 0 \ No newline at end of file diff --git a/unRAID_Essentials/mover_stop.sh b/unRAID_Essentials/mover_stop.sh index e25a894..c24feb2 100644 --- a/unRAID_Essentials/mover_stop.sh +++ b/unRAID_Essentials/mover_stop.sh @@ -1,116 +1,162 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Mover Stop Script ------------------------------------------ -# ----------------------------------------------------------------------------------------------- -# Safely stops the unRAID mover process with a user warning before halting. -# Timeout before stopping is configured in Master.conf under MOVER_STOP_TIMEOUT. -# Supports --dry-run to preview what would happen without making changes. -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= Mover Stop ================================================= +# ============================================================================================== +# Safely stops the unRAID mover process with a warning before halting. +# Warns all logged-in users via wall message, waits the configured timeout, then stops. +# +# ── WHEN TO USE ─────────────────────────────────────────────────────────────────────────────── +# - Before a planned reboot when mover is running mid-cycle +# - Before disk replacement or array operations that need mover stopped +# - Before rsync — mover and rsync simultaneously moving the same files causes corruption +# - Called automatically by maintenance scripts that need the mover stopped first +# +# ── STOP SEQUENCE ───────────────────────────────────────────────────────────────────────────── +# 1. Check if mover is running — exit cleanly if not +# 2. Broadcast wall warning to all logged-in users +# 3. Wait MOVER_STOP_TIMEOUT seconds (default 30) — gives active sessions a chance to note it +# 4. Send SIGTERM — mover can complete its current file operation before exiting +# 5. Wait 5 seconds for graceful exit +# 6. Verify stopped — if still running send SIGKILL (force) +# 7. Final verify — error if still running after SIGKILL +# +# ── SIGTERM vs SIGKILL ──────────────────────────────────────────────────────────────────────── +# SIGTERM first — allows mover to finish the file it is currently moving (no partial files). +# SIGKILL only as fallback — forces immediate stop (may leave partial files on cache or array). +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock — prevents concurrent stop attempts racing each other +# Root check — pkill on emhttp processes requires root +# validate_unraid — notify validated before use +# SIGTERM → verify → SIGKILL sequence — graceful then forced +# Final verify — confirms mover actually stopped +# Silent on clean — mover not running = log() only, no output ✅ +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# MOVER_STOP_TIMEOUT — seconds to warn users before stopping (default 30) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# mover_stop.sh — stop mover with configured timeout +# mover_stop.sh --dry-run — show what would happen +# mover_stop.sh --status — show mover state +# mover_stop.sh --log — verbose output +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" - -# ROOT CHECK +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root" + error "Must be run as root — pkill on emhttp processes requires root" exit 1 fi -success "Running as root" +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" -# Validate MOVER_STOP_TIMEOUT is a valid integer before using it validate_int MOVER_STOP_TIMEOUT "$MOVER_STOP_TIMEOUT" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +acquire_lock + +detect_hosts + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_MOVER Timeout: ${MOVER_STOP_TIMEOUT}s" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_MOVER Timeout: ${MOVER_STOP_TIMEOUT}s" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "" + if pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then + MOVER_PID=$(pgrep -f "emhttp.*Mover" | head -1) + MOVER_START=$(ps -o lstart= -p "$MOVER_PID" 2>/dev/null | xargs) + echo " $ICON_MOVER Mover: RUNNING (PID $MOVER_PID)" + [[ -n "$MOVER_START" ]] && echo " $ICON_TIME Started: $MOVER_START" + else + echo " $ICON_MOVER Mover: not running" + fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -# ----------------------------------------------------------------------------------------------- -# FUNCTIONS -# ----------------------------------------------------------------------------------------------- - -# Returns 0 if the unRAID mover process is currently running, 1 if not. -check_mover_running() { - pgrep -f "emhttp.*Mover" >/dev/null 2>&1 -} - -# Broadcasts a wall message to all logged in users warning mover is stopping. -notify_users() { - warn "Notifying users — mover stopping in ${MOVER_STOP_TIMEOUT}s" - wall "$ICON_WARN unRAID Mover will stop in ${MOVER_STOP_TIMEOUT} second(s)." -} - -# Sends SIGTERM to the mover process via pkill. -# Skips if dry run is active. -stop_mover() { - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would stop unRAID Mover process" - return - fi - - info "Stopping unRAID Mover..." - - if pkill -f "emhttp.*Mover"; then - success "Mover stopped" - else - warn "Could not stop mover — may have already stopped" - fi -} - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_MOVER Mover Stop ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_MOVER Mover Stop ━━━" -echo "$ICON_MOVER Timeout: ${MOVER_STOP_TIMEOUT}s" -echo "$ICON_GEAR Dry Run: $DRY_RUN" -echo "" - -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" - +# ============================================================================================== +# ━━━ Mover Stop ━━━ +# ============================================================================================== START=$(date +%s) -if check_mover_running; then - info "$ICON_MOVER Mover is running" - notify_users - info "Waiting ${MOVER_STOP_TIMEOUT}s before stopping..." +if ! pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then + log "Mover is not running — nothing to do" + exit 0 +fi + +MOVER_PID=$(pgrep -f "emhttp.*Mover" | head -1) +warn "Mover is running (PID $MOVER_PID) — stopping in ${MOVER_STOP_TIMEOUT}s" + +# ── Warn users via wall ─────────────────────────────────────────────────────────────────────── +if [[ "$DRY_RUN" == false ]]; then + wall "$ICON_WARN $MY_ID ($LOCAL_SERVER_NAME) — unRAID Mover stopping in ${MOVER_STOP_TIMEOUT}s" + log "Wall message sent — waiting ${MOVER_STOP_TIMEOUT}s..." sleep "$MOVER_STOP_TIMEOUT" - stop_mover else - info "$ICON_MOVER Mover is not running — nothing to do" + warn "DRY RUN — would send wall warning and wait ${MOVER_STOP_TIMEOUT}s" +fi + +# ── SIGTERM — graceful stop ─────────────────────────────────────────────────────────────────── +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would send SIGTERM to mover (PID $MOVER_PID)" +else + log "Sending SIGTERM to mover (PID $MOVER_PID)..." + pkill -TERM -f "emhttp.*Mover" 2>/dev/null || true + sleep 5 + + # Verify stopped after SIGTERM + if ! pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then + warn "Mover stopped cleanly (SIGTERM) ✅" + else + # ── SIGKILL — forced stop ───────────────────────────────────────────────────────────── + warn "Mover still running after SIGTERM — sending SIGKILL (may leave partial files)" + pkill -KILL -f "emhttp.*Mover" 2>/dev/null || true + sleep 2 + + # Final verify + if pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then + error "Mover still running after SIGKILL — manual intervention needed" + notify "Mover stop failed on $(hostname) ($MY_ID) — process unkillable" \ + "Mover Stop" "warning" + exit 1 + else + warn "Mover force-stopped (SIGKILL) — check for partial files on cache" + fi + fi fi END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY MOVER STOP SUMMARY ━━━━━" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -if check_mover_running; then - echo "$ICON_ERROR Status: $ICON_ERROR STILL RUNNING" - notify "Mover stop failed — mover still running on $(hostname)" "Mover Stop" "warning" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_MOVER Timeout: ${MOVER_STOP_TIMEOUT}s" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" +echo "" + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — no changes made" else - echo "$ICON_DONE Status: $ICON_SUCCESS STOPPED / NOT RUNNING" - notify "Mover stopped on $(hostname)" "Mover Stop" "normal" + log "$ICON_DONE Status: done — mover stopped ✅" fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/unRAID_Essentials/php_fpm_max_children.sh b/unRAID_Essentials/php_fpm_max_children.sh index 7dcc802..227b857 100644 --- a/unRAID_Essentials/php_fpm_max_children.sh +++ b/unRAID_Essentials/php_fpm_max_children.sh @@ -1,141 +1,206 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# ----------------------------- PHP-FPM Max Children Script ------------------------------------ -# ----------------------------------------------------------------------------------------------- -# Persistently sets PHP-FPM pm.max_children on unRAID. -# Config file path and max children value are set in Master.conf. -# Supports --dry-run to preview what would be changed without making changes. -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ============================= PHP-FPM Max Children =========================================== +# ============================================================================================== +# Persistently sets PHP-FPM pm.max_children on unRAID to prevent WebGUI slowdowns. +# Run once at array start via ARRAY_START_SCRIPTS in master.conf. +# Idempotent — completely silent when value is already correct. +# +# ── WHY THIS EXISTS ─────────────────────────────────────────────────────────────────────────── +# unRAID's WebGUI runs through PHP-FPM. The default pm.max_children is very low (4-8). +# Under load — multiple users, Docker operations, heavy dashboard usage — all PHP workers +# saturate and new requests queue. The WebGUI becomes slow or unresponsive. +# +# pm.max_children controls how many PHP worker processes can run simultaneously. +# Raising it allows the WebGUI to handle more concurrent requests without queuing. +# Too high: wastes RAM. Too low: WebGUI slowdowns. +# PHP_MAX_CHILDREN=250 is appropriate for 128GB — ~2MB per worker = ~500MB total. +# +# ── WHY IDEMPOTENT ──────────────────────────────────────────────────────────────────────────── +# This runs at every array start. If the value is already correct there is nothing to do — +# no config write, no PHP-FPM restart. Restarting PHP-FPM unnecessarily disrupts active +# WebGUI sessions and is annoying on every boot. +# +# ── APPLY SEQUENCE ──────────────────────────────────────────────────────────────────────────── +# 1. Read current pm.max_children from PHP_CONF +# 2. If already at target → exit silently (idempotent) +# 3. Verify sed pattern matches before writing +# 4. Apply sed replacement +# 5. Restart PHP-FPM via rc.php-fpm +# 6. Verify PHP-FPM process running after restart +# 7. Verify config file reflects target value +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# Root check — writing to system config requires root +# acquire_lock — prevents concurrent runs at array start +# Idempotent check — only restarts PHP-FPM when value actually changes +# Pattern match check — verifies sed found pm.max_children before writing +# Process verify — confirms PHP-FPM running after restart +# Config verify — reads back config to confirm value applied +# validate_unraid — notify validated before use +# Silent on correct — runs every boot, no noise when already set ✅ +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# PHP_MAX_CHILDREN — target pm.max_children value (default 250) +# PHP_CONF — path to PHP-FPM www.conf (default /etc/php83/php-fpm.d/www.conf) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# php_fpm_max_children.sh — normal run (idempotent) +# php_fpm_max_children.sh --dry-run — show what would change +# php_fpm_max_children.sh --status — show current vs target and process state +# php_fpm_max_children.sh --log — verbose output +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" - -# ROOT CHECK +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root" + error "Must be run as root — writing system config requires root" exit 1 fi -success "Running as root" +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" -# VALIDATION validate_int PHP_MAX_CHILDREN "$PHP_MAX_CHILDREN" require_var PHP_CONF -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +acquire_lock + +detect_hosts + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" - echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_GEAR Config File: $PHP_CONF" - echo "$ICON_PHP Max Children: $PHP_MAX_CHILDREN" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━ $ICON_SUMMARY PHP-FPM STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_GEAR Config file: $PHP_CONF" + echo "$ICON_PHP Target: pm.max_children = $PHP_MAX_CHILDREN" + echo "" + + if [[ -f "$PHP_CONF" ]]; then + CURRENT_VAL=$(grep -E "^pm\.max_children" "$PHP_CONF" 2>/dev/null | \ + awk '{print $NF}') + if [[ "${CURRENT_VAL:-0}" -eq "$PHP_MAX_CHILDREN" ]]; then + echo " $ICON_SUCCESS Current: pm.max_children = $CURRENT_VAL (correct ✅)" + else + echo " $ICON_WARN Current: pm.max_children = ${CURRENT_VAL:-not set} (would update)" + fi + else + echo " $ICON_ERROR Config file not found: $PHP_CONF" + fi + + echo "" + if pgrep -f "php-fpm" >/dev/null 2>&1; then + FPM_COUNT=$(pgrep -fc "php-fpm" 2>/dev/null || echo "?") + echo " $ICON_SUCCESS PHP-FPM: running ($FPM_COUNT worker(s))" + else + echo " $ICON_ERROR PHP-FPM: NOT running" + fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -# ----------------------------------------------------------------------------------------------- -# FUNCTIONS -# ----------------------------------------------------------------------------------------------- - -# Applies pm.max_children to the PHP-FPM config file and restarts the service. -# Verifies the value was applied correctly after restart. -# Skips all changes if dry run is active. -apply_php_max_children() { - local target="pm.max_children = $PHP_MAX_CHILDREN" - - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would set pm.max_children = $PHP_MAX_CHILDREN in $PHP_CONF" - warn "DRY RUN — would restart PHP-FPM service" - return 0 - fi - - # Verify config file exists before attempting changes - if [[ ! -f "$PHP_CONF" ]]; then - error "PHP config file not found: $PHP_CONF" - return 1 - fi - - info "Applying pm.max_children = $PHP_MAX_CHILDREN..." - - if ! sed -i "s/^pm\.max_children.*/$target/" "$PHP_CONF"; then - error "Failed to update PHP config: $PHP_CONF" - return 1 - fi - - success "Config updated" - - info "Restarting PHP-FPM..." - - if ! /etc/rc.d/rc.php-fpm restart; then - error "PHP-FPM restart failed" - return 1 - fi - - success "PHP-FPM restarted" - - # Verify the value was applied correctly - local current - current=$(grep -E "^pm\.max_children" "$PHP_CONF" || true) - - if [[ -n "$current" ]]; then - success "Verified: $current" - logger "Userscript: PHP-FPM updated → $current" - else - warn "Could not verify configuration value — check $PHP_CONF manually" - fi - - return 0 -} - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_PHP PHP-FPM Config ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_PHP PHP-FPM Config ━━━" -echo "$ICON_GEAR Config File: $PHP_CONF" -echo "$ICON_PHP Max Children: $PHP_MAX_CHILDREN" -echo "$ICON_GEAR Dry Run: $DRY_RUN" -echo "" - -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" - +# ============================================================================================== +# ━━━ PHP-FPM Config ━━━ +# ============================================================================================== START=$(date +%s) -PHP_SUCCESS=false -apply_php_max_children && PHP_SUCCESS=true +if [[ ! -f "$PHP_CONF" ]]; then + error "PHP config file not found: $PHP_CONF" + notify "PHP-FPM config not found on $(hostname) ($MY_ID) — $PHP_CONF missing" \ + "PHP-FPM" "warning" + exit 1 +fi + +# ── Idempotent check ───────────────────────────────────────────────────────────────────────── +CURRENT_VAL=$(grep -E "^pm\.max_children" "$PHP_CONF" 2>/dev/null | awk '{print $NF}') +if [[ "${CURRENT_VAL:-0}" -eq "$PHP_MAX_CHILDREN" ]]; then + log "pm.max_children already $PHP_MAX_CHILDREN — no changes needed" + exit 0 +fi + +warn "pm.max_children: ${CURRENT_VAL:-not set} → $PHP_MAX_CHILDREN" + +# ── Verify pattern exists before writing ───────────────────────────────────────────────────── +if ! grep -qE "^pm\.max_children" "$PHP_CONF" 2>/dev/null; then + error "pm.max_children not found in $PHP_CONF — cannot apply" + error "Add 'pm.max_children = $PHP_MAX_CHILDREN' to $PHP_CONF manually" + notify "PHP-FPM pm.max_children not found in config on $(hostname) ($MY_ID)" \ + "PHP-FPM" "warning" + exit 1 +fi + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would set pm.max_children = $PHP_MAX_CHILDREN in $PHP_CONF" + warn "DRY RUN — would restart PHP-FPM" + exit 0 +fi + +# ── Apply setting ───────────────────────────────────────────────────────────────────────────── +log "Applying pm.max_children = $PHP_MAX_CHILDREN..." +if ! sed -i "s/^pm\.max_children.*/pm.max_children = $PHP_MAX_CHILDREN/" "$PHP_CONF"; then + error "Failed to update $PHP_CONF" + notify "PHP-FPM config update failed on $(hostname) ($MY_ID)" \ + "PHP-FPM" "warning" + exit 1 +fi + +log "Config updated" + +# ── Restart PHP-FPM ────────────────────────────────────────────────────────────────────────── +log "Restarting PHP-FPM..." +if ! /etc/rc.d/rc.php-fpm restart >/dev/null 2>&1; then + error "PHP-FPM restart command failed" + notify "PHP-FPM restart failed on $(hostname) ($MY_ID)" \ + "PHP-FPM" "warning" + exit 1 +fi + +sleep 3 # Allow PHP-FPM workers to initialise + +# ── Verify process running ──────────────────────────────────────────────────────────────────── +if ! pgrep -f "php-fpm" >/dev/null 2>&1; then + error "PHP-FPM not running after restart — WebGUI may be broken" + notify "PHP-FPM failed to start after config update on $(hostname) ($MY_ID)" \ + "PHP-FPM" "warning" + exit 1 +fi + +# ── Verify config reflects target ──────────────────────────────────────────────────────────── +APPLIED_VAL=$(grep -E "^pm\.max_children" "$PHP_CONF" 2>/dev/null | awk '{print $NF}') +if [[ "${APPLIED_VAL:-0}" -ne "$PHP_MAX_CHILDREN" ]]; then + warn "Config reads pm.max_children = ${APPLIED_VAL:-unknown} — expected $PHP_MAX_CHILDREN" + warn "Check $PHP_CONF manually" +else + log "Verified: pm.max_children = $APPLIED_VAL ✅" +fi END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY PHP-FPM SUMMARY ━━━━━" -echo "$ICON_GEAR Config File: $PHP_CONF" -echo "$ICON_PHP Max Children: $PHP_MAX_CHILDREN" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no changes made" -elif [[ "$PHP_SUCCESS" == true ]]; then - echo "$ICON_DONE Status: $ICON_SUCCESS DONE" - notify "PHP-FPM max_children set to $PHP_MAX_CHILDREN on $(hostname)" "PHP-FPM" "normal" -else - echo "$ICON_ERROR Status: $ICON_ERROR FAILED" - notify "PHP-FPM config update failed on $(hostname)" "PHP-FPM" "warning" -fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_GEAR Config file: $PHP_CONF" +echo "$ICON_PHP Applied: pm.max_children = $PHP_MAX_CHILDREN" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" +echo "" +log "$ICON_DONE Status: done ✅" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -[[ "$PHP_SUCCESS" == false ]] && exit 1 exit 0 \ No newline at end of file diff --git a/unRAID_Essentials/rsync_stop.sh b/unRAID_Essentials/rsync_stop.sh index f4a350f..186eb83 100644 --- a/unRAID_Essentials/rsync_stop.sh +++ b/unRAID_Essentials/rsync_stop.sh @@ -1,135 +1,184 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Rsync Stop Script ------------------------------------------ -# ----------------------------------------------------------------------------------------------- -# Stops rsync intelligently — auto-detects what's running and acts accordingly. +# ============================================================================================== +# ================================= Rsync Stop ================================================= +# ============================================================================================== +# Stops rsync intelligently on both local and remote servers. +# Auto-detects orchestrators and chooses the safest stop strategy automatically. # -# Default behavior (just run it): -# Detects if an orchestrator (daily/weekly) is running -# If yes → kills rsync subprocess only -# orchestrator sees rsync died → moves to next share or exits cleanly -# If no → kills rsync processes directly (solo rsync.sh run) -# Cleans stale lock files -# Recovers any containers left stopped by interrupted rsync +# ── TWO MODES ───────────────────────────────────────────────────────────────────────────────── +# Default (smart): +# Detects if an orchestrator (daily/weekly/critical sync) is running +# If orchestrator found → kills rsync subprocess only +# Orchestrator sees rsync died → moves to next share or exits cleanly +# If no orchestrator → kills rsync directly (standalone rsync.sh run) +# Cleans stale lock files after kill +# Recovers containers left stopped by interrupted rsync (local only) # -# --full-stop flag (nuclear): -# Kills orchestrator first → then rsync -# Use when: you want everything dead immediately -# daily/weekly loop will NOT continue to next share +# --full-stop (nuclear): +# Kills orchestrator first → then kills rsync +# Orchestrator will NOT continue to next share +# Use when: you need everything dead immediately # -# Both local and remote are handled in one run. -# Remote containers left as-is — docker_watchdog.sh handles remote recovery. +# ── REMOTE HANDLING ─────────────────────────────────────────────────────────────────────────── +# Both local and remote handled in one run via SSH. +# Remote containers left as-is — docker_watchdog.sh handles remote container recovery. +# If remote unreachable → skips remote cleanly, logs warning. # -# Flags: -# (none) ← smart mode — auto-detects, rsync-only if orchestrator running -# --full-stop ← nuclear — kill orchestrator + rsync -# --dry-run ← preview without changes -# ----------------------------------------------------------------------------------------------- +# ── ORCHESTRATOR DETECTION ──────────────────────────────────────────────────────────────────── +# detect_rsync_parent() scans all lock files to find which running process +# has rsync as a descendant. No hardcoded list — works for any orchestrator. +# Returns: "script_name:parent_pid" if found, empty if rsync running standalone. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# Root check — pkill and docker require root +# acquire_lock — prevents concurrent stop attempts racing +# DOCKER_TIMEOUT — all docker calls protected against hung daemon +# SSH_TIMEOUT — all remote SSH calls timeout-protected +# SIGTERM → SIGKILL — graceful then forced for orchestrators +# Container recovery — restarts local containers left stopped by killed rsync +# validate_unraid_cmd — notify validated before use +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# rsync_stop.sh — smart stop (auto-detect) +# rsync_stop.sh --full-stop — kill orchestrator + rsync +# rsync_stop.sh --rsync-only — skip container recovery (called by other scripts) +# rsync_stop.sh --dry-run — preview without changes +# rsync_stop.sh --status — show what's currently running +# rsync_stop.sh --full-stop --dry-run — preview full stop +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" -# Check for --full-stop before parse_args +DOCKER_TIMEOUT=15 +SSH_TIMEOUT=15 + +# ── Parse special flags before parse_args ───────────────────────────────────────────────────── FULL_STOP=false +RSYNC_ONLY_MODE=false FILTERED_ARGS=() for arg in "$@"; do - if [[ "$arg" == "--full-stop" ]]; then - FULL_STOP=true - else - FILTERED_ARGS+=("$arg") - fi + case "$arg" in + --full-stop) FULL_STOP=true ;; + --rsync-only) RSYNC_ONLY_MODE=true ;; + *) FILTERED_ARGS+=("$arg") ;; + esac done parse_args "${FILTERED_ARGS[@]}" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root — pkill and docker require root" + exit 1 +fi + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + +acquire_lock detect_hosts resolve_remote_ip -REMOTE_REACHABLE=true -if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then - warn "$ICON_PING Remote $REMOTE_SERVER_NAME unreachable — will skip remote" - REMOTE_REACHABLE=false +# Remote reachability +REMOTE_REACHABLE=false +if timeout "$SSH_TIMEOUT" ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then + REMOTE_REACHABLE=true + log "$REMOTE_SERVER_NAME reachable ✅" else - info "$ICON_PING $REMOTE_SERVER_NAME reachable" + warn "$REMOTE_SERVER_NAME unreachable — remote operations will be skipped" fi -[[ "$SHOW_STATUS" == true ]] && show_status && exit 0 -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" +[[ "$FULL_STOP" == true ]] && warn "FULL STOP mode — orchestrator + rsync will be killed" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY RSYNC STOP STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)" + echo "" + + LOCAL_PIDS=$(pgrep -x rsync 2>/dev/null | tr '\n' ' ') + echo " $ICON_SYNC Local rsync PIDs: ${LOCAL_PIDS:-none}" + + for lockfile in "$LOCK_DIR"/*.lock; do + [[ -f "$lockfile" ]] || continue + content=$(cat "$lockfile" 2>/dev/null) + pid="${content%%:*}" + name="${content##*:}" + [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && \ + echo " $ICON_RUNNING Lock: $name (PID $pid)" + done + + if [[ "$REMOTE_REACHABLE" == true ]]; then + REMOTE_PIDS=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" \ + root@"$REMOTE_SERVER" "pgrep -x rsync || true" 2>/dev/null | tr '\n' ' ') + echo " $ICON_SYNC Remote rsync PIDs: ${REMOTE_PIDS:-none}" + else + echo " $ICON_WARN Remote: unreachable" + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ── ORCHESTRATOR DETECTION ──────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Scans lock files to find which running process has rsync as a descendant. +# No hardcoded script names — detects any orchestrator automatically. -# ----------------------------------------------------------------------------------------------- -# ━━━ Auto-detect orchestrators ━━━ -# Check if daily or weekly is running on local and remote -# This determines default behavior -# ----------------------------------------------------------------------------------------------- -# ----------------------------------------------------------------------------------------------- -# detect_rsync_parent — scans all lock files, finds which running process has rsync as a child -# No hardcoded list — works for any orchestrator automatically -# -# Returns: "script_name:parent_pid" if found, empty if rsync running standalone -# ----------------------------------------------------------------------------------------------- detect_rsync_parent() { - local found="" - - # Get all rsync PIDs running locally local rsync_pids rsync_pids=$(pgrep -x rsync 2>/dev/null || true) [[ -z "$rsync_pids" ]] && echo "" && return - # Scan all lock files in LOCK_DIR for lockfile in "$LOCK_DIR"/*.lock; do [[ -f "$lockfile" ]] || continue - local content pid locked_name content=$(cat "$lockfile" 2>/dev/null) pid="${content%%:*}" locked_name="${content##*:}" - - # Skip if PID dead or is itself a rsync lock [[ -z "$pid" ]] && continue ! kill -0 "$pid" 2>/dev/null && continue [[ "$locked_name" == rsync_* ]] && continue - # Check if any rsync PID is a child of this lock's PID - local children - children=$(cat /proc/"$pid"/task/"$pid"/children 2>/dev/null || \ - tr ' ' '\n' < /proc/"$pid"/children 2>/dev/null || true) - - # Walk the child tree — rsync may be a grandchild (bash → rsync.sh → rsync) local all_descendants all_descendants=$(pgrep -P "$pid" 2>/dev/null || true) - # Check if any rsync PID is in the descendants while IFS= read -r rsync_pid; do [[ -z "$rsync_pid" ]] && continue + local ppid + ppid=$(awk '/^PPid:/{print $2}' /proc/"$rsync_pid"/status 2>/dev/null || echo "") if echo "$all_descendants" | grep -qw "$rsync_pid" 2>/dev/null || \ - [[ "$(cat /proc/"$rsync_pid"/status 2>/dev/null | awk '/^PPid:/{print $2}')" == "$pid" ]]; then - found="$locked_name:$pid" - break 2 + [[ "$ppid" == "$pid" ]]; then + echo "${locked_name}:${pid}" + return fi done <<< "$rsync_pids" done - - echo "$found" + echo "" } detect_rsync_parent_remote() { [[ "$REMOTE_REACHABLE" != true ]] && echo "" && return - # Run the same logic on remote via SSH - local found - found=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" bash << 'REMOTE_SCRIPT' + timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" \ + root@"$REMOTE_SERVER" bash << 'REMOTE_SCRIPT' 2>/dev/null LOCK_DIR="/tmp/unraid_locks" rsync_pids=$(pgrep -x rsync 2>/dev/null || true) [[ -z "$rsync_pids" ]] && exit 0 - for lockfile in "$LOCK_DIR"/*.lock; do [[ -f "$lockfile" ]] || continue content=$(cat "$lockfile" 2>/dev/null) @@ -138,21 +187,18 @@ for lockfile in "$LOCK_DIR"/*.lock; do [[ -z "$pid" ]] && continue ! kill -0 "$pid" 2>/dev/null && continue [[ "$locked_name" == rsync_* ]] && continue - all_descendants=$(pgrep -P "$pid" 2>/dev/null || true) while IFS= read -r rsync_pid; do [[ -z "$rsync_pid" ]] && continue - ppid=$(awk '/^PPid:/{print $2}' /proc/"$rsync_pid"/status 2>/dev/null) + ppid=$(awk '/^PPid:/{print $2}' /proc/"$rsync_pid"/status 2>/dev/null || echo "") if echo "$all_descendants" | grep -qw "$rsync_pid" 2>/dev/null || \ [[ "$ppid" == "$pid" ]]; then - echo "$locked_name:$pid" + echo "${locked_name}:${pid}" exit 0 fi done <<< "$rsync_pids" done REMOTE_SCRIPT - 2>/dev/null) - echo "$found" } LOCAL_ORCH=$(detect_rsync_parent) @@ -162,27 +208,26 @@ REMOTE_ORCH="" # Determine mode if [[ "$FULL_STOP" == true ]]; then MODE="full-stop" - info "Mode: FULL STOP — orchestrator + rsync will be killed" elif [[ -n "$LOCAL_ORCH" ]] || [[ -n "$REMOTE_ORCH" ]]; then MODE="rsync-only" - [[ -n "$LOCAL_ORCH" ]] && info "Detected local orchestrator: ${LOCAL_ORCH%%:*} — rsync-only mode" - [[ -n "$REMOTE_ORCH" ]] && info "Detected remote orchestrator: ${REMOTE_ORCH%%:*} — rsync-only mode" - info "Orchestrator will continue after rsync is killed" - info "Use --full-stop to also kill the orchestrator" + [[ -n "$LOCAL_ORCH" ]] && \ + warn "Local orchestrator detected: ${LOCAL_ORCH%%:*} — rsync-only mode" + [[ -n "$REMOTE_ORCH" ]] && \ + warn "Remote orchestrator detected: ${REMOTE_ORCH%%:*} — rsync-only mode" + warn "Use --full-stop to also kill the orchestrator" else MODE="rsync-only" - info "No orchestrator detected — killing rsync directly" + log "No orchestrator detected — killing rsync directly" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_STOP Kill Orchestrators (full-stop only) ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ── Kill Orchestrators (full-stop only) ─────────────────────────────────────────────────────── +# ============================================================================================== ORCHESTRATORS_KILLED=() REMOTE_ORCHESTRATORS_KILLED=() kill_orchestrator() { - local script_name="$1" - local pid="$2" + local script_name="$1" pid="$2" local lockfile="$LOCK_DIR/${script_name}.lock" if [[ "$DRY_RUN" == true ]]; then @@ -196,7 +241,7 @@ kill_orchestrator() { sleep 1 if ! kill -0 "$pid" 2>/dev/null; then - success "$script_name stopped ✅" + warn "$script_name stopped (PID $pid) ✅" rm -f "$lockfile" return 0 else @@ -207,70 +252,62 @@ kill_orchestrator() { if [[ "$MODE" == "full-stop" ]]; then echo "" - echo "━━━ $ICON_STOP Orchestrators ━━━" + echo "━━━ $ICON_STOP Kill Orchestrators ━━━" - # Local if [[ -n "$LOCAL_ORCH" ]]; then - name="${LOCAL_ORCH%%:*}" - pid="${LOCAL_ORCH##*:}" - info "Killing local: $name (PID $pid)" - if kill_orchestrator "$name" "$pid"; then - ORCHESTRATORS_KILLED+=("$name") - fi + local_name="${LOCAL_ORCH%%:*}" + local_pid="${LOCAL_ORCH##*:}" + warn "Killing local: $local_name (PID $local_pid)" + kill_orchestrator "$local_name" "$local_pid" && \ + ORCHESTRATORS_KILLED+=("$local_name") else - info "No local orchestrator running" + log "No local orchestrator running" fi - # Remote if [[ "$REMOTE_REACHABLE" == true ]] && [[ -n "$REMOTE_ORCH" ]]; then - name="${REMOTE_ORCH%%:*}" - pid="${REMOTE_ORCH##*:}" - lockfile="$LOCK_DIR/${name}.lock" - info "Killing remote: $name (PID $pid)" + remote_name="${REMOTE_ORCH%%:*}" + remote_pid="${REMOTE_ORCH##*:}" + remote_lock="$LOCK_DIR/${remote_name}.lock" + warn "Killing remote: $remote_name (PID $remote_pid)" if [[ "$DRY_RUN" == false ]]; then - ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ - "kill -TERM '$pid' 2>/dev/null; sleep 2; \ - kill -0 '$pid' 2>/dev/null && kill -KILL '$pid' 2>/dev/null; \ - rm -f '$lockfile'" 2>/dev/null - success "Remote $name stopped ✅" - REMOTE_ORCHESTRATORS_KILLED+=("$name") + timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" \ + root@"$REMOTE_SERVER" \ + "kill -TERM '$remote_pid' 2>/dev/null; sleep 2; \ + kill -0 '$remote_pid' 2>/dev/null && kill -KILL '$remote_pid' 2>/dev/null; \ + rm -f '$remote_lock'" 2>/dev/null + warn "Remote $remote_name stopped ✅" + REMOTE_ORCHESTRATORS_KILLED+=("$remote_name") else - warn "DRY RUN — would kill remote $name (PID $pid)" + warn "DRY RUN — would kill remote $remote_name (PID $remote_pid)" fi elif [[ "$REMOTE_REACHABLE" == true ]]; then - info "No remote orchestrator running" + log "No remote orchestrator running" fi - # Wait for subprocesses to settle - if [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]] || \ - [[ ${#REMOTE_ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then - info "Waiting 3s for subprocesses to settle..." - sleep 3 - fi + [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 || \ + ${#REMOTE_ORCHESTRATORS_KILLED[@]} -gt 0 ]] && sleep 3 fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_STOP Local Rsync ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Local Rsync ━━━ +# ============================================================================================== echo "" echo "━━━ $ICON_STOP Local Rsync ━━━" LOCAL_KILLED=false -LOCAL_PIDS=$(pgrep -x rsync || true) +LOCAL_PIDS=$(pgrep -x rsync 2>/dev/null || true) if [[ -z "$LOCAL_PIDS" ]]; then - info "No rsync processes running locally" + log "No rsync processes running locally" else - info "Found PIDs: $(echo "$LOCAL_PIDS" | tr '\n' ' ')" + warn "Found local rsync PIDs: $(echo "$LOCAL_PIDS" | tr '\n' ' ')" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would kill local rsync" else - if pkill -x rsync; then - success "Local rsync killed ✅" - LOCAL_KILLED=true - else - warn "pkill non-zero — may have already exited" - fi + pkill -x rsync 2>/dev/null && LOCAL_KILLED=true || \ + warn "pkill returned non-zero — rsync may have already exited" + [[ "$LOCAL_KILLED" == true ]] && warn "Local rsync killed ✅" fi fi @@ -280,61 +317,65 @@ for lockfile in "$LOCK_DIR"/rsync_*.lock; do content=$(cat "$lockfile" 2>/dev/null) pid="${content%%:*}" if [[ -n "$pid" ]] && ! kill -0 "$pid" 2>/dev/null; then - info "Cleaning stale lock: $(basename "$lockfile")" + log "Cleaning stale lock: $(basename "$lockfile")" [[ "$DRY_RUN" == false ]] && rm -f "$lockfile" fi done -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_STOP Remote Rsync ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Remote Rsync ━━━ +# ============================================================================================== echo "" -echo "━━━ $ICON_STOP Remote Rsync ($REMOTE_SERVER_NAME) ━━━" +echo "━━━ $ICON_STOP Remote Rsync — $REMOTE_SERVER_NAME ━━━" REMOTE_KILLED=false if [[ "$REMOTE_REACHABLE" == false ]]; then warn "Skipping — $REMOTE_SERVER_NAME unreachable" else - REMOTE_PIDS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ - "pgrep -x rsync || true" 2>/dev/null || true) + REMOTE_PIDS=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" \ + root@"$REMOTE_SERVER" "pgrep -x rsync || true" 2>/dev/null || true) if [[ -z "$REMOTE_PIDS" ]]; then - info "No rsync running on $REMOTE_SERVER_NAME" + log "No rsync running on $REMOTE_SERVER_NAME" else - info "Found PIDs on $REMOTE_SERVER_NAME: $(echo "$REMOTE_PIDS" | tr '\n' ' ')" + warn "Found remote rsync PIDs: $(echo "$REMOTE_PIDS" | tr '\n' ' ')" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would kill remote rsync" else - if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "pkill -x rsync || true" 2>/dev/null; then - success "Remote rsync killed ✅" - REMOTE_KILLED=true - else - warn "Remote pkill non-zero — may have already exited" - fi + timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" \ + root@"$REMOTE_SERVER" "pkill -x rsync || true" 2>/dev/null && \ + REMOTE_KILLED=true || \ + warn "Remote pkill returned non-zero — rsync may have already exited" + [[ "$REMOTE_KILLED" == true ]] && warn "Remote rsync killed ✅" fi fi fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_START $ICON_CONTAINERS Container Recovery ━━━ -# Restart containers left stopped by interrupted rsync -# Only runs if something was actually killed locally -# Remote containers left as-is — docker_watchdog.sh handles remote -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Container Recovery ━━━ +# ============================================================================================== +# Restart local containers left stopped by interrupted rsync. +# Remote containers left for docker_watchdog.sh to recover. +# Skipped with --rsync-only flag (called by other scripts that handle recovery themselves). CONTAINERS_RESTARTED=() +CONTAINERS_FAILED=() + +if [[ "$RSYNC_ONLY_MODE" == false ]] && \ + { [[ "$LOCAL_KILLED" == true ]] || [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; }; then -if [[ "$LOCAL_KILLED" == true ]] || [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then echo "" - echo "━━━ $ICON_START $ICON_CONTAINERS Container Recovery ━━━" - info "Checking all profile containers..." + echo "━━━ $ICON_START Container Recovery ━━━" + log "Checking profile containers for recovery..." declare -A SEEN ALL_CONTAINERS=() - for profile_containers in "${PROFILE_CRITICAL_CONTAINER_NAMES[@]}"; do + for profile_containers in "${PROFILE_CRITICAL_CONTAINER_NAMES[@]:-}"; do read -r -a container_list <<< "$profile_containers" - for c in "${container_list[@]}"; do + for c in "${container_list[@]:-}"; do [[ -z "$c" ]] && continue if [[ -z "${SEEN[$c]:-}" ]]; then SEEN[$c]=1 @@ -344,69 +385,89 @@ if [[ "$LOCAL_KILLED" == true ]] || [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; the done if [[ ${#ALL_CONTAINERS[@]} -eq 0 ]]; then - info "No containers defined — skipping recovery" + log "No profile containers defined — skipping recovery" else for c in "${ALL_CONTAINERS[@]}"; do - STATUS=$(docker inspect -f '{{.State.Running}}' "$c" 2>/dev/null || echo "unknown") - if [[ "$STATUS" == "true" ]]; then - info "$ICON_RUNNING $c — running ✅" - elif [[ "$STATUS" == "false" ]]; then - warn "$ICON_NOT_RUNNING $c — stopped, restarting..." - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would restart $c" - else - if docker start "$c" >/dev/null 2>&1; then - success "$c restarted ✅" - CONTAINERS_RESTARTED+=("$c") + STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ + '{{.State.Running}}' "$c" 2>/dev/null || echo "unknown") + case "$STATUS" in + true) + log "$c — running ✅" + ;; + false) + warn "$c — stopped — restarting..." + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would restart $c" else - error "Failed to restart $c" + if timeout "$DOCKER_TIMEOUT" docker start "$c" >/dev/null 2>&1; then + warn "$c restarted ✅" + CONTAINERS_RESTARTED+=("$c") + else + error "Failed to restart $c" + CONTAINERS_FAILED+=("$c") + fi fi - fi - else - info "$c not found on this host — skipping" - fi + ;; + *) + log "$c not found locally — skipping" + ;; + esac done fi fi -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY RSYNC STOP SUMMARY ━━━━━" -echo " Mode: $MODE" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_GEAR Mode: $MODE" echo "" -echo "$ICON_HOST Local ($LOCAL_SERVER_NAME):" +echo "$ICON_HOST Local ($MY_ID):" [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]] && \ - echo " $ICON_STOPPED Orchestrators killed: ${ORCHESTRATORS_KILLED[*]}" -[[ "$LOCAL_KILLED" == true ]] && \ - echo " $ICON_STOPPED Rsync killed" || \ - echo " $ICON_SUCCESS No rsync was running" + warn " Orchestrators killed: ${ORCHESTRATORS_KILLED[*]}" +if [[ "$LOCAL_KILLED" == true ]]; then + warn " Rsync killed ✅" +else + log " No rsync was running" +fi -echo "$ICON_NET Remote ($REMOTE_SERVER_NAME):" +echo "$ICON_NET Remote ($REMOTE_ID — $REMOTE_SERVER_NAME):" if [[ "$REMOTE_REACHABLE" == false ]]; then - echo " $ICON_WARN Unreachable — skipped" + warn " Unreachable — skipped" else [[ ${#REMOTE_ORCHESTRATORS_KILLED[@]} -gt 0 ]] && \ - echo " $ICON_STOPPED Orchestrators killed: ${REMOTE_ORCHESTRATORS_KILLED[*]}" - [[ "$REMOTE_KILLED" == true ]] && \ - echo " $ICON_STOPPED Rsync killed" || \ - echo " $ICON_SUCCESS No rsync was running" + warn " Orchestrators killed: ${REMOTE_ORCHESTRATORS_KILLED[*]}" + if [[ "$REMOTE_KILLED" == true ]]; then + warn " Rsync killed ✅" + else + log " No rsync was running" + fi fi [[ ${#CONTAINERS_RESTARTED[@]} -gt 0 ]] && \ - echo "$ICON_CONTAINERS Containers recovered: ${CONTAINERS_RESTARTED[*]}" + warn "$ICON_CONTAINERS Containers recovered: ${CONTAINERS_RESTARTED[*]}" +[[ ${#CONTAINERS_FAILED[@]} -gt 0 ]] && \ + echo "$ICON_ERROR Containers failed to restart: ${CONTAINERS_FAILED[*]}" +echo "" if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no changes made" + warn "DRY RUN — no changes made" else - echo "$ICON_DONE Status: $ICON_SUCCESS DONE" + log "$ICON_DONE Status: done ✅" fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -if [[ "$LOCAL_KILLED" == true ]] || [[ "$REMOTE_KILLED" == true ]] || \ - [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then - notify "Rsync stopped on $(hostname) — mode: $MODE — ${#CONTAINERS_RESTARTED[@]} containers recovered" \ - "Rsync Stop" "warning" +# Notify if anything was actually killed or failed +if [[ "$DRY_RUN" == false ]]; then + if [[ ${#CONTAINERS_FAILED[@]} -gt 0 ]]; then + notify "Rsync stop on $(hostname) ($MY_ID) — containers failed to restart: ${CONTAINERS_FAILED[*]}" \ + "Rsync Stop" "warning" + elif [[ "$LOCAL_KILLED" == true || "$REMOTE_KILLED" == true || \ + ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then + notify "Rsync stopped on $(hostname) ($MY_ID) — mode: $MODE${CONTAINERS_RESTARTED:+ — recovered: ${CONTAINERS_RESTARTED[*]}}" \ + "Rsync Stop" "warning" + fi fi \ No newline at end of file diff --git a/unRAID_Essentials/server_reboot.sh b/unRAID_Essentials/server_reboot.sh index 295be85..1ece51f 100644 --- a/unRAID_Essentials/server_reboot.sh +++ b/unRAID_Essentials/server_reboot.sh @@ -1,159 +1,288 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Server Reboot Script --------------------------------------- -# ----------------------------------------------------------------------------------------------- -# Gracefully reboots the unRAID server with a configurable user warning delay. -# Stops Docker and VM Manager cleanly before issuing reboot. -# Reboot delay is configured in Master.conf under REBOOT_SLEEP. -# Supports --dry-run to walk through the sequence without actually rebooting. -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= Server Reboot ============================================== +# ============================================================================================== +# Gracefully reboots the unRAID server with full pre-flight checks and clean shutdown sequence. +# Warns all users, checks for active processes, stops services, syncs disks, then reboots. +# +# ── SHUTDOWN SEQUENCE ───────────────────────────────────────────────────────────────────────── +# 1. Pre-flight warnings — rsync, mover, active Emby sessions (warn not block) +# 2. Wall message to all logged-in terminal users +# 3. unRAID notification to dashboard +# 4. Wait REBOOT_SLEEP seconds (default 30) — gives users time to save work +# 5. Gracefully shutdown VMs (virsh shutdown each, then wait) +# 6. Stop libvirt (VM Manager) +# 7. Stop Docker service +# 8. Sync filesystem buffers to disk +# 9. Reboot +# +# ── PRE-FLIGHT WARNINGS ─────────────────────────────────────────────────────────────────────── +# The following are warnings only — they do not block the reboot. You called this script, +# so you know what you're doing. The warnings give you context before the countdown starts. +# - rsync running → partial files possible if mid-transfer +# - mover running → files may be left on cache or array mid-move +# - Emby sessions → active streams/transcodes will be interrupted +# +# ── VM GRACEFUL SHUTDOWN ────────────────────────────────────────────────────────────────────── +# virsh shutdown sends ACPI power button signal to each VM — same as pressing power button. +# VM gets a chance to flush its own buffers and shutdown cleanly. +# Waits REBOOT_VM_WAIT seconds (default 30) for VMs to shut down before stopping libvirt. +# If VMs don't shut down in time libvirt stops anyway — system reboot takes priority. +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID — used in wall message, notification, and summary. +# Critical on a two-server setup — wall and notifications show WHICH server is rebooting. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# Root check — reboot requires root +# acquire_lock — prevents concurrent reboot calls +# detect_hosts() — MY_ID in all user-facing messages +# validate_unraid_cmd — notify validated before use +# Graceful VM shutdown — VMs get clean ACPI signal before libvirt stops +# sync before reboot — filesystem buffers flushed to disk +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# REBOOT_SLEEP — seconds to warn users before starting shutdown sequence (default 30) +# REBOOT_VM_WAIT — seconds to wait for VMs to shut down gracefully (default 30) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# server_reboot.sh — reboot with 30s warning +# server_reboot.sh --dry-run — walk through sequence without rebooting +# server_reboot.sh --status — show running processes that would be affected +# server_reboot.sh --reason="maintenance" — log reason for reboot +# server_reboot.sh --log — verbose output +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" -parse_args "$@" +# ── Parse --reason flag before parse_args ───────────────────────────────────────────────────── +REBOOT_REASON="manual" +FILTERED_ARGS=() +for arg in "$@"; do + case "$arg" in + --reason=*) REBOOT_REASON="${arg#--reason=}" ;; + *) FILTERED_ARGS+=("$arg") ;; + esac +done -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" +parse_args "${FILTERED_ARGS[@]}" -# ROOT CHECK +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root" + error "Must be run as root — reboot requires root" exit 1 fi -success "Running as root" +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" -# VALIDATION validate_int REBOOT_SLEEP "$REBOOT_SLEEP" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +acquire_lock + +detect_hosts + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made, no reboot will occur" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" - echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_REBOOT Delay: ${REBOOT_SLEEP}s" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━ $ICON_SUMMARY REBOOT STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_REBOOT Delay: ${REBOOT_SLEEP}s" + echo "$ICON_GEAR VM wait: ${REBOOT_VM_WAIT:-30}s" + echo "$ICON_GEAR Reason: $REBOOT_REASON" + echo "" + echo "━━━ Active Processes ━━━" + + pgrep -x rsync >/dev/null 2>&1 && \ + warn " rsync: RUNNING — partial files if rebooted now" || \ + log " rsync: not running" + + pgrep -f "emhttp.*Mover" >/dev/null 2>&1 && \ + warn " mover: RUNNING — files may be left mid-move" || \ + log " mover: not running" + + if command -v virsh >/dev/null 2>&1; then + VM_COUNT=$(virsh list --name 2>/dev/null | grep -c "." || echo 0) + [[ "$VM_COUNT" -gt 0 ]] && \ + warn " VMs: $VM_COUNT running — will be gracefully shut down" || \ + log " VMs: none running" + fi + + if command -v docker >/dev/null 2>&1; then + CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l || echo 0) + log " Docker: $CONTAINER_COUNT container(s) running" + fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -# ----------------------------------------------------------------------------------------------- -# FUNCTIONS -# ----------------------------------------------------------------------------------------------- - -# Broadcasts a wall message warning all logged in users of the upcoming reboot. -notify_users() { - warn "Notifying users — reboot in ${REBOOT_SLEEP}s" - wall "$ICON_WARN unRAID server will reboot in ${REBOOT_SLEEP} second(s). Save your work." -} - -# Stops the Docker service cleanly. -# Warns but continues if Docker is already stopped or fails — shutdown must proceed. -stop_docker() { - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would stop Docker service" - return - fi - - info "Stopping Docker service..." - - if /etc/rc.d/rc.docker stop; then - success "Docker stopped" - else - warn "Docker stop failed or already stopped — continuing" - fi -} - -# Stops the VM Manager (libvirt) cleanly. -# Warns but continues if libvirt is already stopped or fails — shutdown must proceed. -stop_vm_manager() { - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would stop VM Manager (libvirt)" - return - fi - - info "Stopping VM Manager..." - - if /etc/rc.d/rc.libvirt stop; then - success "VM Manager stopped" - else - warn "VM Manager stop failed or already stopped — continuing" - fi -} - -# Flushes filesystem buffers to disk before reboot. -sync_disks() { - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would sync filesystem buffers" - return - fi - - info "Syncing disks..." - - if sync; then - success "Disk sync complete" - else - warn "Sync returned an error — continuing" - fi -} - -# Issues the system reboot command. -# System will not return from this call unless dry-run is active. -reboot_system() { - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would reboot system now" - return - fi - - echo "" - echo "$ICON_REBOOT Rebooting system NOW..." - /sbin/reboot -} - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_REBOOT Reboot Sequence ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_REBOOT Reboot Sequence ━━━" -echo "$ICON_REBOOT Delay: ${REBOOT_SLEEP}s" -echo "$ICON_GEAR Dry Run: $DRY_RUN" +# ============================================================================================== +# ━━━ Pre-flight Warnings ━━━ +# ============================================================================================== echo "" +echo "━━━ $ICON_SHIELD Pre-flight ━━━" +WARNINGS=() -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" +# rsync check — partial files if killed mid-transfer +if pgrep -x rsync >/dev/null 2>&1; then + RSYNC_PIDS=$(pgrep -x rsync | tr '\n' ' ') + warn "rsync is running (PIDs: $RSYNC_PIDS) — partial files possible" + warn "Consider: rsync_stop.sh before rebooting" + WARNINGS+=("rsync running") +fi + +# mover check — files may be left mid-move +if pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then + warn "Mover is running — files may be left mid-move on cache or array" + warn "Consider: mover_stop.sh before rebooting" + WARNINGS+=("mover running") +fi + +# Emby sessions check — active streams interrupted +if [[ -n "${EMBY_URL:-}" ]] && [[ -n "${EMBY_API_KEY:-}" ]]; then + ACTIVE_STREAMS=$(curl -sf --max-time 5 \ + -H "X-Emby-Token: $EMBY_API_KEY" \ + "${EMBY_URL}/Sessions" 2>/dev/null | \ + grep -c "NowPlayingItem" 2>/dev/null || echo 0) + ACTIVE_STREAMS="${ACTIVE_STREAMS//[^0-9]/}" + if [[ "${ACTIVE_STREAMS:-0}" -gt 0 ]]; then + warn "$ACTIVE_STREAMS active Emby stream(s) — will be interrupted" + WARNINGS+=("${ACTIVE_STREAMS} Emby sessions") + fi +fi + +if [[ ${#WARNINGS[@]} -eq 0 ]]; then + log "Pre-flight clean — no active processes to warn about" +else + warn "Proceeding with reboot despite warnings — ${WARNINGS[*]}" +fi + +# ============================================================================================== +# ━━━ Notify and Wait ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_REBOOT Reboot Sequence — $MY_ID ━━━" +echo " Reason: $REBOOT_REASON" +echo " Delay: ${REBOOT_SLEEP}s" +echo " Dry Run: $DRY_RUN" +echo "" START=$(date +%s) if [[ "$REBOOT_SLEEP" -gt 0 ]]; then - notify_users - info "Waiting ${REBOOT_SLEEP}s before shutdown sequence..." - sleep "$REBOOT_SLEEP" + # Wall message — terminal users + wall "$ICON_WARN $MY_ID ($LOCAL_SERVER_NAME) rebooting in ${REBOOT_SLEEP}s — reason: $REBOOT_REASON. Save your work now." + + # unRAID notification — dashboard + if [[ "$DRY_RUN" == false ]]; then + notify "$MY_ID ($LOCAL_SERVER_NAME) rebooting in ${REBOOT_SLEEP}s — reason: $REBOOT_REASON${WARNINGS:+ — warnings: ${WARNINGS[*]}}" \ + "Server Reboot" "warning" + fi + + warn "Waiting ${REBOOT_SLEEP}s before shutdown sequence..." + if [[ "$DRY_RUN" == false ]]; then + sleep "$REBOOT_SLEEP" + else + warn "DRY RUN — skipping sleep" + fi fi -stop_docker -stop_vm_manager -sync_disks -reboot_system +# ============================================================================================== +# ━━━ Graceful VM Shutdown ━━━ +# ============================================================================================== +if command -v virsh >/dev/null 2>&1; then + VM_LIST=$(virsh list --name 2>/dev/null | grep -v "^$" || true) + if [[ -n "$VM_LIST" ]]; then + echo "" + echo "━━━ $ICON_GEAR Graceful VM Shutdown ━━━" + while IFS= read -r vm; do + [[ -z "$vm" ]] && continue + warn "Sending ACPI shutdown to VM: $vm" + if [[ "$DRY_RUN" == false ]]; then + virsh shutdown "$vm" >/dev/null 2>&1 || true + else + warn "DRY RUN — would virsh shutdown $vm" + fi + done <<< "$VM_LIST" -# NOTE: system will not reach here unless --dry-run is active + if [[ "$DRY_RUN" == false ]]; then + VM_WAIT="${REBOOT_VM_WAIT:-30}" + log "Waiting ${VM_WAIT}s for VMs to shut down..." + sleep "$VM_WAIT" + fi + fi +fi + +# ============================================================================================== +# ━━━ Stop VM Manager ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_GEAR Stop VM Manager ━━━" +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would stop VM Manager (libvirt)" +else + if /etc/rc.d/rc.libvirt stop >/dev/null 2>&1; then + warn "VM Manager stopped ✅" + else + warn "VM Manager stop returned non-zero — may already be stopped" + fi +fi + +# ============================================================================================== +# ━━━ Stop Docker ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_CONTAINERS Stop Docker ━━━" +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would stop Docker service" +else + if /etc/rc.d/rc.docker stop >/dev/null 2>&1; then + warn "Docker stopped ✅" + else + warn "Docker stop returned non-zero — may already be stopped" + fi +fi + +# ============================================================================================== +# ━━━ Sync Disks ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_DISK Sync Disks ━━━" +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would sync filesystem buffers" +else + sync + log "Filesystem buffers flushed ✅" +fi + +# ============================================================================================== +# ━━━ Reboot ━━━ +# ============================================================================================== END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- echo "" echo "━━━━━ $ICON_SUMMARY REBOOT SUMMARY ━━━━━" -echo "$ICON_REBOOT Delay: ${REBOOT_SLEEP}s" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_REBOOT Reason: $REBOOT_REASON" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" +[[ ${#WARNINGS[@]} -gt 0 ]] && warn "Warnings: ${WARNINGS[*]}" +echo "" + if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no reboot executed" + warn "DRY RUN — sequence complete, no reboot executed" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" else - echo "$ICON_REBOOT Status: $ICON_WARN SYSTEM SHOULD BE REBOOTING" -fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file + warn "$ICON_REBOOT Rebooting $MY_ID now..." + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + /sbin/reboot +fi \ No newline at end of file diff --git a/unRAID_Essentials/system_watchdog.sh b/unRAID_Essentials/system_watchdog.sh index 0479f53..dde85a4 100644 --- a/unRAID_Essentials/system_watchdog.sh +++ b/unRAID_Essentials/system_watchdog.sh @@ -1,118 +1,170 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- System Watchdog -------------------------------------------- -# ----------------------------------------------------------------------------------------------- -# Last line of defense — reb on oots the system cleanly if it is about to become unstable. +# ============================================================================================== +# ================================= System Watchdog ============================================ +# ============================================================================================== +# Last line of defense — reboots the system cleanly if it is about to become unstable. # Runs continuously as a background process — started by array_start.sh at array start. # Works alongside docker_watchdog.sh which handles container-level healing first. # -# Checks (all toggleable in Master.conf): -# rootfs usage — high rootfs fills rapidly when array is down, crash imminent -# /var/log usage — log spam can fill rootfs, indicates something is broken -# free RAM — critically low RAM means OOM or swap imminent -# ZFS ARC pinned — ARC not releasing after reclaim means memory is stuck -# CPU temperature — sustained tjmax causes throttling or kernel panic -# load average — sustained high load means something is stuck or runaway -# zombie processes — large zombie count indicates serious process management failure -# Docker daemon — unresponsive daemon means containers cannot be managed -# Required containers — stopped containers that should be running (after watchdog skip list) +# ── THREE-TIER RESPONSE SYSTEM ──────────────────────────────────────────────────────────────── # -# Abort conditions (toggleable): -# ZFS pool unhealthy — reboot with bad pool risks data loss -# Parity running — aborting parity is better than crashing mid-check -# Mover running — aborting move is better than crashing mid-move +# TIER 1 — CRITICAL (bypass ALL strikes, reboot immediately) +# Docker daemon unresponsive — nothing can be healed, letting it run makes it worse +# rootfs at 99%+ — writes failing, SSH may stop, no recovery options +# Kernel oops/BUG in dmesg — kernel running with corrupted state +# File descriptor exhaustion — new connections and processes failing silently +# /boot read-only unexpectedly — state files and config writes silently failing # -# Reboot loop protection: -# Tracks reboot timestamps in persistent log on /boot/ -# Rolling window — old entries purge automatically -# If reboot count hits limit in window → shutdown instead of reboot +# TIER 2 — URGENT (bypass strikes when OOM confirms active crisis) +# RAM < MEM_GB AND OOM kills >= OOM_LIMIT in this cycle +# Rationale: OOM kills at this rate means system is dying faster than watchdogs heal +# Without OOM confirmation → standard strike system applies # -# Continuous loop: -# Checks run every SYSTEM_WATCHDOG_INTERVAL seconds (default 300 = 5min) -# Master.conf re-sourced each cycle — config changes picked up without restart -# Silent when healthy — only verbose when trigger or reboot -# Clean shutdown on SIGTERM/SIGINT — sent by array stop +# TIER 3 — STANDARD (N consecutive failures → reboot) +# RAM tiers, load, CPU temp, zombies, /var/log, /tmp, containers, NIC, mdstat # -# All configuration in Master.conf under System Watchdog section. -# Supports --dry-run to show triggered conditions without rebooting. -# ----------------------------------------------------------------------------------------------- +# ── RAM TIERS ───────────────────────────────────────────────────────────────────────────────── +# MEM_WARN_GB (10GB) — warn + notify only +# MEM_SHUTDOWN_GB (6GB) — stop non-essential containers, wait for recovery +# MEM_GB (4GB) — strike system → reboot (bypass if OOM confirms) +# MEM_RECOVER_GB (30GB) — RAM must reach this before containers restart +# +# ── CONTAINER SHUTDOWN LOGIC ────────────────────────────────────────────────────────────────── +# At MEM_SHUTDOWN_GB: stop all containers NOT in SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED +# Excluded: NginxProxyManager, Authelia, Mariadb, Redis, Emby, Dispatcharr +# Stopped containers tracked in shutdown list — won't restart until RAM recovers +# Strike system prevents flip-flopping — shutdown only happens once per degradation event +# +# ── OOM TRACKING ────────────────────────────────────────────────────────────────────────────── +# /proc/vmstat oom_kill counter — read each cycle, delta = kills this cycle +# Included in reboot message with process names from dmesg (diagnostic context) +# Bypass trigger: RAM critical AND kills this cycle >= SYS_WATCHDOG_OOM_LIMIT +# +# ── NEW CHECKS THIS VERSION ─────────────────────────────────────────────────────────────────── +# OOM rate tracking — delta from /proc/vmstat each cycle +# /boot read-only — write test on /boot each cycle +# Kernel oops detection — dmesg BUG/Oops count delta each cycle +# File descriptor exhaustion — /proc/sys/fs/file-nr utilisation +# /tmp usage — tmpfs fill detection with auto-clear attempt +# Array disk errors — mdstat error delta each cycle +# Runaway process — single process >N% CPU sustained (disabled by default) +# NIC state check — primary interface operstate +# sshd check — restart attempt before escalating +# +# ── EXISTING CHECKS ─────────────────────────────────────────────────────────────────────────── +# rootfs usage, /var/log, free RAM, ZFS ARC, CPU temp, load avg, +# zombie processes, Docker daemon, required containers from skip list +# +# ── ABORT CONDITIONS ────────────────────────────────────────────────────────────────────────── +# ZFS pool unhealthy, parity running, mover running — toggleable +# CRITICAL tier bypasses abort conditions — imminent crash overrides data safety +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# Full config under System Watchdog section — see master.conf for all vars +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# system_watchdog.sh — normal start (continuous loop) +# system_watchdog.sh --dry-run — trigger detection without rebooting +# system_watchdog.sh --status — show config and thresholds +# system_watchdog.sh --log — verbose per-cycle output +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup — runs once at start ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" - +# ============================================================================================== +# ━━━ Setup — runs once at start ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi -success "Running as root" +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" acquire_lock "continuous" +detect_hosts + TOTAL_CORES=$(nproc) +DOCKER_TIMEOUT=10 -# Ensure state and persistent files exist -touch "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || { - error "Cannot create state file: $SYS_WATCHDOG_STATE_FILE" - exit 1 -} +# Ensure state files exist +for state_file in "$SYS_WATCHDOG_STATE_FILE" "$SYS_WATCHDOG_REBOOT_LOG" \ + "$SYS_WATCHDOG_FAILED_FILE" "$SYS_WATCHDOG_OOM_FILE"; do + touch "$state_file" 2>/dev/null || { + error "Cannot create state file: $state_file" + exit 1 + } +done -touch "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || { - error "Cannot create reboot log: $SYS_WATCHDOG_REBOOT_LOG" - exit 1 -} +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboots or container shutdowns will occur" -touch "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || { - error "Cannot create failed container list: $SYS_WATCHDOG_FAILED_FILE" - exit 1 -} - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" - echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_HEALTH rootfs threshold: ${SYS_WATCHDOG_ROOTFS_PCT}% toggle: $SYS_WATCHDOG_CHECK_ROOTFS" - echo "$ICON_HEALTH log threshold: ${SYS_WATCHDOG_LOG_PCT}% toggle: $SYS_WATCHDOG_CHECK_LOG" - echo "$ICON_MEM RAM threshold: ${SYS_WATCHDOG_MEM_GB}GB free toggle: $SYS_WATCHDOG_CHECK_RAM" - echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}% toggle: $SYS_WATCHDOG_CHECK_ARC" - echo "$ICON_ZFS ARC release: ${SYS_WATCHDOG_ARC_RELEASE_PCT}%" - echo "$ICON_GEAR CPU temp max: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C toggle: $SYS_WATCHDOG_CHECK_CPU_TEMP" - echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x cores toggle: $SYS_WATCHDOG_CHECK_LOAD" - echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT} toggle: $SYS_WATCHDOG_CHECK_ZOMBIES" - echo "$ICON_CONTAINERS Docker daemon: toggle: $SYS_WATCHDOG_CHECK_DOCKER_DAEMON" - echo "$ICON_CONTAINERS Containers: toggle: $SYS_WATCHDOG_CHECK_CONTAINERS" - echo "$ICON_SHIELD Strike limit: $SYS_WATCHDOG_STRIKE_LIMIT" - echo "$ICON_TIME Interval: ${SYSTEM_WATCHDOG_INTERVAL}s" - echo "$ICON_REBOOT_SMART Reboot limit: $SYS_WATCHDOG_REBOOT_LIMIT in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs" - echo "$ICON_ZFS Abort ZFS unhealthy: $SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" - echo "$ICON_GEAR Abort on parity: $SYS_WATCHDOG_ABORT_ON_PARITY" - echo "$ICON_MOVER Abort on mover: $SYS_WATCHDOG_ABORT_ON_MOVER" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━ $ICON_SUMMARY SYSTEM WATCHDOG STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "" + echo "── Tier 1 — CRITICAL (bypass strikes immediately) ──" + echo "$ICON_DISK rootfs critical: ${SYS_WATCHDOG_ROOTFS_CRITICAL_PCT}%" + echo "$ICON_GEAR FD critical: ${SYS_WATCHDOG_FD_CRITICAL_PCT}%" + echo "$ICON_GEAR /boot read-only: check=${SYS_WATCHDOG_CHECK_BOOT}" + echo "$ICON_GEAR Kernel oops: check=${SYS_WATCHDOG_CHECK_KERNEL_OOPS}" + echo "$ICON_CONTAINERS Docker daemon: check=${SYS_WATCHDOG_CHECK_DOCKER_DAEMON}" + echo "" + echo "── Tier 2 — URGENT (bypass strikes with OOM confirmation) ──" + echo "$ICON_MEM RAM critical: < ${SYS_WATCHDOG_MEM_GB}GB" + echo "$ICON_GEAR OOM limit: ${SYS_WATCHDOG_OOM_LIMIT} kills/cycle" + echo "" + echo "── Tier 3 — STANDARD (strike system) ──" + echo "$ICON_DISK rootfs warn: ${SYS_WATCHDOG_ROOTFS_PCT}%" + echo "$ICON_GEAR /var/log warn: ${SYS_WATCHDOG_LOG_PCT}%" + echo "$ICON_GEAR /tmp warn: ${SYS_WATCHDOG_TMP_PCT}%" + echo "$ICON_MEM RAM warn: < ${SYS_WATCHDOG_MEM_WARN_GB}GB" + echo "$ICON_MEM RAM shutdown: < ${SYS_WATCHDOG_MEM_SHUTDOWN_GB}GB" + echo "$ICON_MEM RAM recover: > ${SYS_WATCHDOG_MEM_RECOVER_GB}GB" + echo "$ICON_MEM RAM reboot: < ${SYS_WATCHDOG_MEM_GB}GB (+ strikes)" + echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}%" + echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x (= $(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER )) on $TOTAL_CORES cores)" + echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT}" + echo "$ICON_GEAR CPU temp max: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C" + echo "$ICON_GEAR Strike limit: ${SYS_WATCHDOG_STRIKE_LIMIT} cycles" + echo "$ICON_TIME Interval: ${SYSTEM_WATCHDOG_INTERVAL}s" + echo "$ICON_REBOOT_SMART Reboot limit: ${SYS_WATCHDOG_REBOOT_LIMIT} in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr" + echo "" + echo "── Container Shutdown Excluded ──" + for c in "${SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED[@]:-}"; do + echo " $ICON_RUNNING $c" + done + echo "" + echo "── Check Toggles ──" + echo " rootfs=$SYS_WATCHDOG_CHECK_ROOTFS log=$SYS_WATCHDOG_CHECK_LOG ram=$SYS_WATCHDOG_CHECK_RAM" + echo " arc=$SYS_WATCHDOG_CHECK_ARC cpu_temp=$SYS_WATCHDOG_CHECK_CPU_TEMP load=$SYS_WATCHDOG_CHECK_LOAD" + echo " zombies=$SYS_WATCHDOG_CHECK_ZOMBIES docker=$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" + echo " containers=$SYS_WATCHDOG_CHECK_CONTAINERS oom=$SYS_WATCHDOG_CHECK_OOM" + echo " tmp=$SYS_WATCHDOG_CHECK_TMP fd=$SYS_WATCHDOG_CHECK_FD boot=$SYS_WATCHDOG_CHECK_BOOT" + echo " kernel_oops=$SYS_WATCHDOG_CHECK_KERNEL_OOPS sshd=$SYS_WATCHDOG_CHECK_SSHD" + echo " network=$SYS_WATCHDOG_CHECK_NETWORK mdstat=$SYS_WATCHDOG_CHECK_MDSTAT" + echo " runaway=$SYS_WATCHDOG_CHECK_RUNAWAY" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboot will be executed" - -# ----------------------------------------------------------------------------------------------- -# STATE HELPERS — defined once, used every cycle -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ── STATE HELPERS ───────────────────────────────────────────────────────────────────────────── +# ============================================================================================== get_strikes() { - local key="$1" - grep -E "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2 + grep -E "^${1}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2 } set_strikes() { @@ -127,20 +179,30 @@ increment_strikes() { local current current=$(get_strikes "$key") [[ -z "$current" ]] && current=0 - ((current++)) + (( current++ )) set_strikes "$key" "$current" echo "$current" } reset_strikes() { - local key="$1" - set_strikes "$key" 0 + set_strikes "$1" 0 +} + +get_state_val() { + grep -E "^${1}=" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d'=' -f2 +} + +set_state_val() { + local key="$1" val="$2" + grep -vE "^${key}=" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp" + echo "${key}=${val}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp" + mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE" } purge_old_reboots() { - local now + local now cutoff now=$(date +%s) - local cutoff=$(( now - SYS_WATCHDOG_REBOOT_WINDOW )) + cutoff=$(( now - SYS_WATCHDOG_REBOOT_WINDOW )) grep -v "^$" "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | while IFS= read -r ts; do [[ "$ts" -gt "$cutoff" ]] && echo "$ts" done > "${SYS_WATCHDOG_REBOOT_LOG}.tmp" @@ -156,6 +218,39 @@ log_reboot() { date +%s >> "$SYS_WATCHDOG_REBOOT_LOG" } +# ============================================================================================== +# ── OOM TRACKING ────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Reads /proc/vmstat oom_kill counter — delta per cycle = rate of OOM kills +# Used for Tier 2 bypass and diagnostic context in reboot messages + +get_oom_delta() { + local current_oom + current_oom=$(grep "^oom_kill " /proc/vmstat 2>/dev/null | awk '{print $2}') + [[ -z "$current_oom" ]] && echo 0 && return + + local prev_oom + prev_oom=$(cat "$SYS_WATCHDOG_OOM_FILE" 2>/dev/null || echo 0) + echo "$current_oom" > "$SYS_WATCHDOG_OOM_FILE" + + local delta=$(( current_oom - prev_oom )) + [[ "$delta" -lt 0 ]] && delta=0 # counter reset on reboot + echo "$delta" +} + +get_oom_victims() { + # Get process names from dmesg that were OOM killed this boot + dmesg -T 2>/dev/null | grep -i "Killed process" | \ + awk '{print $NF}' | sort | uniq -c | sort -rn | head -5 | \ + awk '{printf "%s×%d ", $2, $1}' | sed 's/ $//' +} + +# ============================================================================================== +# ── ABORT CONDITIONS ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Returns 1 if reboot should be aborted, 0 if reboot should proceed +# CRITICAL tier bypasses this function entirely + check_abort_conditions() { local should_abort=false @@ -164,34 +259,35 @@ check_abort_conditions() { unhealthy=$(zpool list -H -o health 2>/dev/null | grep -v ONLINE || true) if [[ -n "$unhealthy" ]]; then if [[ "$SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" == true ]]; then - error "$ICON_ZFS ZFS pool unhealthy — aborting reboot" - notify "System watchdog aborted reboot on $(hostname) — ZFS pool unhealthy" "System Watchdog" "warning" + error "ZFS pool unhealthy — aborting reboot to prevent data loss" + notify "System watchdog aborted reboot on $(hostname) ($MY_ID) — ZFS pool unhealthy" \ + "System Watchdog" "warning" should_abort=true else - warn "$ICON_ZFS ZFS pool unhealthy — continuing reboot" + warn "ZFS pool unhealthy — continuing reboot (ABORT_ON_ZFS_UNHEALTHY=false)" fi fi fi - if [[ -f /var/local/emhttp/parity-date.txt ]]; then - if grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null; then - if [[ "$SYS_WATCHDOG_ABORT_ON_PARITY" == true ]]; then - error "$ICON_DISK Parity check running — aborting reboot" - notify "System watchdog aborted reboot on $(hostname) — parity running" "System Watchdog" "warning" - should_abort=true - else - warn "$ICON_DISK Parity check running — continuing reboot" - fi + if grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null; then + if [[ "$SYS_WATCHDOG_ABORT_ON_PARITY" == true ]]; then + error "Parity check running — aborting reboot" + notify "System watchdog aborted reboot on $(hostname) ($MY_ID) — parity running" \ + "System Watchdog" "warning" + should_abort=true + else + warn "Parity check running — continuing reboot (ABORT_ON_PARITY=false)" fi fi if pgrep -f "mover" >/dev/null 2>&1; then if [[ "$SYS_WATCHDOG_ABORT_ON_MOVER" == true ]]; then - error "$ICON_MOVER Mover running — aborting reboot" - notify "System watchdog aborted reboot on $(hostname) — mover running" "System Watchdog" "warning" + error "Mover running — aborting reboot" + notify "System watchdog aborted reboot on $(hostname) ($MY_ID) — mover running" \ + "System Watchdog" "warning" should_abort=true else - warn "$ICON_MOVER Mover running — continuing reboot" + warn "Mover running — continuing reboot (ABORT_ON_MOVER=false)" fi fi @@ -199,6 +295,11 @@ check_abort_conditions() { return 0 } +# ============================================================================================== +# ── STANDARD STRIKE CHECK ───────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Returns 0 = reboot now | 1 = not yet + run_strike_check() { local key="$1" triggered="$2" description="$3" if [[ "$triggered" == true ]]; then @@ -206,33 +307,101 @@ run_strike_check() { strikes=$(increment_strikes "$key") warn "$description — strike $strikes/$SYS_WATCHDOG_STRIKE_LIMIT" if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then - error "$description hit strike limit — reboot triggered" + error "$description — strike limit hit, reboot triggered" reset_strikes "$key" return 0 fi else local current current=$(get_strikes "$key") - if [[ -n "$current" && "$current" -gt 0 ]]; then - reset_strikes "$key" - fi + [[ -n "$current" && "$current" -gt 0 ]] && reset_strikes "$key" fi return 1 } +# ============================================================================================== +# ── CONTAINER SHUTDOWN (RAM EMERGENCY) ──────────────────────────────────────────────────────── +# ============================================================================================== + +shutdown_non_essential_containers() { + warn "RAM emergency — stopping non-essential containers" + local stopped=() + + # Build exclusion map + declare -A EXCLUDED_MAP + for exc in "${SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED[@]:-}"; do + [[ -n "$exc" ]] && EXCLUDED_MAP["$exc"]=1 + done + + # Stop all running containers not in exclusion list + while IFS= read -r container; do + [[ -z "$container" ]] && continue + if [[ -n "${EXCLUDED_MAP[$container]:-}" ]]; then + log "$container — excluded from RAM shutdown, leaving running" + continue + fi + if [[ "$DRY_RUN" == false ]]; then + timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1 && \ + warn "Stopped $container (RAM emergency)" && \ + stopped+=("$container") || \ + error "Failed to stop $container" + else + warn "DRY RUN — would stop $container (RAM emergency)" + stopped+=("$container") + fi + done < <(timeout "$DOCKER_TIMEOUT" docker ps --format "{{.Names}}" 2>/dev/null) + + if [[ ${#stopped[@]} -gt 0 ]]; then + set_state_val "mem_shutdown_active" "true" + notify "RAM emergency on $(hostname) ($MY_ID) — stopped ${#stopped[@]} containers. Excluded: ${SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED[*]}" \ + "System Watchdog" "warning" + warn "Stopped ${#stopped[@]} containers — waiting for RAM to recover above ${SYS_WATCHDOG_MEM_RECOVER_GB}GB" + fi +} + +restart_non_essential_containers() { + warn "RAM recovered — restarting containers that were stopped in emergency" + if [[ "$DRY_RUN" == false ]]; then + set_state_val "mem_shutdown_active" "false" + fi + # docker_watchdog.sh will detect stopped required containers and restart them + # We just clear the state flag here + warn "Cleared RAM emergency state — docker_watchdog.sh will restart required containers" +} + +# ============================================================================================== +# ── DO REBOOT ───────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# tier: "critical" (bypass abort) | "urgent" | "standard" + do_reboot() { + local tier="${1:-standard}" + shift local triggers=("$@") - if ! check_abort_conditions; then - return + # Get OOM context for reboot message + local oom_victims="" + if [[ "$SYS_WATCHDOG_CHECK_OOM" == true ]]; then + oom_victims=$(get_oom_victims) + [[ -n "$oom_victims" ]] && triggers+=("oom_victims: $oom_victims") + fi + + # Abort check — CRITICAL bypasses this + if [[ "$tier" != "critical" ]]; then + if ! check_abort_conditions; then + return + fi + else + warn "CRITICAL tier — bypassing abort conditions" fi RECENT_REBOOTS=$(count_recent_reboots) - info "Recent reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr window: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT" + log "Recent reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr window: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT" if [[ "$RECENT_REBOOTS" -ge "$SYS_WATCHDOG_REBOOT_LIMIT" ]]; then - error "Reboot limit hit — shutting down instead" - notify "Reboot loop detected on $(hostname) — shutting down after $RECENT_REBOOTS reboots — conditions: ${triggers[*]}" "System Watchdog" "warning" + error "Reboot loop detected — shutting down instead of rebooting" + notify "Reboot loop on $(hostname) ($MY_ID) — shutting down after $RECENT_REBOOTS reboots — ${triggers[*]}" \ + "System Watchdog" "warning" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would shutdown now" return @@ -242,7 +411,18 @@ do_reboot() { return fi - notify "System watchdog reboot triggered on $(hostname) — conditions: ${triggers[*]}" "System Watchdog" "warning" + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " $ICON_REBOOT_SMART SYSTEM WATCHDOG — REBOOT TRIGGERED" + echo " Tier: ${tier^^}" + echo " Host: $MY_ID ($LOCAL_SERVER_NAME)" + for t in "${triggers[@]}"; do + echo " → $t" + done + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + notify "System watchdog ${tier^^} reboot on $(hostname) ($MY_ID) — ${triggers[*]}" \ + "System Watchdog" "warning" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — reboot sequence would begin now" @@ -251,7 +431,8 @@ do_reboot() { log_reboot - info "Shutting down VMs..." + # Graceful shutdown sequence + warn "Shutting down VMs..." if command -v virsh >/dev/null 2>&1; then for VM in $(virsh list --name 2>/dev/null); do [[ -z "$VM" ]] && continue @@ -260,88 +441,265 @@ do_reboot() { sleep 30 fi - info "Stopping Docker containers..." + warn "Stopping Docker containers..." if command -v docker >/dev/null 2>&1; then - docker ps -q | xargs -r docker stop >/dev/null 2>&1 + timeout 60 docker ps -q 2>/dev/null | xargs -r docker stop >/dev/null 2>&1 fi - info "Stopping User Scripts..." + warn "Stopping User Scripts..." pkill -f "/tmp/user.scripts" 2>/dev/null || true - info "Syncing disks..." + warn "Syncing disks..." sync - echo "" - echo "$ICON_REBOOT_SMART Rebooting system NOW..." sleep 5 /sbin/reboot } -# ----------------------------------------------------------------------------------------------- -# CLEAN SHUTDOWN — trap SIGTERM/SIGINT from array stop -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Clean Shutdown ━━━ +# ============================================================================================== WATCHDOG_RUNNING=true cleanup() { echo "" - info "System watchdog received shutdown signal — stopping cleanly" + warn "System watchdog received shutdown signal — stopping cleanly" WATCHDOG_RUNNING=false exit 0 } trap cleanup SIGTERM SIGINT -# ----------------------------------------------------------------------------------------------- -# ━━━ CONTINUOUS MONITORING LOOP ━━━ -# ----------------------------------------------------------------------------------------------- -info "System watchdog started — checking every ${SYSTEM_WATCHDOG_INTERVAL}s" +# ============================================================================================== +# ━━━ Continuous Monitoring Loop ━━━ +# ============================================================================================== +warn "System watchdog started — $MY_ID — checking every ${SYSTEM_WATCHDOG_INTERVAL}s" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" CYCLE=0 while [[ "$WATCHDOG_RUNNING" == true ]]; do - ((CYCLE++)) + (( CYCLE++ )) - # Re-source Master.conf each cycle — picks up config changes without restart - source "$SCRIPT_DIR/../Master.conf" + # Re-source config each cycle — picks up config changes without restart + source "$SCRIPT_DIR/../load_config.sh" + detect_hosts SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 )) TOTAL_CORES=$(nproc) - # Per-cycle triggers — cleared each iteration TRIGGERS=() + CRITICAL_TRIGGERS=() + URGENT_OOM_CONFIRMED=false - # ── rootfs usage ───────────────────────────────────────────────────────────────────────── + # ── OOM Delta — read every cycle for bypass decisions ───────────────────────────────────── + OOM_DELTA=0 + if [[ "$SYS_WATCHDOG_CHECK_OOM" == true ]]; then + OOM_DELTA=$(get_oom_delta) + [[ "$OOM_DELTA" -gt 0 ]] && \ + log "OOM kills this cycle: $OOM_DELTA (limit: ${SYS_WATCHDOG_OOM_LIMIT})" + fi + + # ========================================================================================== + # ━━━ TIER 1 — CRITICAL CHECKS (bypass all strikes, reboot immediately) ━━━ + # ========================================================================================== + + # ── Docker daemon — critical: nothing can heal without it ───────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then + if ! timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then + error "Docker daemon unresponsive — CRITICAL" + + # Attempt daemon restart before rebooting + warn "Attempting Docker daemon restart..." + if [[ "$DRY_RUN" == false ]]; then + /etc/rc.d/rc.docker restart >/dev/null 2>&1 + sleep 15 + if timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then + warn "Docker daemon restarted successfully — continuing monitoring" + else + error "Docker daemon restart failed — adding to CRITICAL triggers" + CRITICAL_TRIGGERS+=("docker_daemon_unresponsive") + fi + else + warn "DRY RUN — would attempt Docker daemon restart" + CRITICAL_TRIGGERS+=("docker_daemon_unresponsive") + fi + else + log "Docker daemon healthy ✅" + fi + fi + + # ── rootfs critical — at 99%+ writes are failing ───────────────────────────────────────── if [[ "$SYS_WATCHDOG_CHECK_ROOTFS" == true ]]; then - ROOTFS_USED=$(df / --output=pcent | tail -1 | tr -d ' %') + ROOTFS_USED=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %') + if [[ "$ROOTFS_USED" -ge "${SYS_WATCHDOG_ROOTFS_CRITICAL_PCT:-99}" ]]; then + error "rootfs ${ROOTFS_USED}% — CRITICAL (writes failing)" + CRITICAL_TRIGGERS+=("rootfs_full=${ROOTFS_USED}%") + fi + fi + + # ── Kernel oops/BUG — kernel running with corrupted state ──────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_KERNEL_OOPS" == true ]]; then + PREV_OOPS=$(get_state_val "kernel_oops_count") + CURRENT_OOPS=$(dmesg 2>/dev/null | grep -cE "BUG:|kernel BUG|Oops:" || echo 0) + CURRENT_OOPS="${CURRENT_OOPS//[^0-9]/}"; CURRENT_OOPS="${CURRENT_OOPS:-0}" + set_state_val "kernel_oops_count" "$CURRENT_OOPS" + + if [[ -n "$PREV_OOPS" && "$PREV_OOPS" =~ ^[0-9]+$ ]]; then + OOPS_DELTA=$(( CURRENT_OOPS - PREV_OOPS )) + if [[ "$OOPS_DELTA" -gt 0 ]]; then + error "Kernel oops/BUG detected — $OOPS_DELTA new since last cycle — CRITICAL" + CRITICAL_TRIGGERS+=("kernel_oops=${OOPS_DELTA}_new") + fi + fi + fi + + # ── File descriptor exhaustion — new connections failing silently ───────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_FD" == true ]]; then + FD_LINE=$(cat /proc/sys/fs/file-nr 2>/dev/null) + FD_OPEN=$(echo "$FD_LINE" | awk '{print $1}') + FD_MAX=$(echo "$FD_LINE" | awk '{print $3}') + if [[ -n "$FD_OPEN" && -n "$FD_MAX" && "$FD_MAX" -gt 0 ]]; then + FD_PCT=$(( FD_OPEN * 100 / FD_MAX )) + if [[ "$FD_PCT" -ge "${SYS_WATCHDOG_FD_CRITICAL_PCT:-95}" ]]; then + error "File descriptors ${FD_PCT}% exhausted (${FD_OPEN}/${FD_MAX}) — CRITICAL" + CRITICAL_TRIGGERS+=("fd_exhaustion=${FD_PCT}%") + else + log "File descriptors: ${FD_PCT}% (${FD_OPEN}/${FD_MAX})" + fi + fi + fi + + # ── /boot read-only — state and config writes failing silently ──────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_BOOT" == true ]]; then + BOOT_TEST="/boot/.watchdog_write_test" + if ! touch "$BOOT_TEST" 2>/dev/null; then + error "/boot is read-only — config writes failing silently — CRITICAL" + CRITICAL_TRIGGERS+=("boot_read_only") + else + rm -f "$BOOT_TEST" 2>/dev/null + log "/boot is writable ✅" + fi + fi + + # ── Act on CRITICAL triggers immediately ───────────────────────────────────────────────── + if [[ ${#CRITICAL_TRIGGERS[@]} -gt 0 ]]; then + echo "" + echo "━━━ $ICON_ERROR CRITICAL — IMMEDIATE REBOOT — Cycle $CYCLE ━━━" + for t in "${CRITICAL_TRIGGERS[@]}"; do + error " CRITICAL: $t" + done + do_reboot "critical" "${CRITICAL_TRIGGERS[@]}" + continue + fi + + # ========================================================================================== + # ━━━ TIER 3 — STANDARD CHECKS (strike system) ━━━ + # ========================================================================================== + + # ── rootfs standard ────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_ROOTFS" == true ]]; then + ROOTFS_USED=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %') TRIGGERED=false [[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]] && TRIGGERED=true run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && \ TRIGGERS+=("rootfs=${ROOTFS_USED}%") fi - # ── /var/log usage ──────────────────────────────────────────────────────────────────────── + # ── /var/log ───────────────────────────────────────────────────────────────────────────── if [[ "$SYS_WATCHDOG_CHECK_LOG" == true ]]; then - LOG_USED=$(df -P /var/log | awk 'NR==2 {print $5}' | tr -d '%') + LOG_USED=$(df -P /var/log 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%') TRIGGERED=false - [[ "$LOG_USED" -ge "$SYS_WATCHDOG_LOG_PCT" ]] && TRIGGERED=true + [[ "${LOG_USED:-0}" -ge "$SYS_WATCHDOG_LOG_PCT" ]] && TRIGGERED=true run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && \ TRIGGERS+=("log=${LOG_USED}%") fi - # ── Free RAM ───────────────────────────────────────────────────────────────────────────── + # ── /tmp ───────────────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_TMP" == true ]]; then + TMP_USED=$(df -P /tmp 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%') + if [[ "${TMP_USED:-0}" -ge "${SYS_WATCHDOG_TMP_CRITICAL_PCT:-98}" ]]; then + # Try to clear before escalating + warn "/tmp ${TMP_USED}% — attempting cleanup..." + find /tmp -type f -mmin +60 -not -name "*.lock" -delete 2>/dev/null + TMP_USED_AFTER=$(df -P /tmp 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%') + if [[ "${TMP_USED_AFTER:-0}" -ge "${SYS_WATCHDOG_TMP_CRITICAL_PCT:-98}" ]]; then + error "/tmp still ${TMP_USED_AFTER}% after cleanup — adding to triggers" + TRIGGERED=true + else + warn "/tmp cleared to ${TMP_USED_AFTER}% ✅" + TRIGGERED=false + fi + elif [[ "${TMP_USED:-0}" -ge "${SYS_WATCHDOG_TMP_PCT:-90}" ]]; then + TRIGGERED=true + else + TRIGGERED=false + fi + run_strike_check "tmp" "$TRIGGERED" "/tmp ${TMP_USED}%" && \ + TRIGGERS+=("tmp=${TMP_USED}%") + fi + + # ── RAM tiers ───────────────────────────────────────────────────────────────────────────── if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) - MEM_GB=$((MEM_KB / 1024 / 1024)) - TRIGGERED=false - [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]] && TRIGGERED=true - run_strike_check "ram" "$TRIGGERED" "RAM ${MEM_GB}GB free" && \ - TRIGGERS+=("low_ram=${MEM_GB}GB") + MEM_GB=$(( MEM_KB / 1024 / 1024 )) + MEM_SHUTDOWN_ACTIVE=$(get_state_val "mem_shutdown_active") + + if [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]]; then + # Tier 2 check — bypass if OOM confirms crisis + if [[ "$SYS_WATCHDOG_CHECK_OOM" == true ]] && \ + [[ "$OOM_DELTA" -ge "$SYS_WATCHDOG_OOM_LIMIT" ]]; then + error "RAM ${MEM_GB}GB + ${OOM_DELTA} OOM kills this cycle — URGENT bypass" + OOM_VICTIMS=$(get_oom_victims) + URGENT_TRIGGERS=("urgent_low_ram=${MEM_GB}GB" "oom_kills=${OOM_DELTA}") + [[ -n "$OOM_VICTIMS" ]] && URGENT_TRIGGERS+=("oom_victims: $OOM_VICTIMS") + do_reboot "urgent" "${URGENT_TRIGGERS[@]}" + continue + fi + # Standard strike path + run_strike_check "ram" true "RAM ${MEM_GB}GB free" && \ + TRIGGERS+=("low_ram=${MEM_GB}GB") + + elif [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_SHUTDOWN_GB" ]]; then + reset_strikes "ram" + # Container shutdown tier — but only once per event + if [[ "$MEM_SHUTDOWN_ACTIVE" != "true" ]]; then + warn "RAM ${MEM_GB}GB — below shutdown threshold ${SYS_WATCHDOG_MEM_SHUTDOWN_GB}GB" + run_strike_check "ram_shutdown" true "RAM shutdown tier ${MEM_GB}GB" && \ + shutdown_non_essential_containers + else + # Already shutdown — check if recovered + if [[ "$MEM_GB" -ge "$SYS_WATCHDOG_MEM_RECOVER_GB" ]]; then + warn "RAM recovered to ${MEM_GB}GB — clearing emergency state" + restart_non_essential_containers + reset_strikes "ram_shutdown" + else + warn "RAM ${MEM_GB}GB — still in emergency shutdown (recover threshold: ${SYS_WATCHDOG_MEM_RECOVER_GB}GB)" + fi + fi + + elif [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_WARN_GB" ]]; then + reset_strikes "ram" + reset_strikes "ram_shutdown" + warn "RAM ${MEM_GB}GB — below warning threshold ${SYS_WATCHDOG_MEM_WARN_GB}GB" + local prev_ram_warn + prev_ram_warn=$(get_strikes "ram_warn_notified") + if [[ "${prev_ram_warn:-0}" -eq 0 ]]; then + notify "RAM warning on $(hostname) ($MY_ID) — ${MEM_GB}GB free (threshold: ${SYS_WATCHDOG_MEM_WARN_GB}GB)" \ + "System Watchdog" "warning" + set_strikes "ram_warn_notified" 1 + fi + else + reset_strikes "ram" + reset_strikes "ram_shutdown" + set_strikes "ram_warn_notified" 0 + log "RAM ${MEM_GB}GB free ✅" + fi fi # ── ZFS ARC ────────────────────────────────────────────────────────────────────────────── if [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]] && [[ -f /proc/spl/kstat/zfs/arcstats ]]; then ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) - ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats) ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX )) TRIGGERED=false if [[ "$ARC_PCT" -ge "$SYS_WATCHDOG_ARC_PINNED_PCT" ]]; then @@ -350,15 +708,16 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do ARC_AFTER_PCT=$(( ARC_AFTER * 100 / ARC_MAX )) [[ "$ARC_AFTER_PCT" -ge "$SYS_WATCHDOG_ARC_RELEASE_PCT" ]] && TRIGGERED=true fi - run_strike_check "arc" "$TRIGGERED" "ZFS ARC pinned" && \ + run_strike_check "arc" "$TRIGGERED" "ZFS ARC pinned ${ARC_PCT}%" && \ TRIGGERS+=("arc_pinned=${ARC_PCT}%") fi - # ── CPU temperature ────────────────────────────────────────────────────────────────────── + # ── CPU temperature ─────────────────────────────────────────────────────────────────────── if [[ "$SYS_WATCHDOG_CHECK_CPU_TEMP" == true ]]; then CPU_TEMP="" if command -v sensors >/dev/null 2>&1; then - CPU_TEMP=$(sensors 2>/dev/null | grep -i "Package id 0\|Tctl\|CPU Temp" | \ + CPU_TEMP=$(sensors 2>/dev/null | \ + grep -i "Package id 0\|Tctl\|CPU Temp" | \ awk '{print $NF}' | tr -d '+°C' | head -1) fi if [[ -n "$CPU_TEMP" ]]; then @@ -383,61 +742,146 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do # ── Zombie processes ───────────────────────────────────────────────────────────────────── if [[ "$SYS_WATCHDOG_CHECK_ZOMBIES" == true ]]; then - ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null) - ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}" - ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}" - ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}" + ZOMBIE_COUNT=$(ps aux 2>/dev/null | awk '{print $8}' | grep -c "^Z$" || echo 0) + ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}"; ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}" TRIGGERED=false [[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]] && TRIGGERED=true run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && \ TRIGGERS+=("zombies=${ZOMBIE_COUNT}") fi - # ── Docker daemon ──────────────────────────────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then - TRIGGERED=false - ! timeout 10 docker ps >/dev/null 2>&1 && TRIGGERED=true - run_strike_check "docker_daemon" "$TRIGGERED" "Docker daemon unresponsive" && \ - TRIGGERS+=("docker_daemon") + # ── Array disk errors — accumulating mdstat errors ──────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_MDSTAT" == true ]]; then + PREV_MD_ERRORS=$(get_state_val "mdstat_errors") + CURRENT_MD_ERRORS=$(grep -oP "(?<=\[)[^\]]*[U_][^\]]*(?=\])" \ + /proc/mdstat 2>/dev/null | grep -o "_" | wc -l || echo 0) + CURRENT_MD_ERRORS="${CURRENT_MD_ERRORS//[^0-9]/}"; CURRENT_MD_ERRORS="${CURRENT_MD_ERRORS:-0}" + set_state_val "mdstat_errors" "$CURRENT_MD_ERRORS" + + if [[ -n "$PREV_MD_ERRORS" && "$PREV_MD_ERRORS" =~ ^[0-9]+$ ]]; then + MD_DELTA=$(( CURRENT_MD_ERRORS - PREV_MD_ERRORS )) + if [[ "$MD_DELTA" -ge "${SYS_WATCHDOG_MDSTAT_ERROR_LIMIT:-5}" ]]; then + TRIGGERED=true + run_strike_check "mdstat" "$TRIGGERED" \ + "mdstat errors +${MD_DELTA} (total: ${CURRENT_MD_ERRORS})" && \ + TRIGGERS+=("mdstat_errors=+${MD_DELTA}") + else + run_strike_check "mdstat" false "mdstat" > /dev/null + fi + fi fi - # ── Required containers from skip list ─────────────────────────────────────────────────── + # ── Network interface state ─────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_NETWORK" == true ]]; then + NIC="${SYS_WATCHDOG_NIC:-eth0}" + NIC_STATE=$(cat "/sys/class/net/${NIC}/operstate" 2>/dev/null || echo "unknown") + TRIGGERED=false + [[ "$NIC_STATE" != "up" ]] && TRIGGERED=true + run_strike_check "network" "$TRIGGERED" "${NIC} state: ${NIC_STATE}" && \ + TRIGGERS+=("nic_down=${NIC}") + fi + + # ── sshd — try restart before escalating ───────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_SSHD" == true ]]; then + if ! pgrep -x sshd >/dev/null 2>&1; then + warn "sshd not running — attempting restart..." + if [[ "$DRY_RUN" == false ]]; then + /etc/rc.d/rc.sshd start >/dev/null 2>&1 + sleep 3 + if pgrep -x sshd >/dev/null 2>&1; then + warn "sshd restarted successfully ✅" + reset_strikes "sshd" + notify "sshd was down on $(hostname) ($MY_ID) — restarted automatically" \ + "System Watchdog" "warning" + else + error "sshd restart failed — remote access unavailable" + run_strike_check "sshd" true "sshd not running" && \ + TRIGGERS+=("sshd_down") + fi + else + warn "DRY RUN — would restart sshd" + fi + else + reset_strikes "sshd" + fi + fi + + # ── Runaway process ─────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_RUNAWAY" == true ]]; then + RUNAWAY_PCT="${SYS_WATCHDOG_RUNAWAY_CPU_PCT:-90}" + TOP_CPU_PCT=$(ps aux 2>/dev/null | awk 'NR>1{print $3}' | sort -rn | head -1) + TOP_CPU_INT=$(printf "%.0f" "${TOP_CPU_PCT:-0}") + TOP_CPU_NAME=$(ps aux 2>/dev/null | sort -k3 -rn | awk 'NR==2{print $11}') + TRIGGERED=false + [[ "$TOP_CPU_INT" -ge "$RUNAWAY_PCT" ]] && TRIGGERED=true + # Runaway uses SYS_WATCHDOG_RUNAWAY_STRIKES not global strike limit + if [[ "$TRIGGERED" == true ]]; then + RAWAY_S=$(increment_strikes "runaway") + RLIMIT="${SYS_WATCHDOG_RUNAWAY_STRIKES:-3}" + warn "Runaway ${TOP_CPU_NAME} ${TOP_CPU_PCT}% CPU -- strike $RAWAY_S/$RLIMIT" + if (( RAWAY_S >= RLIMIT )); then + error "Runaway process ${TOP_CPU_NAME} -- strike limit hit" + reset_strikes "runaway" + TRIGGERS+=("runaway=${TOP_CPU_NAME}@${TOP_CPU_PCT}%") + fi + else + RAWAY_CUR=$(get_strikes "runaway") + [[ "${RAWAY_CUR:-0}" -gt 0 ]] && reset_strikes "runaway" + fi + fi + + # ── Required containers from docker_watchdog skip list ──────────────────────────────────── if [[ "$SYS_WATCHDOG_CHECK_CONTAINERS" == true ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then FAILED_CONTAINERS=() while IFS= read -r container; do [[ -z "$container" ]] && continue - STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") + STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ + '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") [[ "$STATUS" != "true" ]] && FAILED_CONTAINERS+=("$container") done < "$SYS_WATCHDOG_FAILED_FILE" - if [[ ${#FAILED_CONTAINERS[@]} -gt 0 ]]; then - run_strike_check "failed_containers" "true" "required containers stopped" && \ - TRIGGERS+=("containers=${FAILED_CONTAINERS[*]}") - fi + TRIGGERED=false + [[ ${#FAILED_CONTAINERS[@]} -gt 0 ]] && TRIGGERED=true + run_strike_check "failed_containers" "$TRIGGERED" \ + "required containers stopped: ${FAILED_CONTAINERS[*]:-}" && \ + TRIGGERS+=("containers=${FAILED_CONTAINERS[*]:-}") fi - # ── Evaluate triggers ──────────────────────────────────────────────────────────────────── + # ========================================================================================== + # ━━━ Evaluate Standard Triggers ━━━ + # ========================================================================================== if [[ ${#TRIGGERS[@]} -gt 0 ]]; then echo "" echo "━━━ $ICON_REBOOT_SMART System Watchdog — Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━" for t in "${TRIGGERS[@]}"; do echo " $ICON_REBOOT_SMART $t" done + [[ "$OOM_DELTA" -gt 0 ]] && echo " OOM kills this cycle: $OOM_DELTA" echo "" - do_reboot "${TRIGGERS[@]}" + do_reboot "standard" "${TRIGGERS[@]}" else log "Cycle $CYCLE — system healthy ($(date '+%H:%M:%S'))" + # Heartbeat — periodic proof of life if [[ "${SYSTEM_WATCHDOG_HEARTBEAT:-true}" == true ]]; then HB_SECONDS=$(( ${SYSTEM_WATCHDOG_HEARTBEAT_HOURS:-1} * 3600 )) - UPTIME_SECONDS=$(( CYCLE * SYSTEM_WATCHDOG_INTERVAL )) - if [[ "$HB_SECONDS" -gt 0 ]] && (( UPTIME_SECONDS % HB_SECONDS < SYSTEM_WATCHDOG_INTERVAL )) && [[ "$UPTIME_SECONDS" -gt 0 ]]; then - HB_UPTIME_HR=$(( UPTIME_SECONDS / 3600 )) - info "♥ system_watchdog alive — ~${HB_UPTIME_HR}hr uptime ($(date '+%H:%M:%S'))" + UPTIME_APPROX=$(( CYCLE * SYSTEM_WATCHDOG_INTERVAL )) + if [[ "$HB_SECONDS" -gt 0 ]] && \ + (( UPTIME_APPROX % HB_SECONDS < SYSTEM_WATCHDOG_INTERVAL )) && \ + [[ "$UPTIME_APPROX" -gt 0 ]]; then + HB_HR=$(( UPTIME_APPROX / 3600 )) + warn "♥ system_watchdog alive — $MY_ID — ~${HB_HR}hr uptime ($(date '+%H:%M:%S'))" fi fi fi + # ── State file heartbeat — keep mtime fresh every cycle ─────────────────────────────────── + # docker_watchdog.sh uses state file mtime to detect stale RAM emergency flags. + # If all checks pass with no set_state_val calls (e.g. KERNEL_OOPS + MDSTAT both disabled), + # mtime would not update and stale guard would incorrectly resume docker_watchdog.sh. + # Writing watchdog_cycle each tick guarantees mtime stays current while watchdog runs. + set_state_val "watchdog_cycle" "$CYCLE" + # Sleep until next cycle — interruptible by SIGTERM sleep "$SYSTEM_WATCHDOG_INTERVAL" & wait $! diff --git a/unRAID_Essentials/user_scripts_stop.sh b/unRAID_Essentials/user_scripts_stop.sh index 4de8ece..e6e20ff 100644 --- a/unRAID_Essentials/user_scripts_stop.sh +++ b/unRAID_Essentials/user_scripts_stop.sh @@ -1,126 +1,231 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- User Script Stop ------------------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ============================= User Scripts Stop ============================================== +# ============================================================================================== # Stops all running User Script processes spawned by the unRAID User Scripts plugin. # Identifies processes by their /tmp/user.scripts path signature. -# Supports --dry-run to preview what would be killed without making changes. -# ----------------------------------------------------------------------------------------------- +# Shows script names not just PIDs — you know what's being stopped. +# +# ── WHEN TO USE ─────────────────────────────────────────────────────────────────────────────── +# - Before a planned reboot when scripts are running mid-cycle +# - When a script is stuck and won't respond to the Abort button in the UI +# - Called automatically by server_reboot.sh as part of shutdown sequence +# - Emergency stop of all background ecosystem scripts +# +# ── HOW IT IDENTIFIES PROCESSES ─────────────────────────────────────────────────────────────── +# Scans /proc/*/cmdline for processes whose command line contains "/tmp/user.scripts". +# The unRAID User Scripts plugin stages all scripts in /tmp/user.scripts/ before execution. +# This is more reliable than process name matching which can vary. +# +# ── STOP SEQUENCE PER PROCESS ───────────────────────────────────────────────────────────────── +# 1. Send SIGTERM — allows script to trap and clean up gracefully +# 2. Wait 5 seconds +# 3. Check if still running → SIGKILL (force) if SIGTERM ignored +# 4. Verify dead after SIGKILL +# +# ── SELF-EXCLUSION ──────────────────────────────────────────────────────────────────────────── +# If this script itself is run via the User Scripts plugin it would find its own PID. +# Self-exclusion prevents this script from killing itself mid-execution. +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# Root check — kill requires root for other users' processes +# acquire_lock — prevents concurrent stop attempts +# Self-exclusion — never kills its own process tree +# SIGTERM → SIGKILL — graceful then forced +# Verify after kill — confirms processes are actually dead +# validate_unraid_cmd — notify validated before use +# Silent when clean — no processes running = log() only ✅ +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# user_scripts_stop.sh — stop all user scripts +# user_scripts_stop.sh --dry-run — show what would be stopped +# user_scripts_stop.sh --status — show currently running user scripts +# user_scripts_stop.sh --log — verbose output +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" +MY_PID=$$ +MY_PPID=$PPID -# ROOT CHECK +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root" + error "Must be run as root — kill requires root for other users' processes" exit 1 fi -success "Running as root" +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +acquire_lock + +detect_hosts + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no processes will be killed" + +# ============================================================================================== +# ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# Get script name from PID — extracts meaningful name from /tmp/user.scripts path +get_script_name() { + local pid="$1" + local cmdline + cmdline=$(tr '\0' ' ' < /proc/"$pid"/cmdline 2>/dev/null || echo "") + # Extract the script filename from the /tmp/user.scripts/... path + echo "$cmdline" | grep -o '/tmp/user\.scripts[^ ]*' | \ + awk -F/ '{print $NF}' | head -1 || echo "pid-$pid" +} + +# Get all user script PIDs — excludes self and own parent process tree +get_user_script_pids() { + local -a pids=() + while IFS= read -r pid; do + [[ -z "$pid" ]] && continue + # Self-exclusion — don't kill our own process or parent + [[ "$pid" == "$MY_PID" ]] && continue + [[ "$pid" == "$MY_PPID" ]] && continue + pids+=("$pid") + done < <( + for dir in /proc/[0-9]*/cmdline; do + pid="${dir%/cmdline}" + pid="${pid#/proc/}" + if grep -ql '/tmp/user\.scripts' "$dir" 2>/dev/null; then + echo "$pid" + fi + done + ) + printf '%s\n' "${pids[@]}" +} + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" - echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_PLUGIN Target: /tmp/user.scripts processes" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━ $ICON_SUMMARY USER SCRIPTS STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "" + + mapfile -t PIDS < <(get_user_script_pids) + + if [[ ${#PIDS[@]} -eq 0 ]]; then + log "No User Script processes running" + else + echo " ${#PIDS[@]} User Script process(es) running:" + for pid in "${PIDS[@]}"; do + name=$(get_script_name "$pid") + elapsed=$(ps -o etimes= -p "$pid" 2>/dev/null | tr -d ' ') + runtime=$(format_duration "${elapsed:-0}") + echo " $ICON_RUNNING PID $pid — $name (${runtime})" + done + fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -# ----------------------------------------------------------------------------------------------- -# FUNCTIONS -# ----------------------------------------------------------------------------------------------- - -# Returns PIDs of all processes running under /tmp/user.scripts -# These are processes spawned by the unRAID User Scripts plugin. -get_user_script_pids() { - /usr/bin/ps -eo pid,cmd | grep "/tmp/user.scripts" | grep -v grep | awk '{print $1}' -} - -# Kills all running User Script processes one by one. -# Reports each PID killed or skipped in dry run mode. -stop_user_scripts() { - local pids - pids=$(get_user_script_pids) - - if [[ -z "$pids" ]]; then - info "$ICON_PLUGIN No running User Script processes found — nothing to do" - return - fi - - local count=0 - - for pid in $pids; do - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would kill User Script PID $pid" - else - info "Killing User Script PID $pid..." - - if kill "$pid" 2>/dev/null; then - success "Killed PID $pid" - else - warn "Could not kill PID $pid — may have already exited" - fi - fi - - count=$((count + 1)) - done - - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would have targeted $count process(es)" - else - info "$count process(es) targeted" - fi -} - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_PLUGIN User Script Stop ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ User Scripts Stop ━━━ +# ============================================================================================== echo "" -echo "━━━ $ICON_PLUGIN User Script Stop ━━━" -echo "$ICON_PLUGIN Target: User Scripts Plugin processes" -echo "$ICON_GEAR Dry Run: $DRY_RUN" -echo "" - -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" +echo "━━━ $ICON_PLUGIN User Scripts Stop — $MY_ID ━━━" START=$(date +%s) -stop_user_scripts +mapfile -t PIDS < <(get_user_script_pids) + +KILLED=() +FAILED=() +SKIPPED=() + +if [[ ${#PIDS[@]} -eq 0 ]]; then + log "No User Script processes running — nothing to do" +else + warn "${#PIDS[@]} User Script process(es) found" + echo "" + + for pid in "${PIDS[@]}"; do + name=$(get_script_name "$pid") + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would stop: $name (PID $pid)" + SKIPPED+=("$name") + continue + fi + + # Verify still running before trying to kill + if ! kill -0 "$pid" 2>/dev/null; then + log "$name (PID $pid) — already exited" + continue + fi + + # SIGTERM — graceful stop + log "Sending SIGTERM to $name (PID $pid)..." + kill -TERM "$pid" 2>/dev/null || true + sleep 5 + + # Check if stopped after SIGTERM + if ! kill -0 "$pid" 2>/dev/null; then + warn "Stopped: $name (PID $pid) ✅" + KILLED+=("$name") + continue + fi + + # SIGKILL — forced stop + warn "$name still running after SIGTERM — sending SIGKILL" + kill -KILL "$pid" 2>/dev/null || true + sleep 2 + + # Final verify + if ! kill -0 "$pid" 2>/dev/null; then + warn "Force-stopped: $name (PID $pid) ✅" + KILLED+=("$name") + else + error "Failed to kill: $name (PID $pid)" + FAILED+=("$name") + fi + done +fi END=$(date +%s) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY USER SCRIPTS STOP SUMMARY ━━━━━" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" +echo "" -if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no processes killed" +if [[ ${#PIDS[@]} -eq 0 ]]; then + log "No processes were running" +elif [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would have stopped ${#SKIPPED[@]} process(es): ${SKIPPED[*]}" else - REMAINING=$(get_user_script_pids) - if [[ -z "$REMAINING" ]]; then - echo "$ICON_DONE Status: $ICON_SUCCESS ALL PROCESSES STOPPED" - notify "User Scripts stopped on $(hostname)" "User Script Stop" "warning" - else - echo "$ICON_WARN Status: $ICON_WARN SOME PROCESSES MAY STILL BE RUNNING" - notify "User Script stop completed but some processes may still be running on $(hostname)" "User Script Stop" "warning" - fi + [[ ${#KILLED[@]} -gt 0 ]] && warn "Stopped (${#KILLED[@]}): ${KILLED[*]}" + [[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed (${#FAILED[@]}): ${FAILED[*]}" fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file +echo "" +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — no changes made" +elif [[ ${#FAILED[@]} -gt 0 ]]; then + echo "$ICON_ERROR Status: SOME PROCESSES COULD NOT BE KILLED" + notify "User Scripts stop failed on $(hostname) ($MY_ID) — unkillable: ${FAILED[*]}" \ + "User Scripts Stop" "warning" +else + log "$ICON_DONE Status: done ✅" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ ${#FAILED[@]} -gt 0 ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/unRAID_Essentials/webgui_restart.sh b/unRAID_Essentials/webgui_restart.sh index 617a823..fe74d0c 100644 --- a/unRAID_Essentials/webgui_restart.sh +++ b/unRAID_Essentials/webgui_restart.sh @@ -1,198 +1,267 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- WebGUI Watchdog -------------------------------------------- -# ----------------------------------------------------------------------------------------------- -# Monitors unRAID's WebGUI and restarts it if unresponsive. -# Uses an escalating restart strategy — tries nginx first, then emhttp if needed. -# emhttp is the unRAID management daemon — restarting it is more disruptive than nginx -# but recovers cleanly. Notification sent on any restart so you know what happened. +# ============================================================================================== +# ================================= WebGUI Watchdog ============================================ +# ============================================================================================== +# Monitors the unRAID WebGUI and restarts services if unresponsive. +# Uses a three-step escalating strategy — lightest fix first, heaviest last. +# Run every 5-10 minutes via User Scripts plugin. +# Silent when healthy — only produces output when something needs fixing. # -# Escalation path: -# Check WebGUI → unresponsive → restart nginx → recheck -# Still unresponsive → restart emhttp → recheck -# Still unresponsive → notify warning, manual intervention needed +# ── ESCALATION PATH ─────────────────────────────────────────────────────────────────────────── +# Check WebGUI → responding → log() + exit 0 (completely silent ✅) # -# Run every 5-10 minutes via cron/User Scripts plugin. -# All configuration in Master.conf under WebGUI Watchdog section. -# Supports --dry-run to show what would be restarted without acting. -# ----------------------------------------------------------------------------------------------- +# Not responding: +# Step 1 — Restart nginx +# Lightest fix — handles most transient WebGUI failures +# nginx crash, worker stuck, connection timeout +# Wait WEBGUI_NGINX_WAIT seconds → recheck +# +# Step 2 — Restart php-fpm +# WebGUI runs through PHP-FPM — worker exhaustion causes silent failure +# php-fpm workers saturated → new requests queue → WebGUI appears frozen +# system_tuning_monitor.sh tracks usage — this recovers it +# Wait WEBGUI_PHP_WAIT seconds → recheck +# +# Step 3 — Restart emhttp +# Heaviest fix — emhttp is the unRAID management daemon +# Array, Docker, shares stay running — only WebGUI management restarts +# Takes longer to recover — WEBGUI_EMHTTP_WAIT gives it time +# Wait WEBGUI_EMHTTP_WAIT seconds → recheck +# +# All three failed → notify warning, manual intervention needed → exit 1 +# +# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── +# detect_hosts() sets MY_ID — used in all notifications and summary. +# Critical on two-server setup — which server's WebGUI failed? +# +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# acquire_lock — prevents concurrent runs double-restarting services +# detect_hosts() — MY_ID in all notifications +# Process verify — pgrep check after each service restart +# Silent healthy — completely silent on healthy cycle ✅ +# validate_unraid_cmd — notify validated before use +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# WEBGUI_URL — URL to check (default http://localhost) +# WEBGUI_TIMEOUT — curl timeout in seconds (default 5) +# WEBGUI_NGINX_WAIT — seconds after nginx restart before rechecking (default 15) +# WEBGUI_PHP_WAIT — seconds after php-fpm restart before rechecking (default 10) +# WEBGUI_EMHTTP_WAIT — seconds after emhttp restart before rechecking (default 30) +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# webgui_restart.sh — check and recover if needed +# webgui_restart.sh --dry-run — show what would be restarted +# webgui_restart.sh --status — show current WebGUI and service states +# webgui_restart.sh --log — verbose output +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" - +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi -success "Running as root" +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +acquire_lock + +detect_hosts + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no services will be restarted" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" - echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_WEBGUI URL: $WEBGUI_URL" - echo "$ICON_WEBGUI Curl timeout: ${WEBGUI_TIMEOUT}s" - echo "$ICON_WEBGUI Nginx wait: ${WEBGUI_NGINX_WAIT}s" - echo "$ICON_WEBGUI emhttp wait: ${WEBGUI_EMHTTP_WAIT}s" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_WEBGUI URL: $WEBGUI_URL" + echo "$ICON_WEBGUI Timeouts: curl=${WEBGUI_TIMEOUT}s nginx=${WEBGUI_NGINX_WAIT}s php=${WEBGUI_PHP_WAIT:-10}s emhttp=${WEBGUI_EMHTTP_WAIT}s" + echo "" - # Show current state if curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1; then - echo "$ICON_WEBGUI WebGUI: $ICON_RUNNING responding" + echo " $ICON_SUCCESS WebGUI: responding ✅" else - echo "$ICON_WEBGUI WebGUI: $ICON_NOT_RUNNING not responding" + echo " $ICON_ERROR WebGUI: NOT responding" fi + + pgrep -x nginx >/dev/null 2>&1 && \ + echo " $ICON_SUCCESS nginx: running ✅" || \ + echo " $ICON_ERROR nginx: NOT running" + + pgrep -f "php-fpm" >/dev/null 2>&1 && \ + FPM_COUNT=$(pgrep -fc "php-fpm" 2>/dev/null || echo "?") && \ + echo " $ICON_SUCCESS php-fpm: running ($FPM_COUNT workers) ✅" || \ + echo " $ICON_ERROR php-fpm: NOT running" + + pgrep -x emhttp >/dev/null 2>&1 && \ + echo " $ICON_SUCCESS emhttp: running ✅" || \ + echo " $ICON_ERROR emhttp: NOT running" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no services will be restarted" - -# ----------------------------------------------------------------------------------------------- -# FUNCTIONS -# ----------------------------------------------------------------------------------------------- - -# Check if WebGUI is responding +# ============================================================================================== +# ── CHECK AND ESCALATE ──────────────────────────────────────────────────────────────────────── +# ============================================================================================== check_webgui() { curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1 } -# Restart nginx — lightweight fix, try first -restart_nginx() { - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would restart nginx" - return 0 - fi - - info "$ICON_WEBGUI Restarting nginx..." - if /etc/rc.d/rc.nginx restart >/dev/null 2>&1; then - success "nginx restarted" - return 0 - else - error "nginx restart failed" - return 1 - fi -} - -# Restart emhttp — heavier fix, escalate if nginx didn't help -# emhttp drives the array, Docker management, shares — recovers cleanly but takes longer -restart_emhttp() { - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would restart emhttp" - return 0 - fi - - info "$ICON_WEBGUI Restarting emhttp..." - if /etc/rc.d/rc.emhttp restart >/dev/null 2>&1; then - success "emhttp restarted" - return 0 - else - error "emhttp restart failed" - return 1 - fi -} - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_WEBGUI WebGUI Watchdog ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_WEBGUI WebGUI Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -echo "$ICON_WEBGUI URL: $WEBGUI_URL" -echo "" - START=$(date +%s) +RECOVERY_ACTION="" +RECOVERY_OK=false -# Initial check -info "Checking WebGUI..." +log "WebGUI check — $WEBGUI_URL" +# ── Healthy — completely silent ─────────────────────────────────────────────────────────────── if check_webgui; then - success "$ICON_WEBGUI WebGUI is responding — nothing to do" - echo "" - echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━" - echo "$ICON_WEBGUI Status: $ICON_RUNNING HEALTHY" - echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + log "WebGUI responding — healthy ✅" exit 0 fi -# WebGUI not responding — begin escalation -warn "$ICON_WEBGUI WebGUI is not responding at $WEBGUI_URL" - -# ── Step 1: Restart nginx ── +# ── Not responding — begin escalation ──────────────────────────────────────────────────────── echo "" -echo "━━━ $ICON_WEBGUI Step 1 — Nginx Restart ━━━" +echo "━━━ $ICON_WEBGUI WebGUI Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)" +echo "" +warn "WebGUI not responding at $WEBGUI_URL — beginning escalation" -restart_nginx +# ── Step 1 — nginx restart ──────────────────────────────────────────────────────────────────── +echo "" +echo "━━━ Step 1 — nginx Restart ━━━" -if [[ "$DRY_RUN" == false ]]; then - info "Waiting ${WEBGUI_NGINX_WAIT}s for nginx to recover..." +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would restart nginx" +else + warn "Restarting nginx..." + if /etc/rc.d/rc.nginx restart >/dev/null 2>&1; then + # Verify nginx actually running + sleep 2 + if pgrep -x nginx >/dev/null 2>&1; then + warn "nginx restarted ✅" + else + error "nginx not running after restart command" + fi + else + error "nginx restart command failed" + fi + + log "Waiting ${WEBGUI_NGINX_WAIT}s for nginx to recover..." sleep "$WEBGUI_NGINX_WAIT" if check_webgui; then - success "$ICON_WEBGUI WebGUI recovered after nginx restart" - notify "WebGUI recovered on $(hostname) after nginx restart" "WebGUI Watchdog" "warning" - - echo "" - echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━" - echo "$ICON_WEBGUI Status: $ICON_SUCCESS RECOVERED via nginx restart" - echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - exit 0 + RECOVERY_ACTION="nginx restart" + RECOVERY_OK=true fi +fi + +# ── Step 2 — php-fpm restart ────────────────────────────────────────────────────────────────── +if [[ "$RECOVERY_OK" == false ]]; then + echo "" + echo "━━━ Step 2 — php-fpm Restart ━━━" + warn "WebGUI still not responding — restarting php-fpm" + warn "WebGUI may be frozen due to worker exhaustion (check system_tuning_monitor.sh)" + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would restart php-fpm" + else + if /etc/rc.d/rc.php-fpm restart >/dev/null 2>&1; then + sleep 2 + if pgrep -f "php-fpm" >/dev/null 2>&1; then + warn "php-fpm restarted ✅" + else + error "php-fpm not running after restart command" + fi + else + error "php-fpm restart command failed" + fi - warn "WebGUI still not responding after nginx restart — escalating to emhttp" + log "Waiting ${WEBGUI_PHP_WAIT:-10}s for php-fpm to recover..." + sleep "${WEBGUI_PHP_WAIT:-10}" + + if check_webgui; then + RECOVERY_ACTION="php-fpm restart" + RECOVERY_OK=true + fi + fi fi - -# ── Step 2: Restart emhttp ── -echo "" -echo "━━━ $ICON_WEBGUI Step 2 — emhttp Restart ━━━" -warn "Restarting emhttp — this is the unRAID management daemon" -warn "Array, Docker management and shares remain running but WebGUI will be briefly unavailable" -restart_emhttp +# ── Step 3 — emhttp restart ─────────────────────────────────────────────────────────────────── +if [[ "$RECOVERY_OK" == false ]]; then + echo "" + echo "━━━ Step 3 — emhttp Restart ━━━" + warn "WebGUI still not responding — restarting emhttp (unRAID management daemon)" + warn "Array, Docker, and shares remain running — WebGUI management will briefly restart" -if [[ "$DRY_RUN" == false ]]; then - info "Waiting ${WEBGUI_EMHTTP_WAIT}s for emhttp to recover..." - sleep "$WEBGUI_EMHTTP_WAIT" + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would restart emhttp" + else + if /etc/rc.d/rc.emhttp restart >/dev/null 2>&1; then + sleep 2 + if pgrep -x emhttp >/dev/null 2>&1; then + warn "emhttp restarted ✅" + else + error "emhttp not running after restart command" + fi + else + error "emhttp restart command failed" + fi - if check_webgui; then - success "$ICON_WEBGUI WebGUI recovered after emhttp restart" - notify "WebGUI recovered on $(hostname) after emhttp restart — check system health" "WebGUI Watchdog" "warning" + log "Waiting ${WEBGUI_EMHTTP_WAIT}s for emhttp to recover..." + sleep "$WEBGUI_EMHTTP_WAIT" - echo "" - echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━" - echo "$ICON_WEBGUI Status: $ICON_SUCCESS RECOVERED via emhttp restart" - echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - exit 0 + if check_webgui; then + RECOVERY_ACTION="emhttp restart" + RECOVERY_OK=true + fi fi fi -# ── Both restarts failed ── END=$(date +%s) +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━" -echo "$ICON_WEBGUI Status: $ICON_ERROR UNRECOVERED — manual intervention needed" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_WEBGUI URL: $WEBGUI_URL" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" +echo "" -if [[ "$DRY_RUN" == false ]]; then - notify "WebGUI unrecovered on $(hostname) after nginx and emhttp restart — manual intervention needed" "WebGUI Watchdog" "warning" - exit 1 +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — no services restarted" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 fi +if [[ "$RECOVERY_OK" == true ]]; then + warn "$ICON_SUCCESS WebGUI recovered via: $RECOVERY_ACTION" + notify "WebGUI recovered on $(hostname) ($MY_ID) via $RECOVERY_ACTION — monitor for recurrence" \ + "WebGUI Watchdog" "warning" +else + echo "$ICON_ERROR Status: UNRECOVERED — all three restart steps failed" + echo "$ICON_ERROR Manual intervention needed:" + echo " 1. Check: pgrep -x nginx emhttp" + echo " 2. Check: journalctl -u nginx --since '10 minutes ago'" + echo " 3. Try: server_reboot.sh if nothing else works" + notify "WebGUI UNRECOVERED on $(hostname) ($MY_ID) — nginx + php-fpm + emhttp restart all failed — manual intervention needed" \ + "WebGUI Watchdog" "warning" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ "$RECOVERY_OK" == false && "$DRY_RUN" == false ]] && exit 1 exit 0 \ No newline at end of file diff --git a/user_script_plug-in.sh b/user_script_plug-in.sh index a41407e..11abe22 100644 --- a/user_script_plug-in.sh +++ b/user_script_plug-in.sh @@ -1,18 +1,55 @@ #!/bin/bash # ============================================================================================== -# ================================= User Script Template ======================================= +# ============================= USER SCRIPTS MASTER TEMPLATE =================================== # ============================================================================================== # -# This file is the master template for all scripts run via the unRAID User Scripts plugin. -# Copy and paste the contents of this file into a new User Script entry in the plugin, -# then uncomment the script you want to run and set your schedule. +# Paste this file into a User Script entry. Uncomment ONE script block and set the schedule. +# Every script in the ecosystem is listed here — from the orchestrators that run it all, +# down to the individual scripts you can run standalone for specific tasks. # -# All scripts live at: /mnt/user/appdata/unraid_scripts/ -# All configuration at: /mnt/user/appdata/unraid_scripts/Master.conf -# Shared functions at: /mnt/user/appdata/unraid_scripts/common.sh +# ── HOW THIS ECOSYSTEM WORKS ────────────────────────────────────────────────────────────────── +# When used as intended, only a handful of orchestrators need to be scheduled. +# The orchestrators handle everything else in the correct order — they call child scripts, +# manage timing and dependencies, track pass/fail, and send one notification per window. # -# ============================================================================================== -# Changelog: +# You do not need to schedule every script below. The orchestrators cover it all: +# +# array_start.sh at array start — launches ALL startup scripts in order +# transcode_management.sh every 3 min — cleanup then manager (order critical) +# critical_sync_maintenance.sh every 15 min — auth + Emby dirty sync + partnership +# arrs_failed_stalled_recovery.sh every 6 hours — blocklist + re-search failed imports +# daily_sync_maintenance.sh 1am daily — git + rsync + media + restart +# rsync.sh --profile=emby-failover every 30 min — Emby watch state dirty sync +# weekly_sync_maintenance.sh 2:30am Sunday — clean sync + image updates +# sunday_morning_coffee_report.sh 7am Sunday — full weekly digest +# weekly_health_digest.sh 8am daily — profile-controlled health notification +# webgui_restart.sh every 10 min — WebGUI availability watchdog +# system_tuning_monitor.sh every 6 hours — inotify + php-fpm trend tracking +# clear_logs.sh 5am Sunday — size-threshold log cleanup +# [Sunday monitor block] 6am-11am Sun — certs, SMART, backups, reports +# +# ── INDIVIDUAL SCRIPTS ──────────────────────────────────────────────────────────────────────── +# Every child script is also listed below, individually. +# If you only want part of the ecosystem — say, just the arr cleanup scripts, or just +# the transcode manager — uncomment only those blocks and schedule them directly. +# All scripts are fully self-contained and work standalone. +# +# ── ALL SCRIPTS SUPPORT ─────────────────────────────────────────────────────────────────────── +# --dry-run run without making any changes — shows exactly what would happen +# --status show current configuration and state, then exit +# --log verbose output — every decision logged +# +# ── PATHS ───────────────────────────────────────────────────────────────────────────────────── +# Scripts: /mnt/user/appdata/unraid_scripts/ +# Configuration: /mnt/user/appdata/unraid_scripts/master.conf +# Per-host: /mnt/user/appdata/unraid_scripts/master_host1.conf (or master_host2.conf) +# Library: /mnt/user/appdata/unraid_scripts/common.sh +# +# ── PLUGIN SETTINGS (apply to every entry) ──────────────────────────────────────────────────── +# Run in background: YES — required for all scripts, even one-shots +# Run as root: YES — all scripts require root +# +# ── CHANGELOG ───────────────────────────────────────────────────────────────────────────────── # v1.0 — Initial template # v1.1 — MAX_RSYNC_PROCS removed, --status flag, full directory tree, changelog # v1.2 — Docker Essentials, Media, Transcodes, System Watchdog added @@ -21,382 +58,1002 @@ # v1.5 — media_management.sh orchestrator, corrected filenames, git repo section # v1.6 — lidarr_cleanup.sh, sonarr_cleanup.sh, radarr_cleanup.sh added # v1.7 — cert_monitor.sh added -# v1.8 — Monitors/ folder added with all monitoring scripts +# v1.8 — Monitors/ folder with all monitoring scripts # v1.9 — transcode_management.sh added to Orchestrators/ # v2.0 — Major restructure: array_start.sh single entry point, daily/weekly orchestrators, -# sunday_morning_coffee_report.sh, continuous_scripts_status.sh, -# downloaders_reset.sh, docker_network_connect ensure+connect combined, +# sunday_morning_coffee_report.sh, docker_network_connect combined, # inotify_tuning.sh, arr path translation, version checking, nuclear mode flags -# v2.1 — critical_sync_maintenance.sh added, Partnership/ folder added, -# DAILY_MAINTENANCE_SCRIPTS consolidates all daily jobs, -# WEEKLY_SYNC_SHARES renamed from WEEKLY_SYNC_JOBS, -# CRITICAL_MAINTENANCE_SCRIPTS + CRITICAL_SYNC_SHARES added, -# downloaders_reset moved to CRITICAL_MAINTENANCE_SCRIPTS, -# system_tuning_monitor.sh added to Monitors/, -# rsync_stop.sh auto-detects orchestrators via /proc children, -# check_remote_disks() auto-detects XFS/ZFS from disks.ini, -# temp check reads thresholds from dynamix.cfg, -# WEEKLY_SYNC_UPDATES renamed from CRITICAL_SYNC_UPDATES +# v2.1 — critical_sync_maintenance.sh, Partnership/ folder, system_tuning_monitor.sh, +# arrs_failed_stalled_recovery.sh, rsync_stop.sh orchestrator-aware, +# WEEKLY_SYNC_UPDATES, temp thresholds from dynamix.cfg +# v2.2 — Orchestrators at top with full inline detail. All child scripts listed +# individually below for standalone use. Inline descriptions on every entry. +# Correct schedules verified against script headers. sunday_morning_coffee_report +# and rsync_emby_failover added as separate scheduled entries. # ============================================================================================== -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ━━━ 📂 Repository Structure ━━━ -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +# ============================================================================================== +# ██ ORCHESTRATORS — RUN THE WHOLE ECOSYSTEM ██████████████████████████████████████████████████ +# ============================================================================================== # -# /mnt/user/appdata/unraid_scripts/ -# ├── Master.conf # All user configuration — edit this file only -# ├── common.sh # Shared library — functions used by all scripts -# ├── README.md # Project overview and quick start -# ├── user_script_plug-in.sh # This file — copy into User Scripts plugin -# ├── git_pull_execute.sh # Pulls latest scripts from Gitea repo -# │ -# ├── Partnership/ -# │ ├── partnership_manage.sh # Lifecycle manager — onboard/offboard/transfer/check -# │ └── README-Partnership.md -# │ -# ├── Failover/ -# │ ├── failover.sh # Mutual container failover — runs continuously -# │ ├── failover_test.sh # Controlled failover simulation — run manually -# │ └── README-Failover.md -# │ -# ├── Monitors/ -# │ ├── continuous_scripts_status.sh # Live status of all continuous loop scripts — run manually -# │ ├── backup_verify.sh # Random sample checksum verification vs remote -# │ ├── bandwidth_monitor.sh # Daily transfer logging + weekly summary -# │ ├── cert_monitor.sh # SSL certificate expiry — direct openssl check -# │ ├── emby_session_report.sh # Weekly Emby usage statistics via API -# │ ├── smart_health.sh # Drive SMART attribute monitoring -# │ ├── system_tuning_monitor.sh # inotify + php-fpm usage tracking — every 6hr -# │ ├── weekly_health_digest.sh # Aggregated system health — always/smart/weekly -# │ ├── zfs_memory_snapshot.sh # Weekly ZFS health and memory diagnostic report -# │ └── README-Monitors.md -# │ -# ├── Orchestrators/ -# │ ├── array_start.sh # Single entry point — launches all array-start scripts -# │ ├── daily_sync_maintenance.sh # Daily — git pull, media sync, maintenance jobs, docker restart -# │ ├── weekly_sync_maintenance.sh # Weekly — Emby + auth sync, updates, weekly restart -# │ ├── critical_sync_maintenance.sh # Every 15min — downloaders reset, auth sync, partnership check -# │ ├── transcode_management.sh # Every 3min — cleanup then manager in correct order -# │ ├── sunday_morning_coffee_report.sh # Sunday 7am — full weekly system health report -# │ └── README-Orchestrators.md -# │ -# ├── Rsync/ -# │ ├── rsync.sh # Core rsync script — called per share or profile -# │ ├── rsync_stop.sh # Stops rsync + orchestrators — auto-detects via /proc -# │ └── README-Rsync_Setup.md -# │ -# ├── Docker_Essentials/ -# │ ├── docker_watchdog.sh # Two-tier container health monitor — self-healing -# │ ├── docker_daily_restart.sh # Restarts configured containers daily -# │ ├── docker_weekly_restart.sh # Restarts configured containers weekly -# │ ├── docker_network_connect.sh # Ensures custom networks exist + connects containers -# │ ├── downloaders_reset.sh # Every 15min reset for slskd, SABnzbd, qBittorrent -# │ └── README-Docker_Essentials.md -# │ -# ├── Media/ -# │ ├── media_shares_permissions.sh # Applies permissions to all media shares -# │ ├── media_cleaner.sh # Removes junk files — profiles: anime, media -# │ ├── lidarr_cleanup.sh # Removes orphaned music files via Lidarr API -# │ ├── sonarr_cleanup.sh # Removes orphaned TV files via Sonarr API -# │ ├── radarr_cleanup.sh # Removes orphaned movie files via Radarr API -# │ ├── arrs_failed_stalled_recovery.sh # Auto blocklist + re-search failed/stalled downloads -# │ └── README-Media.md -# │ -# ├── Transcodes/ -# │ ├── ramdisk_setup.sh # Creates ramdisk + symlink — runs at array start -# │ ├── transcode_manager.sh # Monitors ramdisk usage, manages symlink direction -# │ ├── transcode_cleanup.sh # Removes old inactive transcode files -# │ └── README-Transcoding.md -# │ -# ├── Tools/ -# │ ├── bulk_permissions_repair.sh # Targeted permission repair for one share -# │ ├── container_data_export.sh # Export container appdata to tar archive -# │ ├── emby_database_repair.sh # SQLite integrity check on Emby databases -# │ ├── failover_state_reset.sh # Reset failover state file to NORMAL -# │ ├── recreate_shares.sh # Recreates share dirs from .cfg files after incident -# │ ├── watchdog_skip_list_manager.sh # Manage container watchdog skip lists -# │ ├── zfs_pool_scrub.sh # Trigger ZFS scrub with completion report -# │ └── README-Tools.md -# │ -# └── unRAID_Essentials/ -# ├── clear_logs.sh # Clears unRAID system and Docker log files -# ├── docker_syslog_filter.sh # Filters Docker veth noise from syslog -# ├── inotify_tuning.sh # Bumps inotify limits — prevents file event exhaustion -# ├── mover_stop.sh # Safely stops the unRAID mover -# ├── php_fpm_max_children.sh # Sets PHP-FPM max children value -# ├── server_reboot.sh # Graceful server reboot with user warning -# ├── system_watchdog.sh # System health monitor — last line of defense -# ├── user_scripts_stop.sh # Stops running User Scripts plugin jobs -# ├── webgui_restart.sh # WebGUI watchdog — nginx + emhttp restart -# └── README-Unraid_Essentials.md +# These are the only entries you need to schedule when using the ecosystem as intended. +# Each orchestrator calls child scripts in the correct order with correct timing. +# Configure what runs via master.conf — never by editing the orchestrator scripts. # -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ━━━ ⚠️ Critical Setup Notes ━━━ -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# ============================================================================================== + + +# ── ARRAY START ─────────────────────────────────────────────────────────────────────────────── +# Schedule: At Startup of Array +# Background: YES — required (starts continuous background processes) # -# EMBY TRANSCODE DOCKER MOUNT — MUST USE EXTRA PARAMETERS: -# Do NOT add /mnt/ram-transcode as a standard path mapping in the Emby template. -# Standard bind mounts use rprivate propagation — Docker locks the mount inode on -# the first symlink flip and new sessions land on SSD permanently for that run. +# The ONLY "At Startup of Array" entry needed. Reads ARRAY_START_SCRIPTS from master.conf +# and launches each in order. Detects one-shot vs continuous scripts automatically. +# Auto-fixes execute permissions on any script missing them. # -# In Emby Extra Parameters add: +# Launches in order: +# inotify_tuning.sh raise kernel inotify limits BEFORE containers inherit them +# docker_syslog_filter.sh suppress Docker veth noise BEFORE containers create interfaces +# php_fpm_max_children.sh set PHP-FPM workers to 250 BEFORE WebGUI serves requests +# ramdisk_setup.sh create tmpfs ramdisk + transcode symlink BEFORE Emby starts +# docker_network_connect.sh ensure custom networks exist and containers are connected +# system_watchdog.sh [continuous] three-tier server stability watchdog +# docker_watchdog.sh [continuous] two-tier container self-healing watchdog +# failover.sh [continuous] mutual failover state machine +# +# bash /mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh + + +# ── TRANSCODE MANAGEMENT ────────────────────────────────────────────────────────────────────── +# Schedule: */3 * * * * (every 3 minutes) +# Background: YES +# +# Cleanup first, then manager — order is non-negotiable. Stale segment files are removed +# BEFORE the manager checks ramdisk usage, so threshold decisions are based on real active +# session load, not inflated counts from sessions that already ended. +# +# transcode_cleanup.sh: +# Remove HLS segment files older than TRANSCODE_MAX_AGE (20min) not open by any process. +# One lsof call per location (not one per file — critical on busy Live TV systems). +# transcoding-temp directory NEVER deleted even when empty (prevents SSD session drift). +# After cleanup: triggers flip-back to ramdisk if usage dropped below RAMDISK_LOW_GB. +# +# transcode_manager.sh: +# Check ramdisk usage vs RAMDISK_WARN_GB. Flip /mnt/ram-transcode symlink if needed. +# Hysteresis: WARN→SSD flip, usage must drop to LOW before flipping back (prevents oscillation). +# Safety every cycle: missing symlink → recreate, ramdisk gone → auto-flip SSD, +# missing transcoding-temp → recreate immediately. +# Show active Emby sessions with play method. Write daily stats to TRANSCODE_DAILY_LOG. +# +# NOTE: Emby Extra Parameters MUST include: # --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared +# Without shared propagation, symlink flips are silently ignored after the first flip. # -# shared propagation means host symlink changes are visible inside the container -# in real time — symlink flips work correctly for the lifetime of the container. -# -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ━━━ 🚀 Script Commands — Uncomment the one you want to run ━━━ -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# bash /mnt/user/appdata/unraid_scripts/Orchestrators/transcode_management.sh -# ━━━ Orchestrators ━━━ -# Orchestrators are the main runtime engines — they handle everything within their domain -# in the correct order automatically. Individual scripts below can still be run manually. -# -# array_start.sh — At Startup of Array — single entry, launches everything -# Configurable via ARRAY_START_SCRIPTS in Master.conf -# -# daily_sync_maintenance.sh — 0 1 * * * (1am daily) -# media share sync → DAILY_MAINTENANCE_SCRIPTS jobs -# git pull, permissions, cleaners, arr cleanup, docker restart -# -# weekly_sync_maintenance.sh — 30 2 * * 0 (Sunday 2:30am) -# stop containers → updates → WEEKLY_SYNC_SHARES → -# restart → WEEKLY_MAINTENANCE_SCRIPTS -# -# critical_sync_maintenance.sh — */15 * * * * (every 15 minutes) -# CRITICAL_MAINTENANCE_SCRIPTS → CRITICAL_SYNC_SHARES → -# partnership --check -# downloaders reset, auth stack sync, Emby dirty sync -# -# transcode_management.sh — */3 * * * * (every 3 minutes) -# transcode cleanup → transcode manager (correct order) -# -# sunday_morning_coffee_report.sh — 0 7 * * 0 (Sunday 7am) -# full weekly system health report — ready when you wake up ☕ -# -# arrs_failed_stalled_recovery.sh — 0 */6 * * * (every 6 hours) -# blocklist + re-search failed imports and stalled downloads -# -# system_tuning_monitor.sh — 0 */6 * * * (every 6 hours) -# snapshot inotify + php-fpm — feeds sunday coffee report -# -#/mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh -#/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh -#/mnt/user/appdata/unraid_scripts/Orchestrators/weekly_sync_maintenance.sh -#/mnt/user/appdata/unraid_scripts/Orchestrators/critical_sync_maintenance.sh -#/mnt/user/appdata/unraid_scripts/Orchestrators/transcode_management.sh -#/mnt/user/appdata/unraid_scripts/Orchestrators/sunday_morning_coffee_report.sh -#/mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh -#/mnt/user/appdata/unraid_scripts/Monitors/system_tuning_monitor.sh -# ━━━ Partnership ━━━ -#/mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --status -#/mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --onboard --dry-run -#/mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --onboard -#/mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --offboard --dry-run -#/mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --offboard -#/mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --transfer --dry-run -#/mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --transfer --confirm=i-understand-this-transfers-ownership +# ── CRITICAL SYNC MAINTENANCE ───────────────────────────────────────────────────────────────── +# Schedule: */15 * * * * (every 15 minutes) +# Background: YES +# +# Keeps failover-critical data current on HOST2. 15 minutes is the maximum staleness +# for the auth stack and Emby watch states. Run on HOST1 — bidirectional via detect_hosts(). +# +# rsync Critical-Data: +# Auth stack to HOST2: NPM proxy rules, TLS certs, LLDAP user accounts, +# Authelia config and policies. HOST2 auth always within 15min of HOST1. +# +# rsync Emby dirty sync: +# Watch states, user activity, library delta — Emby stays running on both sides. +# WAL/SHM files excluded (unsafe to copy mid-write). HOST2 Emby restarts after sync +# to pick up config changes. +# +# partnership --check: +# Reads remote state file. Increments offline counter on failed sync. +# Triggers auto-offboard at PARTNERSHIP_OFFLINE_THRESHOLD (30 days) unreachable. +# +# Silent on clean cycles: runs 96 times per day, no output when healthy. +# +# bash /mnt/user/appdata/unraid_scripts/Orchestrators/critical_sync_maintenance.sh -# ━━━ Failover ━━━ -# Runs continuously — launched by array_start.sh via ARRAY_START_SCRIPTS. -# Run manually only for testing or if array_start.sh is not in use. -# -#/mnt/user/appdata/unraid_scripts/Failover/failover.sh -#/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh --dry-run -#/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh -# ━━━ Monitors ━━━ -#/mnt/user/appdata/unraid_scripts/Monitors/continuous_scripts_status.sh -# ^^ run manually anytime — live status of system_watchdog, docker_watchdog, failover -#/mnt/user/appdata/unraid_scripts/Monitors/system_tuning_monitor.sh -# ^^ run manually for live snapshot — scheduled every 6hr via cron -#/mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh -#/mnt/user/appdata/unraid_scripts/Monitors/bandwidth_monitor.sh --report -#/mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh -#/mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh -#/mnt/user/appdata/unraid_scripts/Monitors/smart_health.sh -#/mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh -#/mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh +# ── ARRS FAILED/STALLED RECOVERY ───────────────────────────────────────────────────────────── +# Schedule: 0 */6 * * * (every 6 hours) +# Background: YES +# +# Arrs never retry importFailed items automatically — this script handles it while you sleep. +# Configured arrs: Sonarr (HOST1+HOST2), Radarr (HOST1+HOST2), Lidarr (HOST1 only) +# +# Finds items with these statuses, acts on EACH: +# importFailed downloaded but couldn't import → blocklist + remove + new search +# importPending stuck in import queue → blocklist + remove + new search +# error serious failure state → blocklist + remove + new search +# stalled no seeders or no progress → blocklist + remove + new search +# +# Never touches: items with status "downloading" or "imported" +# Age gate: items newer than ARR_IMPORT_RECOVERY_AGE (6hr default) are skipped — arr may still retry. +# +# bash /mnt/user/appdata/unraid_scripts/Orchestrators/arrs_failed_stalled_recovery.sh -# ━━━ Rsync — Appdata Profiles ━━━ -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Critical-Data -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Important-Data -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Gmer4Lfe -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Media_Server/Emby --profile=emby-failover -# ^^ dirty sync — Emby stays running — called by critical_sync_maintenance.sh every 15min -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Media_Server/Emby -# ^^ do NOT schedule — called by weekly_sync_maintenance.sh Sunday 2:30am only -#/mnt/user/appdata/unraid_scripts/Rsync/rsync_stop.sh -# ^^ smart stop — auto-detects orchestrators via /proc, rsync-only if orchestrator running -#/mnt/user/appdata/unraid_scripts/Rsync/rsync_stop.sh --full-stop -# ^^ nuclear — kills orchestrator first then rsync -# ━━━ Rsync — Individual Media Shares (ad hoc) ━━━ -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Movies -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Tv_Shows -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Music -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Music_Videos -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Shows -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Shows-Old -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Movies -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Movies-Old -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Books -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Intros -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Kids_Movies -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Kids_Tv_Shows -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Nextcloud -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Sports -#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/stand-up_comedy +# ── DAILY SYNC MAINTENANCE ──────────────────────────────────────────────────────────────────── +# Schedule: 0 1 * * * (1am daily — runs on BOTH servers independently) +# Background: YES +# +# Full nightly maintenance window. detect_hosts() sets correct sync direction per server. +# HOST1 pushes: Movies, Tv_Shows, Music, Kids content, Sports, stand-up comedy → HOST2 +# HOST2 pushes: Anime_Shows, Anime_Movies → HOST1 +# Personal encrypted shares: appended after standard shares — remote receives ciphertext only. +# +# Execution order (order is not arbitrary — each step depends on the previous): +# 1. git_pull_execute.sh pull latest scripts FIRST — everything below runs on current code +# 2. rsync DAILY_SYNC_SHARES push this server's truth shares to other server +# Temp exit codes: 1=WARN (skip share, continue), 2=CRIT (abort all syncs, notify) +# 3. media_shares_permissions.sh apply nobody:users 755/664 — arr cleanup depends on this +# 4. media_cleaner.sh anime remove scene junk BEFORE orphan detection (cleaner results) +# media_cleaner.sh media +# 5. lidarr_cleanup.sh delete orphaned media — 7 safety checks before any deletion +# sonarr_cleanup.sh notify_emby_scan() after each — ghost entries cleared immediately +# radarr_cleanup.sh +# 6. docker_daily_restart.sh restart: NPM, Authelia, Dispatcharr, Dispatcharr-Basic, ErsatzTV-Emby +# +# Notify: silent on success. One failure notification with job list on any failure. +# +# bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh -# ━━━ Docker Essentials ━━━ -#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_watchdog.sh -# ^^ runs continuously — launched by array_start.sh -#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_daily_restart.sh -# ^^ called by daily_sync_maintenance.sh via DAILY_MAINTENANCE_SCRIPTS -#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh -# ^^ called by weekly_sync_maintenance.sh via WEEKLY_MAINTENANCE_SCRIPTS -#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh -# ^^ ensures custom networks exist + connects containers — called by array_start.sh -#/mnt/user/appdata/unraid_scripts/Docker_Essentials/downloaders_reset.sh --dry-run -#/mnt/user/appdata/unraid_scripts/Docker_Essentials/downloaders_reset.sh -# ^^ called by critical_sync_maintenance.sh every 15min via CRITICAL_MAINTENANCE_SCRIPTS -# ━━━ Media ━━━ -# All media jobs run via DAILY_MAINTENANCE_SCRIPTS — scripts below for manual runs only. -# Always --dry-run first on arr cleanup scripts. +# ── RSYNC EMBY FAILOVER ─────────────────────────────────────────────────────────────────────── +# Schedule: */30 * * * * (every 30 minutes) +# Background: YES # -#/mnt/user/appdata/unraid_scripts/Media/media_shares_permissions.sh -#/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh anime --dry-run -#/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh anime -#/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh media --dry-run -#/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh media -#/mnt/user/appdata/unraid_scripts/Media/lidarr_cleanup.sh --dry-run --log -#/mnt/user/appdata/unraid_scripts/Media/lidarr_cleanup.sh -#/mnt/user/appdata/unraid_scripts/Media/lidarr_cleanup.sh --i-know-what-im-doing --skip-strike-list -#/mnt/user/appdata/unraid_scripts/Media/sonarr_cleanup.sh --dry-run --log -#/mnt/user/appdata/unraid_scripts/Media/sonarr_cleanup.sh -#/mnt/user/appdata/unraid_scripts/Media/radarr_cleanup.sh --dry-run --log -#/mnt/user/appdata/unraid_scripts/Media/radarr_cleanup.sh -#/mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh --dry-run -#/mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh +# Keeps HOST2 Emby within 30 minutes of HOST1 on watch states and library changes. +# Direct rsync.sh call (not an orchestrator). Emby stays running on both sides. +# +# Syncs: users.db, library.db, authentication.db, config/ +# Skips: *.wal *.shm (unsafe mid-write), transcodes/, logs/, cache/ (volatile/local only) +# After: HOST2 Emby restarts to pick up any config changes from the sync. +# Result: if HOST1 fails, users resume from at most 30 minutes stale. +# +# bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ +# /mnt/user/Media_Server/Emby --profile=emby-failover -# ━━━ Transcodes ━━━ -# transcode_management.sh runs cleanup then manager — schedule that, not the individuals. -# ramdisk_setup.sh runs once at array start via ARRAY_START_SCRIPTS. -# -#/mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh -#/mnt/user/appdata/unraid_scripts/Orchestrators/transcode_management.sh -#/mnt/user/appdata/unraid_scripts/Transcodes/transcode_manager.sh --dry-run -#/mnt/user/appdata/unraid_scripts/Transcodes/transcode_manager.sh -#/mnt/user/appdata/unraid_scripts/Transcodes/transcode_cleanup.sh --dry-run -#/mnt/user/appdata/unraid_scripts/Transcodes/transcode_cleanup.sh -# ━━━ Tools ━━━ -#/mnt/user/appdata/unraid_scripts/Tools/bulk_permissions_repair.sh /mnt/user/ShareName -#/mnt/user/appdata/unraid_scripts/Tools/container_data_export.sh ContainerName /path/to/appdata /path/to/output -#/mnt/user/appdata/unraid_scripts/Tools/emby_database_repair.sh -#/mnt/user/appdata/unraid_scripts/Tools/failover_state_reset.sh --status -#/mnt/user/appdata/unraid_scripts/Tools/failover_state_reset.sh -#/mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh -#/mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --status -#/mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --clear ContainerName -#/mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --clear-all -#/mnt/user/appdata/unraid_scripts/Tools/zfs_pool_scrub.sh +# ── WEEKLY SYNC MAINTENANCE ─────────────────────────────────────────────────────────────────── +# Schedule: 30 2 * * 0 (Sunday 2:30am — 4.5 hours before coffee report) +# Background: YES +# +# Full Sunday maintenance window. Containers stop for clean sync. Image updates happen +# at zero extra downtime since containers are already stopped for the rsync anyway. +# +# Execution order: +# 1. Pre-flight checks connectivity, remote Docker daemon, remote rootfs space +# 2. Stop local containers Emby + auth stack on this server +# 3. Stop remote containers Emby + auth stack on remote via SSH +# 4. Pull images locally if WEEKLY_SYNC_UPDATES=true (already stopped = free downtime) +# 5. Pull images remotely if WEEKLY_SYNC_UPDATES_REMOTE=true (SSH to remote) +# 6. rsync WEEKLY_SYNC_SHARES +# Emby full clean mirror — databases checkpointed on shutdown, all config included +# Critical-Data full clean auth stack mirror +# 7. Start remote containers dependency order, new image, verify each container up +# 8. Start local containers dependency order, new image, verify each container up +# 9. docker_weekly_restart.sh restart less-critical services: NextCloud, AdGuard, Immich +# +# Why weekly not nightly for Emby: HOST2 builds warm thumbnail cache throughout the week. +# Nightly sync resets that cache — cold loads every morning. Weekly: cache warm 6 days, +# resets Sunday night while users sleep. +# +# bash /mnt/user/appdata/unraid_scripts/Orchestrators/weekly_sync_maintenance.sh -# ━━━ unRAID Essentials ━━━ -#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/clear_logs.sh -#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/docker_syslog_filter.sh -#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/inotify_tuning.sh -#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/mover_stop.sh -#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/php_fpm_max_children.sh -#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/server_reboot.sh -#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/system_watchdog.sh -#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/user_scripts_stop.sh -#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/webgui_restart.sh -# ━━━ Git ━━━ -#/mnt/user/appdata/unraid_scripts/git_pull_execute.sh +# ── SUNDAY MORNING COFFEE REPORT ───────────────────────────────────────────────────────────── +# Schedule: 0 7 * * 0 (Sunday 7am — after 2:30am maintenance completes) +# Background: YES +# +# Full weekly system overview. Reads data from the week — state files, logs, APIs, live checks. +# +# Sections: +# System uptime, memory, boot/cache drive usage, watchdog-triggered reboots this week +# Array disk count, parity status, ZFS pool health, all drive temperatures +# Transcodes weekly ramdisk peak, flip count, ramdisk vs SSD session split +# Media arr orphan stats (count removed, GB freed), current queue depth per arr +# Rsync weekly transfer totals, per-profile breakdown, failures, large transfers +# Watchdog system + docker watchdog running state, active strikes, skip list +# Security SSL cert days remaining per domain (live TLS connection — not NPM's API) +# Emby version, active streams now (direct vs transcode), weekly play count, top users +# Health inotify peak/avg/warnings, php-fpm peak/avg, SMART pass/fail, git sync state +# Issues anything requiring attention collected from above sections +# +# Footer: ✅ enjoy your Sunday ☕ OR ⚠️ N issue(s) need attention +# Delivery: unRAID notification + Discord webhook (if DISCORD_WEBHOOK configured) +# +# bash /mnt/user/appdata/unraid_scripts/Orchestrators/sunday_morning_coffee_report.sh -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ━━━ ⚙️ Arguments ━━━ -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +# ── WEEKLY HEALTH DIGEST ───────────────────────────────────────────────────────────────────── +# Schedule: 0 8 * * * (8am daily — DIGEST_PROFILE controls when it actually notifies) +# Background: YES # -# --dry-run Preview what would happen — no changes made -# --log Enable verbose logging output -# --no-log Disable logging (overrides Master.conf) -# --status Print resolved config and current state then exit -# --help Show usage information -# KEY=VALUE Override any Master.conf variable for this run only +# Reads ecosystem state files and compiles a digest notification. Read-only, writes nothing. +# Schedule never changes — switch behaviour by changing DIGEST_PROFILE in master.conf only. # -# ━━━ Special arguments ━━━ -# failover.sh --status — check current state without restarting loop -# failover.sh --dry-run --log — test logic without touching containers -# media_cleaner.sh anime --dry-run — profile required as first argument -# bandwidth_monitor.sh --report — generate weekly summary report -# lidarr/sonarr/radarr --dry-run --log — always test arr cleanup before running live -# lidarr/sonarr/radarr --i-know-what-im-doing --skip-strike-list — nuclear mode -# transcode_management.sh --dry-run — passes through to both child scripts -# watchdog_skip_list_manager.sh --status — view skip list and restart history -# watchdog_skip_list_manager.sh --clear Name — clear specific container -# failover_state_reset.sh --status — view current failover state file -# rsync_stop.sh — smart stop, auto-detects what is running -# rsync_stop.sh --full-stop — nuclear, kills orchestrator first -# partnership_manage.sh --status — show current partnership state -# partnership_manage.sh --check — called automatically every 15min, do not run manually +# DIGEST_PROFILE options (master.conf): +# always notify every run regardless — daily digest even when healthy +# smart notify only when something worth reporting is found: +# active watchdog strikes, non-NORMAL failover, cert within CERT_WARN_DAYS, +# bandwidth event exceeded BANDWIDTH_WARN_GB +# weekly notify once per week on DIGEST_DAY (Sunday), silent all other days # -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ━━━ 📋 Recommended Schedules ━━━ -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# Data sources: failover state, watchdog strike files, container skip list, +# bandwidth log, transcode daily log, cert expiry (live openssl check) # -# ━━━ At Startup of Array ━━━ -# array_start.sh — single entry point, launches everything below -# configure what runs in ARRAY_START_SCRIPTS in Master.conf +# bash /mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh + + +# ── WEBGUI RESTART ──────────────────────────────────────────────────────────────────────────── +# Schedule: */10 * * * * (every 10 minutes) +# Background: YES # -# Launched by array_start.sh (configured via ARRAY_START_SCRIPTS): -# ramdisk_setup.sh — creates ramdisk before Emby starts (one-shot) -# docker_syslog_filter.sh — suppress veth noise before logs fill (one-shot) -# php_fpm_max_children.sh — WebGUI performance tuning (one-shot) -# inotify_tuning.sh — bump inotify limits (one-shot) -# docker_network_connect.sh — ensure custom networks exist + connect (one-shot) -# system_watchdog.sh — system health monitor (continuous) -# docker_watchdog.sh — container health monitor (continuous) -# failover.sh — mutual failover (continuous) — enable when HOST2 ready +# Monitors WebGUI availability and recovers it automatically when unresponsive. +# Silent when healthy — 144 runs per day produce no output when WebGUI responds normally. +# Notification fires on any restart so you know what recovered and when. # -# ━━━ Every 3 minutes ━━━ -# */3 * * * * transcode_management.sh (cleanup + manager — correct order) +# Escalation (each step only runs if previous step's recheck still fails): +# 1. curl http://localhost unresponsive? +# 2. Restart nginx wait WEBGUI_NGINX_WAIT=15s, recheck +# 3. Restart php-fpm wait WEBGUI_PHP_WAIT=10s, recheck +# handles PHP worker saturation (WebGUI feels slow) +# 4. Restart emhttp (core daemon) wait WEBGUI_EMHTTP_WAIT=30s, recheck +# 5. All failed → notify warning, manual intervention needed # -# ━━━ Every 15 minutes ━━━ -# */15 * * * * critical_sync_maintenance.sh (downloaders reset + auth sync + partnership check) +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/webgui_restart.sh + + +# ── SYSTEM TUNING MONITOR ──────────────────────────────────────────────────────────────────── +# Schedule: 0 */6 * * * (every 6 hours) +# Background: YES # -# ━━━ Every 6 hours ━━━ -# 0 */6 * * * arrs_failed_stalled_recovery.sh (blocklist + re-search failed/stalled) -# 0 */6 * * * system_tuning_monitor.sh (inotify + php-fpm snapshot) +# Snapshots inotify and PHP-FPM utilisation. Writes one bounded log entry. +# sunday_morning_coffee_report.sh reads this log for weekly peak/avg/warning counts. # -# ━━━ Daily ━━━ -# 0 1 * * * daily_sync_maintenance.sh (git pull + media sync + maintenance + restart) -# 0 8 * * * weekly_health_digest.sh (profile controls if/when it sends) +# inotify: current instances vs kernel limit. Warns above INOTIFY_WARN_PCT (80%). +# Exhaustion symptom: downloads complete but arrs don't detect them, Live TV stutters, +# library updates stop — no obvious error message anywhere. # -# ━━━ Weekly ━━━ -# 30 2 * * 0 weekly_sync_maintenance.sh (Emby + auth sync + updates + weekly restart) -# 0 7 * * 0 sunday_morning_coffee_report.sh (full weekly health report — Sunday 7am) +# php-fpm: active workers vs PHP_MAX_CHILDREN. Warns above PHP_FPM_WARN_PCT (80%). +# Exhaustion symptom: WebGUI slows, settings saves hang, container UI starts timeout. # -# ━━━ Manual only ━━━ -# continuous_scripts_status.sh (live status of all watchdogs + failover) -# rsync_stop.sh (stop rsync — smart auto-detect) -# rsync_stop.sh --full-stop (stop everything including orchestrator) -# partnership_manage.sh --onboard (set up mirror relationship) -# partnership_manage.sh --offboard (clean separation — either server) -# partnership_manage.sh --transfer (flip ownership — owner only) +# Log: TUNING_MONITOR_LOG — bounded, trimmed to TUNING_LOG_RETENTION days on each write. # -# ━━━ Rsync profiles — individual schedules ━━━ -# Arrs_Stack — daily or every few days (arr databases change on every download) -# Critical-Data — handled by critical_sync_maintenance.sh every 15min — no separate schedule -# Important-Data — daily (NextCloud file changes) -# Gmer4Lfe — daily or weekly (personal appdata) -# Emby — handled by weekly_sync_maintenance.sh Sunday 2:30am — no separate schedule -# emby-failover — handled by critical_sync_maintenance.sh every 15min — no separate schedule +# bash /mnt/user/appdata/unraid_scripts/Monitors/system_tuning_monitor.sh + + +# ── CLEAR LOGS ──────────────────────────────────────────────────────────────────────────────── +# Schedule: 0 5 * * 0 (Sunday 5am — after 2:30am maintenance, before 6am monitors) +# Background: YES +# +# Clears system and Docker container logs using size thresholds. +# Only clears logs large enough to be worth clearing — preserves recent history on small logs. +# +# System logs (/var/log/syslog, messages, dmesg): only if > LOG_MIN_SIZE_MB (10MB) +# Docker container logs: only if > LOG_DOCKER_MAX_MB (100MB) per container +# Active containers (Emby, SABnzbd) grow fastest — 100MB threshold catches them. +# Truncation not rotation: > file keeps file descriptor valid for running services. +# +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/clear_logs.sh + + +# ── SUNDAY MONITOR BLOCK ───────────────────────────────────────────────────────────────────── +# Schedule: See individual entries below — 6am through 11am Sunday +# Background: YES on all +# +# Create a separate User Script entry for each monitor. Results feed into the 7am coffee report. +# All monitors are read-only — observe and report, never act. Safe to run any time. +# +# 0 6 * * 0 bash /mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh +# 0 7 * * 0 bash /mnt/user/appdata/unraid_scripts/Monitors/smart_health.sh +# 0 9 * * 0 bash /mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh +# 0 10 * * 0 bash /mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh +# 0 11 * * 0 bash /mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh +# 0 11 * * 0 bash /mnt/user/appdata/unraid_scripts/Monitors/bandwidth_monitor.sh --report + + +# ============================================================================================== +# ██ INDIVIDUAL SCRIPTS — STANDALONE AND SELECTIVE USE ███████████████████████████████████████ +# ============================================================================================== +# +# Everything below is called by the orchestrators above when using the full ecosystem. +# Listed here so you can: +# → Run one script manually without triggering the full orchestrator +# → Use only specific parts of the ecosystem (just arr cleanup, just transcode manager) +# → Test individual scripts before scheduling the orchestrator +# → Schedule individual scripts on different timing than the orchestrator uses +# +# Every script is fully standalone — sources its own config, runs its own pre-flight checks. +# +# ============================================================================================== + + +# ────────────────────────────────────────────────────────────────────────────────────────────── +# FAILOVER +# ────────────────────────────────────────────────────────────────────────────────────────────── + +# failover.sh — [continuous] mutual failover state machine +# Started by array_start.sh on both servers independently. +# Every FAILOVER_CHECK_INTERVAL (120s) pings: remote Tailscale IP + 8.8.8.8 +# States: NORMAL / FAILOVER / NO_INTERNET / DARK +# FAILOVER: starts remote containers in tiers across 24 hours: +# Tier 1 immediate: DDNS + Emby + auth (NPM/Authelia/LLDAP) + VaultWarden + Live TV +# Tier 2 at 4hr: NextCloud + Immich + Jellyseerr +# Tier 3 at 12hr: AdGuard + Gitea + uptime monitoring +# Tier 4 at 24hr: arrs + downloaders +# Auth stack (NPM, Authelia, LLDAP) runs warm on both servers always — no failover startup delay. +# DDNS: one domain, one DDNS container active at all times. Split-brain prevented by design. +# Handback: stop remote DDNS → rsync writeback → start containers → start local DDNS. +# +# bash /mnt/user/appdata/unraid_scripts/Failover/failover.sh +# bash /mnt/user/appdata/unraid_scripts/Failover/failover.sh --status +# bash /mnt/user/appdata/unraid_scripts/Failover/failover.sh --dry-run + +# failover_test.sh — controlled full-lifecycle failover test using iptables simulation +# ALWAYS run --dry-run first. Live test starts/stops real containers — brief service interruption. +# iptables safety trap removes the block rule on ANY exit including ctrl-c and crashes. +# Remote is always reachable after the test completes regardless of what happened. +# 8 phases: pre-flight → block remote (iptables) → detect FAILOVER → verify Tier 1 started +# → restore remote → wait handback → verify NORMAL restored → full pass/fail report. +# FAILOVER_TEST_BLOCK_WAIT must be > FAILOVER_CHECK_INTERVAL + buffer (default: 150s). +# FAILOVER_TEST_HANDBACK_WAIT must cover: strike confirmation + rsync + container start (default: 360s). +# +# bash /mnt/user/appdata/unraid_scripts/Failover/failover_test.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Failover/failover_test.sh + +# failover_state_reset.sh — reset failover state file to NORMAL +# Resets state file ONLY — does NOT start or stop any containers. +# Use when: state stuck in FAILOVER after testing, after killing failover.sh mid-cycle, +# after manual intervention left state inconsistent, after a failed handback. +# Shows current state file before asking for confirmation. +# Verify before resetting: right containers on right server, DDNS correct, +# both servers Tailscale visible, failover.sh not currently running. +# +# bash /mnt/user/appdata/unraid_scripts/Failover/failover_state_reset.sh --status +# bash /mnt/user/appdata/unraid_scripts/Failover/failover_state_reset.sh +# bash /mnt/user/appdata/unraid_scripts/Failover/failover_state_reset.sh --force + + +# ────────────────────────────────────────────────────────────────────────────────────────────── +# RSYNC +# ────────────────────────────────────────────────────────────────────────────────────────────── + +# rsync.sh — core sync script, call with share path and optional --profile=name +# Pre-flight every run: connectivity, version parity, remote Docker daemon, +# remote rootfs space, remote share exists. Profile matched by directory basename (lowercased). +# Drive temperature: reads thresholds from dynamix.cfg. Exit 1=WARN (skip), 2=CRIT (caller aborts). +# Logs each transfer to BANDWIDTH_LOG automatically (read by bandwidth_monitor --report). +# +# Profile behaviour — what containers stop: +# critical-data Mariadb-Authelia, Redis-Authelia, NginxProxyManager, Lldap-Gmer4Lfe +# + delayed restart: Authelia, Authelia-Secondary (30s after dependencies) +# important-data Postgres-NextCloud + delayed: NextCloud +# arrs_stack Sonarr, Radarr, Lidarr, Prowlarr, Bazarr, Pinchflat +# emby Emby both sides (weekly clean sync — both instances stopped) +# emby-failover nothing stopped (Emby stays running — dirty sync, WAL/SHM excluded) +# [no profile] no containers stopped (media shares, plain data) +# +# bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ +# /mnt/user/appdata-Failover/Critical-Data +# bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ +# /mnt/user/appdata-Failover/Important-Data +# bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ +# /mnt/user/appdata-Failover/Arrs_Stack +# bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ +# /mnt/user/Media_Server/Emby --profile=emby +# bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ +# /mnt/user/Media_Server/Emby --profile=emby-failover +# bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ +# /mnt/user/appdata-Failover/Gmer4Lfe +# bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Movies +# bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Tv_Shows +# bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Music +# bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Shows +# bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Movies + +# rsync_stop.sh — stop rsync intelligently on local and remote +# Smart mode (default): kills rsync subprocess only, orchestrator exits cleanly, +# any stopped containers are recovered by docker_watchdog.sh on next cycle. +# --full-stop: kills orchestrator first then rsync — emergency full stop. +# --rsync-only: stop rsync, skip container recovery check (used by partnership_manage --offboard). +# +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/rsync_stop.sh --status +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/rsync_stop.sh +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/rsync_stop.sh --full-stop + + +# ────────────────────────────────────────────────────────────────────────────────────────────── +# DOCKER ESSENTIALS +# ────────────────────────────────────────────────────────────────────────────────────────────── + +# docker_watchdog.sh — [continuous] two-tier container self-healing monitor +# Started by array_start.sh. Every DOCKER_WATCHDOG_INTERVAL (900s = 15min). +# +# Tier 1 — explicit per-container (configured in master_host*.conf): +# Memory hard limits: immediate restart when exceeded — no strikes, no waiting +# CPU strike system: 2 consecutive cycles above HARD_CPU_THRESHOLD → restart +# HTTP health checks: curl to configured URL — 2 consecutive failures → restart +# Required containers: found stopped → restart attempt → failure → skip list +# Dependency ordering: database must be running before dependent is restarted +# +# Tier 2 — global scan of ALL running containers (when WATCHDOG_SCAN_ALL=true): +# HEALTHCHECK unhealthy → restart +# OOM kill detected → restart + notify +# Crash loop (RestartCount climbing) → notify, above WATCHDOG_CRASH_LIMIT → restart → skip list +# Dead state → remove + start +# Non-zero exit → restart +# +# Startup grace: WATCHDOG_STARTUP_GRACE (600s) — restarts suppressed, checks still log +# Restart loop: 3 restarts in 1hr → skip list on /boot/config → critical notification +# RAM emergency: reads mem_shutdown_active from system_watchdog state file, defers all restarts +# Silent on clean cycles — only outputs events and hourly heartbeat +# +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_watchdog.sh --status +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_watchdog.sh --dry-run + +# watchdog_skip_list_manager.sh — view and manage the container skip list +# When docker_watchdog restarts the same container 3 times in 1hr → skip-listed. +# Skip list persists on /boot/config — survives reboots. Still broken after reboot = stays listed. +# Auto-clear: watchdog removes a container from the skip list when it sees it running. +# Manual clear needed only when container is stuck stopped after you've fixed the problem. +# +# Recovery workflow: +# --status → understand situation (skip list, running/stopped state, restart history) +# fix the underlying problem (check docker logs, disk space, database integrity) +# --clear ContainerName → remove from skip list + clear restart history +# docker start ContainerName → confirm fix works before handing back to watchdog +# watchdog resumes normal monitoring on next cycle automatically +# +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/watchdog_skip_list_manager.sh --status +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/watchdog_skip_list_manager.sh --clear ContainerName +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/watchdog_skip_list_manager.sh --clear ContainerName --force +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/watchdog_skip_list_manager.sh --clear-all +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/watchdog_skip_list_manager.sh --dry-run + +# docker_daily_restart.sh — restart degradation-prone containers nightly +# Called by daily_sync_maintenance.sh. Also safe to run ad hoc. +# Running → docker restart (graceful). Stopped → left stopped (state respected). Missing → skip. +# Dependency ordering via WATCHDOG_DEPENDENCIES — databases before applications. +# Restart verification: checks container still up after settle period, notifies if not. +# Configured via HOST*_DAILY_RESTART_CONTAINERS (master_host*.conf): +# NginxProxyManager, Authelia, Dispatcharr, Dispatcharr-Basic, ErsatzTV-Emby +# +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_daily_restart.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_daily_restart.sh --status +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_daily_restart.sh + +# docker_weekly_restart.sh — restart less-critical services weekly +# Called by weekly_sync_maintenance.sh after sync completes and containers are back up. +# Targets services that benefit from weekly clean start but don't stop for the sync itself. +# Same rules as daily: running→restart, stopped→leave, missing→skip. +# Configured via HOST*_WEEKLY_RESTART_CONTAINERS (master_host*.conf): +# NextCloud, AdGuard-Home, Immich +# +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh --status +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh + +# docker_network_connect.sh — ensure custom networks exist and containers are connected +# Called by array_start.sh. Idempotent — silent when everything is already correct. +# Network missing → create (bridge driver, auto-assigned subnet) → notify. +# Network creation should only happen after an unRAID update wiped networks — notify tells you. +# Configured via HOST*_NETWORK_CONNECT_NETWORKS and HOST*_NETWORK_CONNECT_CONTAINERS. +# Use after unRAID updates when containers can't reach each other on custom networks. +# +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh --status +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh + + +# ────────────────────────────────────────────────────────────────────────────────────────────── +# UNRAID ESSENTIALS +# ────────────────────────────────────────────────────────────────────────────────────────────── + +# system_watchdog.sh — [continuous] three-tier server stability last-resort watchdog +# Started by array_start.sh. Every SYSTEM_WATCHDOG_INTERVAL (300s = 5min). +# All 18 checks independently toggleable per host in master_host*.conf. +# +# Tier 1 CRITICAL — bypass ALL strikes, reboot immediately: +# Docker daemon hung → attempt rc.docker restart → still hung → reboot +# rootfs >= 99% → reboot (writes failing, system cannot recover) +# Kernel BUG/Oops delta → reboot (kernel running with corrupted state) +# FD exhaustion >= 95% → reboot (new connections failing, Docker can't spawn) +# /boot read-only → reboot (state file writes silently failing) +# +# Tier 2 URGENT — bypass strikes when OOM confirms RAM crisis: +# RAM < MEM_GB (4GB) AND >= OOM_LIMIT (3) OOM kills this cycle → reboot +# Without OOM confirmation: standard strike system applies. +# +# Tier 3 STANDARD — N consecutive failures → reboot: +# RAM tiers: warn (10GB), stop containers (6GB, excluded list respected), reboot (4GB) +# Load average, CPU temperature, /var/log usage, /tmp usage, zombie processes, +# sshd, NIC operstate, mdstat errors, kernel oops rate, FD rate +# +# RAM emergency: writes mem_shutdown_active=true → docker_watchdog defers all restarts. +# Reboot loop protection: N reboots in X hours → shutdown instead. +# State file heartbeat: writes watchdog_cycle=N every cycle (docker_watchdog stale guard). +# +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/system_watchdog.sh --status +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/system_watchdog.sh --dry-run + +# inotify_tuning.sh — raise Linux inotify kernel limits at array start +# Called by array_start.sh FIRST — must run before containers start (they inherit limits). +# Settings reset on each reboot — script reapplies on every array start. Idempotent. +# +# Limits set: +# max_user_watches → 1048576 (default 8192 — VSCode alone needs 50-200K for large projects) +# max_user_instances → 1024 (default 128) +# max_queued_events → 32768 (default 16384) +# +# If Code-Server started before this ran → docker restart Code-Server to inherit new limits. +# Exhaustion symptom: downloads complete but arrs don't detect them, Live TV stutters. +# +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/inotify_tuning.sh --status +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/inotify_tuning.sh + +# php_fpm_max_children.sh — set PHP-FPM pm.max_children at array start +# Called by array_start.sh. Default is 4-8 workers — inadequate for a busy multi-user server. +# Sets PHP_MAX_CHILDREN (250). 250 × ~2MB idle = ~500MB. Acceptable on 64GB+. +# Symptom of saturation: WebGUI slow, settings saves hang, container UI starts timeout. +# Resets on each reboot — reapplied at array start. Idempotent: silent when already correct. +# +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/php_fpm_max_children.sh --status +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/php_fpm_max_children.sh + +# docker_syslog_filter.sh — suppress Docker veth/docker0 interface log noise +# Called by array_start.sh before containers start. Creates rsyslog drop rule. +# Without this: 50+ containers at array start = 200-400 lines of kernel veth messages. +# Real events (mount failures, permission errors) are invisible in that noise. +# Idempotent: compares expected filter content exactly — only writes when changed. +# +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/docker_syslog_filter.sh --status +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/docker_syslog_filter.sh + +# webgui_restart.sh — WebGUI availability watchdog (see ORCHESTRATORS section above) +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/webgui_restart.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/webgui_restart.sh --status + +# clear_logs.sh — size-threshold log cleanup (see ORCHESTRATORS section above) +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/clear_logs.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/clear_logs.sh --status + +# mover_stop.sh — stop unRAID mover cleanly before array operations +# Wall message to logged-in users. Waits MOVER_STOP_TIMEOUT seconds. +# SIGTERM (allows mover to finish current file — no partial files). Verify. SIGKILL last resort. +# Use before: planned reboots with mover running, disk replacement, array maintenance. +# +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/mover_stop.sh --status +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/mover_stop.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/mover_stop.sh + +# server_reboot.sh — graceful reboot with pre-flight warnings and clean shutdown sequence +# Pre-flight warnings (inform not block): rsync running, mover running, active Emby sessions. +# Sequence: wall + unRAID notification → wait REBOOT_SLEEP (30s) → virsh shutdown VMs +# → wait REBOOT_VM_WAIT → stop libvirt → stop Docker → sync → /sbin/reboot +# --dry-run walks the full sequence without rebooting — shows all active processes. +# +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/server_reboot.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/server_reboot.sh --status +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/server_reboot.sh + +# user_scripts_stop.sh — stop all running User Script processes +# Finds processes by /tmp/user.scripts path signature. Shows script names, not just PIDs. +# SIGTERM → verify → SIGKILL. Self-exclusion prevents killing itself. +# Use when plugin Abort button didn't work, or before a reboot to clean up running scripts. +# Called automatically by server_reboot.sh before reboot. +# +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/user_scripts_stop.sh --status +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/user_scripts_stop.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/user_scripts_stop.sh + +# git_pull_execute.sh — pull latest scripts from Gitea and set execute permissions +# Deployment mechanism for the ecosystem. Push from VS Code → Gitea → run on both servers. +# Called automatically by daily_sync_maintenance.sh as the first job in the daily window. +# +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/git_pull_execute.sh + + +# ────────────────────────────────────────────────────────────────────────────────────────────── +# MEDIA +# ────────────────────────────────────────────────────────────────────────────────────────────── +# +# All media scripts run via daily_sync_maintenance.sh in the correct order: +# permissions → cleaners → arr cleanup +# +# ORDER MATTERS: arr cleanup depends on correct ownership (permissions first) and clean +# folders (junk removal before orphan detection — otherwise junk files appear as orphans). +# Listed here for standalone testing or selective use outside the daily window. + +# media_shares_permissions.sh — apply correct ownership and permissions to all media shares +# Applies nobody:users 755 (dirs) / 664 (files) to HOST*_MEDIA_PERMISSION_SHARES. +# Two separate passes: find -type d and find -type f — directories get 755, files get 664. +# Idempotent: only changes what's wrong. On large libraries takes 20-30 minutes. +# Common drift causes: scp/cp leaves root:root, container missing PUID=99 PGID=100, +# rsync without --chown flag, admin copies directly into share. +# Correct PUID/PGID in Docker template env vars is the permanent fix — this is the safety net. +# +# bash /mnt/user/appdata/unraid_scripts/Media/media_shares_permissions.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Media/media_shares_permissions.sh --status +# bash /mnt/user/appdata/unraid_scripts/Media/media_shares_permissions.sh + +# media_cleaner.sh — remove scene junk files from media share folders +# Run BEFORE arr cleanup — removes non-media files that would otherwise appear as orphans. +# Patterns: *.sfv *.md5 *.sha1 *.nfo *.url *.lnk *.rar *.zip *.info *.torrent +# *.sample* *.proof* *sync-conflict* *.scr *.exe *.srr *.log *.json +# Two profiles with separate folder lists (configured in master_host*.conf): +# anime HOST*_ANIME_CLEAN_FOLDERS — anime share folders +# media HOST*_MEDIA_CLEAN_FOLDERS — Movies, Tv_Shows, Music, Sports etc. +# ALWAYS --dry-run when adding new patterns or folders — verify before committing. +# +# bash /mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh anime --dry-run +# bash /mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh media --dry-run +# bash /mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh anime +# bash /mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh media + +# lidarr_cleanup.sh — delete orphaned music files not tracked by Lidarr +# HOST1 ONLY — HOST1 owns /mnt/user/Music and runs Lidarr. Exits cleanly on HOST2. +# +# Classification per file on disk: +# TRACKED → in Lidarr API response → never touch +# PROTECTED → matches LIDARR_PROTECTED_PATTERNS → never touch +# (*.jpg *.png = cover art, *.nfo = metadata, *.lrc = synced lyrics) +# ORPHAN → music extension, not tracked, older than LIDARR_ORPHAN_AGE (7 days), +# not open by any process → DELETE +# JUNK → not a music extension, not protected → DELETE (any age) +# RECENT → not tracked, under 7 days → skip (may be mid-import) +# +# 7 safety checks ALL must pass before any deletion: +# Lidarr running + healthy. API reachable. API version matches. Artist count > 0. +# Tracked file count > 0. Count >= 80% of last known (detects database issues). +# Total deletion size < LIDARR_MAX_DELETE_GB (50GB — catches path misconfigurations). +# +# After any deletion: notify_emby_scan() triggers Emby "Clean Missing Files" immediately. +# NUCLEAR MODE (use with full understanding): --i-know-what-im-doing --skip-strike-list +# Both flags: age check bypassed + size gate bypassed. Deletes everything not tracked. +# +# bash /mnt/user/appdata/unraid_scripts/Media/lidarr_cleanup.sh --dry-run --log +# bash /mnt/user/appdata/unraid_scripts/Media/lidarr_cleanup.sh --status +# bash /mnt/user/appdata/unraid_scripts/Media/lidarr_cleanup.sh + +# sonarr_cleanup.sh — delete orphaned TV files not tracked by Sonarr +# HOST-AWARE: HOST1 manages Tv_Shows (via HOST1_SONARR_TV_ROOT). +# HOST2 manages Anime_Shows (via HOST2_SONARR_TV_ROOT). +# detect_hosts() selects correct root — same script works on both servers. +# Same 7-check safety system as lidarr_cleanup. +# Protected: *.jpg *.png (episode/show artwork), *.nfo, *.srt *.sub *.ass *.ssa (Bazarr subtitles) +# After deletion: notify_emby_scan() clears ghost entries from Emby library immediately. +# +# bash /mnt/user/appdata/unraid_scripts/Media/sonarr_cleanup.sh --dry-run --log +# bash /mnt/user/appdata/unraid_scripts/Media/sonarr_cleanup.sh --status +# bash /mnt/user/appdata/unraid_scripts/Media/sonarr_cleanup.sh + +# radarr_cleanup.sh — delete orphaned movie files not tracked by Radarr +# HOST-AWARE: HOST1 manages Movies (via HOST1_RADARR_MOVIES_ROOT). +# HOST2 manages Anime_Movies (via HOST2_RADARR_MOVIES_ROOT). +# Same 7-check safety system as lidarr_cleanup. +# Protected: *.jpg *.png (movie artwork, fanart, thumbnails), *.nfo, subtitles +# After deletion: notify_emby_scan() clears ghost entries from Emby library immediately. +# +# bash /mnt/user/appdata/unraid_scripts/Media/radarr_cleanup.sh --dry-run --log +# bash /mnt/user/appdata/unraid_scripts/Media/radarr_cleanup.sh --status +# bash /mnt/user/appdata/unraid_scripts/Media/radarr_cleanup.sh + + +# ────────────────────────────────────────────────────────────────────────────────────────────── +# TRANSCODES +# ────────────────────────────────────────────────────────────────────────────────────────────── + +# ramdisk_setup.sh — create tmpfs ramdisk and transcode symlink at array start +# Called by array_start.sh. MUST run before Emby starts. +# Idempotent: if ramdisk already mounted → report status and exit cleanly, do not remount. +# +# Creates: +# RAMDISK_PATH tmpfs mount (HOST*_RAMDISK_SIZE ceiling — only uses RAM actually needed) +# transcoding-temp/ pre-created inside ramdisk — Emby finds this before the SSD version. +# Without pre-creation: Emby creates transcoding-temp at first writable +# location which may be SSD, routing ALL sessions there until restart. +# TRANSCODE_SSD SSD fallback directory (and SSD/transcoding-temp for consistency) +# TRANSCODE_LINK symlink → RAMDISK_PATH (always reset to ramdisk at array start) +# +# IMPORTANT: Emby Extra Parameters must include: +# --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared +# Without shared propagation: symlink flips silently ignored after first flip (rprivate locks inode). +# +# Verify setup: +# mountpoint /mnt/ramdisk_transcodes should say "is a mountpoint" +# readlink /mnt/ram-transcode should show /mnt/ramdisk_transcodes +# ls /mnt/ramdisk_transcodes/ should show transcoding-temp/ +# +# bash /mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh --status +# bash /mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh + +# transcode_manager.sh — manage ramdisk/SSD symlink based on current usage +# Called by transcode_management.sh AFTER cleanup. Can be run standalone for testing. +# Modes (TRANSCODE_MANAGER_MODE in master.conf): +# smart auto-flip between ramdisk and SSD based on thresholds (default) +# ramdisk always ramdisk, never flip to SSD (warn if above threshold) +# ssd always SSD, never ramdisk (use post-flip drain or maintenance) +# +# Safety checks every cycle regardless of mode: +# Missing/broken symlink → recreate pointing at ramdisk +# Ramdisk disappeared → auto-flip to SSD, notify warning +# SSD path missing → disable SSD fallback / error in ssd mode +# transcoding-temp missing from ramdisk → recreate immediately (prevents SSD session drift) +# Permissions drift → fix silently every run +# +# Multi-server: TRANSCODE_SERVERS array supports Emby, Jellyfin, Plex +# DO NOT add Tdarr here — full-file encoding fills ramdisk rapidly. +# +# bash /mnt/user/appdata/unraid_scripts/Transcodes/transcode_manager.sh --status +# bash /mnt/user/appdata/unraid_scripts/Transcodes/transcode_manager.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Transcodes/transcode_manager.sh + +# transcode_cleanup.sh — remove stale HLS segment files from ramdisk and SSD fallback +# Called by transcode_management.sh BEFORE transcode_manager.sh (order critical). +# A file is eligible only when: older than TRANSCODE_MAX_AGE (20min) AND not open by any process. +# lsof called ONCE per location (not once per file — critical performance on Live TV systems). +# transcoding-temp directory NEVER deleted even when empty — protects against SSD session drift. +# After cleanup: if ramdisk usage dropped below RAMDISK_LOW_GB, triggers flip-back to ramdisk. +# +# bash /mnt/user/appdata/unraid_scripts/Transcodes/transcode_cleanup.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Transcodes/transcode_cleanup.sh + + +# ────────────────────────────────────────────────────────────────────────────────────────────── +# MONITORS +# ────────────────────────────────────────────────────────────────────────────────────────────── +# +# All monitor scripts observe and report — they NEVER act on what they find. +# Read-only except bandwidth_monitor (one bounded log append per rsync) and +# system_tuning_monitor (one bounded log append per 6hr snapshot). +# Safe to run at any time. + +# zfs_memory_snapshot.sh — weekly ZFS + ARC + memory diagnostic report +# ZFS pool status (ONLINE/DEGRADED/FAULTED) per pool. Pools in ZFS_REPORT_IGNORE_POOLS excluded. +# ARC: current size vs max, hit rate, metadata vs data ratio. Warns if ARC > ZFS_REPORT_ARC_WARN_PCT (90%). +# Memory: total, free, available. Warns if RAM < ZFS_REPORT_FREE_WARN_GB. +# Docker: top ZFS_REPORT_DOCKER_TOP (10) containers by memory usage. +# Kernel pressure: vmstat 3-sample snapshot. +# Output: console + /var/log/zfs-weekly-health.log (enables week-over-week comparison). +# +# bash /mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh --status +# bash /mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh + +# smart_health.sh — drive SMART attribute monitoring +# Checks every /dev/sd* and /dev/nvme*. Drives in HOST*_SMART_IGNORE_DRIVES are skipped. +# Temperature thresholds read from /boot/config/plugins/dynamix/dynamix.cfg (same as unRAID UI). +# Notifies on: Reallocated_Sector_Ct > 0 (bad sector swapped = warning sign), +# Current_Pending_Sector > 0 (suspected bad, not confirmed), Offline_Uncorrectable > 0 (data loss risk), +# temperature above thresholds, SMART overall-health FAILED (get data off immediately). +# Silent when all drives pass all checks. +# +# bash /mnt/user/appdata/unraid_scripts/Monitors/smart_health.sh --status +# bash /mnt/user/appdata/unraid_scripts/Monitors/smart_health.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Monitors/smart_health.sh + +# cert_monitor.sh — SSL certificate expiry via live TLS connection +# Connects via openssl s_client — checks cert actually being served to clients. +# Catches: renewed-but-not-reloaded (nginx not reloaded after certbot renewal), +# wrong cert served, chain issues visible externally but not internally. +# If a user would see a certificate error in their browser, this catches it first. +# Configured via HOST*_CERT_MONITOR_DOMAINS in master_host*.conf. +# Thresholds: > 30 days = silent, <= 30 = warning, <= CERT_CRIT_DAYS (7) = urgent. +# +# bash /mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh --status +# bash /mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh + +# backup_verify.sh — rsync mirror integrity via independent MD5 checksums +# Randomly samples BACKUP_VERIFY_SAMPLE (10) files per share. Computes MD5 both sides via SSH. +# Does not trust rsync exit code — verifies actual file content matches. +# rsync exit 0 does NOT guarantee content is correct. Hardware corruption produces files +# that exist with matching size/mtime but different content. rsync misses this. MD5 doesn't. +# Pre-flight: check_connectivity() and check_remote_array() prevent false MISSING alarms. +# Notifies on MISMATCH or excessive MISSING files. +# +# bash /mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh --status +# bash /mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh + +# emby_session_report.sh — Emby weekly streaming usage statistics +# Queries HOST*_EMBY_URL for past EMBY_REPORT_DAYS (7) days of activity. +# Reports: server version, active now (direct vs transcode), library size (movies/episodes/songs), +# weekly play count, top EMBY_REPORT_TOP_N content, most active users. +# Notifies if transcode ratio > 80% — usually indicates client/format compatibility issue. +# +# bash /mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh --status +# bash /mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh + +# bandwidth_monitor.sh — rsync transfer history and weekly summary report +# --report: reads BANDWIDTH_LOG and generates weekly summary per profile. +# Shows: run count, total GB transferred, average duration, failure count, large transfers. +# --log-transfer: called AUTOMATICALLY by rsync.sh after each sync — never call manually. +# Log format: DATE|TIME|profile|duration|status|bytes_transferred +# Version-stable field names — survives rsync output format changes between versions. +# BANDWIDTH_LOG bounded to BANDWIDTH_LOG_RETENTION (90) days — never grows unbounded. +# +# bash /mnt/user/appdata/unraid_scripts/Monitors/bandwidth_monitor.sh --report +# bash /mnt/user/appdata/unraid_scripts/Monitors/bandwidth_monitor.sh --status + +# weekly_health_digest.sh — profile-controlled ecosystem health digest (see ORCHESTRATORS) +# bash /mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh --status + +# system_tuning_monitor.sh — inotify + PHP-FPM trend tracking (see ORCHESTRATORS) +# bash /mnt/user/appdata/unraid_scripts/Monitors/system_tuning_monitor.sh --status + +# continuous_scripts_status.sh — live dashboard for all three background processes +# The single command that shows you what the entire continuous-running ecosystem is doing. +# Shows for system_watchdog, docker_watchdog, and failover: +# Running state, PID, uptime, approximate cycle count +# Active strikes per check. Required container states. Memory-monitored container usage. +# Skip list contents (any entry needs human attention). +# Failover state, active tiers, outage duration, remote Tailscale visibility. +# No schedule — on demand only. Run any time you want a complete status snapshot. +# +# bash /mnt/user/appdata/unraid_scripts/Monitors/continuous_scripts_status.sh +# bash /mnt/user/appdata/unraid_scripts/Monitors/continuous_scripts_status.sh --log + + +# ────────────────────────────────────────────────────────────────────────────────────────────── +# PARTNERSHIP +# ────────────────────────────────────────────────────────────────────────────────────────────── + +# partnership_manage.sh — two-server auth-sharing relationship lifecycle manager +# Automated --check runs inside critical_sync_maintenance.sh. Below are manual operation modes. +# +# --status: show both server state files (ACTIVE/INACTIVE), offline counter, owner server, +# current WebUI redirect state on HOST2, whether critical sync is enabled. +# +# --onboard: HOST1/owner only. Reconfigures HOST2 Docker template WebUI URLs to redirect +# to HOST1 via Tailscale (click NPM on HOST2 → opens HOST1's NPM). +# Verifies WebUI routing works. Writes ACTIVE state on both servers. +# ALWAYS --dry-run first — shows exact template changes before anything is touched. +# +# --offboard: either server can initiate. Stops rsync. Final sync to ensure HOST2 has +# current state. Reconfigures HOST2 WebUI URLs back to localhost (HOST2 manages +# its own auth independently from now on). Grace period then Tailscale removal. +# Deferred finalisation: if remote unreachable, completes automatically when reachable. +# ALWAYS --dry-run first. 10-second countdown before live execution. +# +# --transfer: owner only. Flips auth ownership (HOST1→HOST2 or back). Health strike system +# verifies both servers healthy before proceeding. Requires confirmation string: +# "i-understand-this-transfers-ownership" (long by design — cannot type accidentally). +# +# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --status +# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --onboard --dry-run +# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --onboard +# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --offboard --dry-run +# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --offboard +# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --transfer --dry-run +# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh \ +# --transfer --confirm=i-understand-this-transfers-ownership + + +# ────────────────────────────────────────────────────────────────────────────────────────────── +# TOOLS +# ────────────────────────────────────────────────────────────────────────────────────────────── +# +# Situational utilities — run when something specific needs fixing, not on a schedule. +# Each exists because a specific situation arose that required bash commands to resolve. +# All support --dry-run and --status. + +# bulk_permissions_repair.sh — apply nobody:users 755/664 to a specific path immediately +# Targeted fix — do not wait for next nightly media_shares_permissions.sh run. +# Use when: scp/cp left root:root files blocking arr operations right now. +# Accepts one or multiple paths on the same command line. +# +# bash /mnt/user/appdata/unraid_scripts/Tools/bulk_permissions_repair.sh /mnt/user/Movies --dry-run +# bash /mnt/user/appdata/unraid_scripts/Tools/bulk_permissions_repair.sh /mnt/user/Movies +# bash /mnt/user/appdata/unraid_scripts/Tools/bulk_permissions_repair.sh /mnt/user/Movies /mnt/user/Music + +# container_data_export.sh — stop container, archive appdata, verify archive, restart +# Use before: major version updates with "no rollback" migrations, pool migrations, +# container removal, risky config changes that need a clean restore point. +# Sequence: space check → stop container → tar.gz archive → verify integrity → restart. +# Container is always restarted even if archiving fails — never leaves container stopped. +# Syntax: container_data_export.sh ContainerName /path/to/appdata /output/directory +# +# bash /mnt/user/appdata/unraid_scripts/Tools/container_data_export.sh \ +# Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/ --dry-run +# bash /mnt/user/appdata/unraid_scripts/Tools/container_data_export.sh \ +# Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/ + +# emby_database_repair.sh — SQLite integrity check on all Emby databases +# Use when: Emby crashes repeatedly with database errors, after power loss with Emby running. +# Stops Emby. Runs PRAGMA integrity_check on every .db file. Restarts Emby. Reports per database. +# Recovery guidance per database (this script does NOT auto-delete — human decision required): +# library.db → CORRUPT: safe to delete (Emby fully rebuilds from media files) +# users.db → CORRUPT: deleting resets ALL accounts + watch history (check backup) +# authentication.db → CORRUPT: safe to delete (API keys regenerated on restart) +# activity.db → CORRUPT: safe to delete (it's a log only) +# Config path detected automatically from Docker volume mounts — no configuration needed. +# +# bash /mnt/user/appdata/unraid_scripts/Tools/emby_database_repair.sh --status +# bash /mnt/user/appdata/unraid_scripts/Tools/emby_database_repair.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Tools/emby_database_repair.sh + +# zfs_pool_scrub.sh — trigger ZFS scrub on all pools, wait for completion +# ZFS checksums every block and verifies — catches silent data corruption before you +# discover it on reads. Safe to run while system is active (low I/O priority). +# Pools in ZFS_REPORT_IGNORE_POOLS excluded from default all-pool scrub. +# Specify a pool name to scrub it specifically (bypasses the ignore list). +# Notifies when complete with error summary. +# +# bash /mnt/user/appdata/unraid_scripts/Tools/zfs_pool_scrub.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Tools/zfs_pool_scrub.sh +# bash /mnt/user/appdata/unraid_scripts/Tools/zfs_pool_scrub.sh poolname + +# recreate_shares.sh — create share directories on disks after fresh install or rebuild +# Use when: share .cfg files restored to fresh install but directories missing on disks. +# Reads /boot/config/shares/*.cfg, parses shareInclude list, creates /mnt/diskN/sharename. +# chown nobody:users on creation. Run once — then initial rsync populates the content. +# Without this: rsync aborts "remote share missing" even though the share shows in the UI. +# +# bash /mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh --status +# bash /mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh + + +# ============================================================================================== +# ██ QUICK REFERENCE — COMPLETE SCHEDULE █████████████████████████████████████████████████████ +# ============================================================================================== +# +# At Startup of Array: +# Orchestrators/array_start.sh (single entry — handles everything) +# +# */3 * * * * every 3 minutes: +# Orchestrators/transcode_management.sh +# +# */10 * * * * every 10 minutes: +# unRAID_Essentials/webgui_restart.sh +# +# */15 * * * * every 15 minutes: +# Orchestrators/critical_sync_maintenance.sh +# +# */30 * * * * every 30 minutes: +# Rsync/rsync.sh /mnt/user/Media_Server/Emby --profile=emby-failover +# +# 0 */6 * * * every 6 hours: +# Orchestrators/arrs_failed_stalled_recovery.sh +# Monitors/system_tuning_monitor.sh +# +# 0 1 * * * 1am daily: +# Orchestrators/daily_sync_maintenance.sh +# +# 0 5 * * 0 Sunday 5am: +# unRAID_Essentials/clear_logs.sh +# +# 0 6 * * 0 Sunday 6am: +# Monitors/zfs_memory_snapshot.sh +# +# 0 7 * * 0 Sunday 7am: +# Orchestrators/sunday_morning_coffee_report.sh +# Monitors/smart_health.sh +# +# 0 8 * * * 8am daily: +# Monitors/weekly_health_digest.sh +# +# 0 9 * * 0 Sunday 9am: +# Monitors/cert_monitor.sh +# +# 0 10 * * 0 Sunday 10am: +# Monitors/backup_verify.sh +# +# 0 11 * * 0 Sunday 11am: +# Monitors/emby_session_report.sh +# Monitors/bandwidth_monitor.sh --report +# +# 30 2 * * 0 Sunday 2:30am: +# Orchestrators/weekly_sync_maintenance.sh # # ============================================================================================== \ No newline at end of file