From e13f2fa14fea4fcb8c0914423f711fadbfe10803 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Tue, 19 May 2026 20:00:10 -0400 Subject: [PATCH] feat: slskd reconnect guard in downloaders_reset, mass v2 sync - downloaders_reset: connection check block before slskd API sections; triggers PUT /api/v0/server reconnect if disconnected, polls 60s, gates Stuck Searches and Dead Transfer Records on SLSKD_CONNECTED - Sync all modified/new/deleted files from v2 refactor across Docker_Essentials, Media, Monitors, Partnership, Rsync, Tools, Transcodes, unRAID_Essentials, common.sh, master confs, and new Manual/README docs --- Docker_Essentials/Manual-Docker_Essentials.md | 546 ++++++++ Docker_Essentials/README-Docker_Essentials.md | 1026 +-------------- Docker_Essentials/docker_container_stop.sh | 131 +- Docker_Essentials/docker_daily_restart.sh | 124 +- Docker_Essentials/docker_network_connect.sh | 124 +- Docker_Essentials/docker_update.sh | 142 +- Docker_Essentials/docker_update_remaining.sh | 90 +- Docker_Essentials/docker_watchdog.sh | 244 +++- Docker_Essentials/docker_weekly_restart.sh | 101 +- Docker_Essentials/downloaders_reset.sh | 54 +- Fallback/Manual-Fallback.md | 613 +++++++++ Fallback/README-Failover.md | 1165 ----------------- Fallback/README-Fallback.md | 275 ++++ Fallback/fallback_test.sh | 131 +- Initial_run/partnership_onboard.sh | 181 --- Kernel/README-Kernel.md | 91 ++ Kernel/decision_engine.sh | 190 +-- Media/Manual-Media.md | 636 +++++++++ Media/README-Media.md | 1093 +++------------- Media/arr_sync.sh | 146 ++- Media/arrs_failed_stalled_recovery.sh | 84 +- Media/lidarr_cleanup.sh | 129 +- Media/lidarr_missing_art.sh | 85 +- Media/media_cleaner.sh | 50 +- Media/media_shares_permissions.sh | 68 +- Media/playback_aware_lidarr_discovery.sh | 25 +- Media/radarr_cleanup.sh | 112 +- Media/radarr_tmdb_removed.sh | 56 +- Media/sonarr_cleanup.sh | 112 +- Media/sonarr_tvdb_removed.sh | 54 +- Monitors/Manual-Monitors.md | 532 ++++++++ Monitors/README-Monitors.md | 1016 ++------------ Monitors/backup_verify.sh | 126 +- Monitors/bandwidth_monitor.sh | 107 +- Monitors/cert_monitor.sh | 117 +- Monitors/emby_session_report.sh | 113 +- Monitors/smart_health.sh | 104 +- Monitors/system_tuning_monitor.sh | 129 +- Monitors/weekly_health_digest.sh | 138 +- Monitors/zfs_memory_snapshot.sh | 139 +- Partnership/Manual-Partnership.md | 623 +++++++++ Partnership/README-Partnership.md | 1025 ++------------- Partnership/partnership_manager.sh | 606 ++++----- Partnership/partnership_offboard.sh | 678 ++++++++++ Partnership/partnership_onboard.sh | 640 +++++++++ {Initial_run => Partnership}/ssh_setup.sh | 12 +- Rsync/Manual-Rsync.md | 721 ++++++++++ Rsync/README-Rsync.md | 133 ++ Rsync/README-Rsync_Setup.md | 993 -------------- Rsync/rsync.sh | 167 ++- Tools/Manual-Tools.md | 600 +++++++++ Tools/README-Tools.md | 923 ++----------- Tools/bulk_permissions_repair.sh | 105 +- Tools/claude_startup.sh | 35 +- Tools/container_data_export.sh | 89 +- Tools/emby_database_repair.sh | 92 +- Tools/fallback_state_reset.sh | 84 +- Tools/recreate_shares.sh | 93 +- Tools/watchdog_skip_list_manager.sh | 94 +- Tools/zfs_pool_scrub.sh | 101 +- Transcodes/Manual-Transcoding.md | 460 +++++++ Transcodes/README-Transcoding.md | 797 ++--------- Transcodes/ramdisk_setup.sh | 138 +- Transcodes/transcode_cleanup.sh | 124 +- Transcodes/transcode_manager.sh | 143 +- common.sh | 4 + master_host1.conf | 26 + master_host2.conf | 22 + unRAID_Essentials/Manual-Unraid_Essentials.md | 768 +++++++++++ unRAID_Essentials/README-Unraid_Essentials.md | 937 +++---------- unRAID_Essentials/clear_logs.sh | 102 +- unRAID_Essentials/docker_syslog_filter.sh | 99 +- unRAID_Essentials/inotify_tuning.sh | 122 +- unRAID_Essentials/mover_stop.sh | 90 +- unRAID_Essentials/php_fpm_max_children.sh | 108 +- unRAID_Essentials/resource_watchdog.sh | 110 +- unRAID_Essentials/rsync_stop.sh | 108 +- unRAID_Essentials/server_reboot.sh | 111 +- unRAID_Essentials/system_watchdog.sh | 132 +- unRAID_Essentials/user_scripts_stop.sh | 89 +- unRAID_Essentials/webgui_restart.sh | 117 +- 81 files changed, 12164 insertions(+), 10656 deletions(-) create mode 100644 Docker_Essentials/Manual-Docker_Essentials.md create mode 100644 Fallback/Manual-Fallback.md delete mode 100644 Fallback/README-Failover.md create mode 100644 Fallback/README-Fallback.md delete mode 100755 Initial_run/partnership_onboard.sh create mode 100644 Kernel/README-Kernel.md create mode 100644 Media/Manual-Media.md create mode 100644 Monitors/Manual-Monitors.md create mode 100644 Partnership/Manual-Partnership.md create mode 100755 Partnership/partnership_offboard.sh create mode 100755 Partnership/partnership_onboard.sh rename {Initial_run => Partnership}/ssh_setup.sh (97%) create mode 100644 Rsync/Manual-Rsync.md create mode 100644 Rsync/README-Rsync.md delete mode 100644 Rsync/README-Rsync_Setup.md create mode 100644 Tools/Manual-Tools.md create mode 100644 Transcodes/Manual-Transcoding.md create mode 100644 unRAID_Essentials/Manual-Unraid_Essentials.md diff --git a/Docker_Essentials/Manual-Docker_Essentials.md b/Docker_Essentials/Manual-Docker_Essentials.md new file mode 100644 index 0000000..d6448d0 --- /dev/null +++ b/Docker_Essentials/Manual-Docker_Essentials.md @@ -0,0 +1,546 @@ +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# 🐳 DOCKER ESSENTIALS — Manual +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Configuration reference, setup procedures, and operational workflows. +For folder overview and design philosophy see `README-Docker_Essentials.md`. +For per-script detail see the script headers directly. + +--- + +## ━━━ WATCHDOG CONFIGURATION ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +All watchdog configuration lives in `master_host*.conf` (per-container lists) and +`master.conf` (shared thresholds and toggles). `detect_hosts()` aliases all +`HOST1_` / `HOST2_` prefixed vars to their unprefixed names at runtime — scripts +always read the right values for the server they're running on. + +--- + +### ── Memory Hard Limits ──────────────────────────────────────────────────────── + +```bash +# master_host1.conf +# Format: "ContainerName:LimitInMB" +# Immediate restart when exceeded — no strike system. Memory leaks are not spikes. +# +# Sizing guidance: +# Check normal peak: docker stats ContainerName +# Set limit at ~150-200% of normal peak +# Emby peaks ~12GB under heavy transcode load → 18GB gives headroom +# without triggering on legitimate load spikes +# +HOST1_WATCHDOG_CONTAINERS=( + "Emby:18432" # 18GB — peaks ~12GB under heavy transcode load + "LidaTube:6144" # 6GB — YouTube downloader, grows with large queues + "Tdarr:6144" # 6GB — video transcoder, memory-intensive by nature + "Code-Server:1024" # 1GB — IDE, should be light; 1GB is generous +) +``` + +A soft warning fires at `SOFT_MEM_THRESHOLD=80` percent of the hard limit — early +visibility into a container approaching its ceiling before a restart is triggered. + +--- + +### ── CPU Thresholds ──────────────────────────────────────────────────────────── + +```bash +# master.conf +# CPU is normalised against total core count — meaningful regardless of hardware. +# +# Why normalised: +# 85% on one core of a 16-core machine = 5.3% normalised → ignore it +# 85% normalised on a 16-core machine = 13.6 cores worth → runaway process +# +# CPU uses a STRIKE SYSTEM — not immediate restart like memory. +# Brief spikes are normal (Tdarr encoding, Emby transcoding, SABnzbd unpacking). +# The strike system ignores spikes and acts on sustained high usage. +# +# Strike 1: above HARD_CPU_THRESHOLD this cycle → warn, increment strike +# Strike 2: above threshold next cycle → restart, reset counter +# Recovery: drops below threshold any cycle → reset counter to 0 +# +SOFT_CPU_THRESHOLD=50 # warn at 50% normalised — informational only +HARD_CPU_THRESHOLD=85 # strike at 85% normalised +CPU_FAIL_LIMIT=2 # consecutive strikes before restart +``` + +--- + +### ── HTTP Health Checks ──────────────────────────────────────────────────────── + +```bash +# master_host1.conf +# Format: "ContainerName:http://host:port/optional-path" +# Hits the actual service endpoint on every watchdog cycle. +# "Container running" and "service responding" are not the same thing. +# +# Uses a STRIKE SYSTEM — network hiccups and brief restarts happen. +# Two consecutive non-responses before acting prevents false positives. +# +# The path can be a lightweight health endpoint or the root URL. +# Docker's own HEALTHCHECK requires the image to define it — most don't. +# These checks work regardless of what the image defines. +# +HOST1_WATCHDOG_CONTAINER_URLS=( + "Emby:http://localhost:8096" # Emby WebUI root — fast to respond + "NginxProxyManager:http://localhost:81" # NPM admin interface +) + +# master.conf +CURL_TIMEOUT=5 # seconds before non-response counts as a failure +RESP_FAIL_LIMIT=2 # consecutive failures before restart +``` + +--- + +### ── Required Containers ─────────────────────────────────────────────────────── + +```bash +# master_host1.conf +# Containers that must always be running. +# Found stopped → watchdog attempts restart every cycle until running or skip-listed. +# Uses the STRIKE SYSTEM — one miss might be mid-restart. +# Persistent failure → skip list → critical notification. +# +# These are the containers whose absence breaks everything else: +# +HOST1_WATCHDOG_REQUIRED_CONTAINERS=( + "NginxProxyManager" # reverse proxy — all external traffic routes through this + "Authelia" # SSO authentication — all protected services need it + "Mariadb-Authelia" # Authelia database — must be up before Authelia starts + "Redis-Authelia" # Authelia session cache — same startup dependency +) +``` + +--- + +### ── Dependency Ordering ─────────────────────────────────────────────────────── + +```bash +# master_host1.conf +# Format: "DependentContainer:dependency1 dependency2" +# Multiple dependencies space-separated. All must be running before dependent restarts. +# +# When a container and its dependency are both down: +# → restart the dependency first +# → skip the dependent this cycle +# → next cycle: dependency healthy → dependent restarts cleanly +# +# Without this: Authelia starts, can't connect to MariaDB (still starting), +# exits immediately, strike 1. Next cycle: same, strike 2. Skip list. +# MariaDB was fine the whole time. +# +HOST1_WATCHDOG_DEPENDENCIES=( + "Authelia:Mariadb-Authelia Redis-Authelia" + "Authelia-Secondary:Mariadb-Authelia Redis-Authelia-Secondary" + "NextCloud:Postgres-NextCloud" +) +``` + +The same dependency configuration is used by `docker_daily_restart.sh` and +`docker_weekly_restart.sh` — configure once, applies everywhere. + +--- + +### ── Startup Grace Period ────────────────────────────────────────────────────── + +```bash +# master.conf +# Suppress restart actions for N seconds after array start. +# Checks still run and log — only restart actions are suppressed. +# Clock starts from when the watchdog process itself starts. +# +# Without this: false-positive restarts fire in the first minutes after +# every array start while containers are still initialising. +# +WATCHDOG_STARTUP_GRACE=600 # 10 minutes +``` + +--- + +### ── Tier 2 Global Scan ──────────────────────────────────────────────────────── + +Tier 2 scans every running container when `WATCHDOG_SCAN_ALL=true`. No per-container +configuration required — it's the catch-all for everything not explicitly in Tier 1. + +```bash +# master.conf +WATCHDOG_SCAN_ALL=true # enable global scan +WATCHDOG_RESTART_UNHEALTHY=true # act on Docker HEALTHCHECK failures +WATCHDOG_NOTIFY_OOM=true # detect and notify kernel OOM kills +WATCHDOG_NOTIFY_CRASHLOOP=true # detect escalating restart counts +WATCHDOG_RESTART_DEAD=true # recover containers in dead state +WATCHDOG_RESTART_CRASHED=true # restart containers that exited non-zero +WATCHDOG_CRASH_LIMIT=5 # RestartCount above this → restart + skip list + +# Containers excluded from Tier 2 entirely. +# Use for containers you intentionally stop/start manually, or containers that +# have benign non-zero exits as part of their normal operation. +WATCHDOG_SCAN_IGNORE=( + "my-one-shot-container" # runs and exits normally — not a crash +) +``` + +Each toggle is independent — disable any check that produces false positives in your +environment without affecting the others. + +--- + +### ── Restart Loop Protection ────────────────────────────────────────────────── + +```bash +# master.conf +# N restarts within a rolling window → skip list + critical notification. +# The watchdog stops touching the container entirely. +# Skip list lives on /boot/config/ — survives reboots intentionally. +# A container bad enough to be skip-listed is still broken after a reboot. +# +WATCHDOG_CONTAINER_RESTART_LIMIT=3 # restarts in the window before skip list +WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours +WATCHDOG_CONTAINER_RESTART_LOG="/boot/config/container_restart_history.db" +SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" +``` + +**Auto-clear:** The watchdog checks the skip list every cycle and removes any container +it finds running. If the container recovers on its own, monitoring resumes automatically. +Manual clear is only needed when the container is stuck stopped. + +--- + +### ── Notification Batching ──────────────────────────────────────────────────── + +```bash +# master.conf +# All events from one cycle collected → one notification at end of cycle. +# +# Why: a shared database going down can cascade 10+ containers failing +# simultaneously. Without batching: 10 individual pings. With batching: +# one summary listing all affected containers. Actionable vs overwhelming. +# +WATCHDOG_BATCH_NOTIFY=true +``` + +--- + +### ── State Files Reference ──────────────────────────────────────────────────── + +| File | Config Var | Location | Resets | Purpose | +|------|-----------|----------|--------|---------| +| Strike counts | `WATCHDOG_STATE_FILE` | `/tmp/` | On reboot | Per-container CPU/HTTP strike counters | +| Skip list | `SYS_WATCHDOG_FAILED_FILE` | `/boot/config/` | Never (manual / auto-clear) | Containers that exhausted restart attempts | +| Restart history | `WATCHDOG_CONTAINER_RESTART_LOG` | `/boot/config/` | Auto-purge after window | Restart loop detection data | +| Shared state | `SYS_WATCHDOG_STATE_FILE` | `/tmp/` | On reboot | RAM emergency flag + cycle heartbeat from `system_watchdog.sh` | + +`/tmp/` resets on reboot — correct, strike counts before a reboot are meaningless after it. +`/boot/config/` survives reboots — correct, a skip-listed container is still broken after a reboot. + +--- + +## ━━━ RESTART SCHEDULE CONFIGURATION ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +--- + +### ── Daily Restart List ─────────────────────────────────────────────────────── + +```bash +# master_host1.conf +# Restarted every night at 1am via daily_sync_maintenance.sh. +# +# Good candidates: +# Reverse proxies — connection table fills slowly over weeks +# Authentication services — session cache benefits from periodic clearing +# Live TV schedulers — accumulated scheduling state slows decisions +# Download managers — connection pool maintenance +# +HOST1_DAILY_RESTART_CONTAINERS=( + "NginxProxyManager" # connection table fills slowly over weeks + "Authelia" # session cache benefits from periodic clearing + "Dispatcharr" # Live TV scheduler accumulates state + "Dispatcharr-Basic" # secondary Live TV scheduler — same reason + "ErsatzTV-Emby" # channel schedule builder, stale entries accumulate +) +``` + +This list also drives `docker_update.sh` in normal mode — containers added here get +their images updated daily before the restart. Add a container once, it gets both. + +--- + +### ── Weekly Restart List ────────────────────────────────────────────────────── + +```bash +# master_host1.conf +# Restarted every Sunday at 2:30am via weekly_sync_maintenance.sh. +# Runs AFTER the sync window's own restart of critical containers (Emby, auth stack). +# +# Daily vs Weekly decision: +# Daily: connection-heavy infrastructure — degrades faster (proxy, auth, Live TV) +# Weekly: productivity and media services — degrades slowly (NextCloud, AdGuard, Immich) +# +HOST1_WEEKLY_RESTART_CONTAINERS=( + "NextCloud" # file sync — benefits from clean weekly start + "AdGuard-Home" # DNS — cache and stat accumulation + "Immich" # photo library — index/cache maintenance +) +``` + +--- + +## ━━━ NETWORK CONFIGURATION ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +```bash +# master_host1.conf +# Networks to ensure exist + containers to connect to each network. +# Many-to-many: every container connects to every network listed. +# +# Containers do not need to be running — script handles missing containers +# gracefully (warns + skips). They connect on the next array start. +# +HOST1_NETWORK_CONNECT_NETWORKS=( + "high-availability" # main internal network — most containers should be on this +) + +HOST1_NETWORK_CONNECT_CONTAINERS=( + "memcached" # NextCloud's cache — needs to reach NextCloud AIO network + "Npm-CrowdSec" # CrowdSec bouncer — needs to reach NPM's network +) +``` + +> **Timing dependency:** Networks created by Docker Compose stacks (e.g. NextCloud AIO) +> only exist after those stacks start. If this script runs before the Compose stack, +> the network won't exist yet and the connection fails this run. It will succeed on the +> next array start. Schedule Compose stacks early in `ARRAY_START_SCRIPTS` order to +> minimise the window. + +--- + +## ━━━ CONTAINER UPDATE CONFIGURATION ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +```bash +# master.conf +DAILY_CONTAINER_UPDATES=true # enable/disable daily image pull + # docker_daily_restart.sh still runs regardless + # update and restart are independent + +WEEKLY_REMAINING_UPDATES=true # enable/disable weekly remainder pull + prune + # to disable: set false or remove from WEEKLY_MAINTENANCE_SCRIPTS +``` + +`docker_update.sh` in normal mode targets `DAILY_RESTART_CONTAINERS` — the same list +used by `docker_daily_restart.sh`. No second list to maintain. + +`docker_update_remaining.sh` derives its target list automatically: +all running containers minus `DAILY_RESTART_CONTAINERS` minus `WEEKLY_RESTART_CONTAINERS`. +Everything gets updated at least once per week with no explicit configuration. + +--- + +## ━━━ FULL CONFIGURATION REFERENCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +### ── master_host*.conf ──────────────────────────────────────────────────────── + +```bash +# Per-host — varies between HOST1 and HOST2 + +# Tier 1 — memory limits ("ContainerName:LimitInMB") +HOST1_WATCHDOG_CONTAINERS=() + +# Tier 1 — HTTP health check endpoints ("ContainerName:http://host:port") +HOST1_WATCHDOG_CONTAINER_URLS=() + +# Tier 1 — must always be running +HOST1_WATCHDOG_REQUIRED_CONTAINERS=() + +# Tier 1 + 2 — dependency ordering ("Dependent:dep1 dep2") +HOST1_WATCHDOG_DEPENDENCIES=() + +# Daily restart list (also drives docker_update.sh normal mode) +HOST1_DAILY_RESTART_CONTAINERS=() + +# Weekly restart list +HOST1_WEEKLY_RESTART_CONTAINERS=() + +# Networks to ensure exist +HOST1_NETWORK_CONNECT_NETWORKS=() + +# Containers to connect to every configured network +HOST1_NETWORK_CONNECT_CONTAINERS=() +``` + +--- + +### ── master.conf ────────────────────────────────────────────────────────────── + +```bash +# Shared — applies to both servers + +# ── Watchdog timing ──────────────────────────────────────────────────────── +DOCKER_WATCHDOG_INTERVAL=900 # seconds between cycles (15 minutes) +WATCHDOG_STARTUP_GRACE=600 # seconds before restarts begin after boot +CONTAINER_DELAY=15 # seconds between dependency + dependent restart + +# ── Memory ───────────────────────────────────────────────────────────────── +SOFT_MEM_THRESHOLD=80 # warn at % of hard limit (no restart) + +# ── CPU ──────────────────────────────────────────────────────────────────── +SOFT_CPU_THRESHOLD=50 # warn threshold — normalised % +HARD_CPU_THRESHOLD=85 # strike threshold — normalised % +CPU_FAIL_LIMIT=2 # consecutive strikes before restart + +# ── HTTP health check ────────────────────────────────────────────────────── +CURL_TIMEOUT=5 # seconds before curl times out +RESP_FAIL_LIMIT=2 # consecutive failures before restart + +# ── Restart loop protection ──────────────────────────────────────────────── +WATCHDOG_CONTAINER_RESTART_LIMIT=3 # restarts before skip list +WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours + +# ── Tier 2 global scan ───────────────────────────────────────────────────── +WATCHDOG_SCAN_ALL=true +WATCHDOG_SCAN_IGNORE=() +WATCHDOG_RESTART_UNHEALTHY=true +WATCHDOG_NOTIFY_OOM=true +WATCHDOG_NOTIFY_CRASHLOOP=true +WATCHDOG_CRASH_LIMIT=5 +WATCHDOG_RESTART_DEAD=true +WATCHDOG_RESTART_CRASHED=true + +# ── Notifications ────────────────────────────────────────────────────────── +WATCHDOG_BATCH_NOTIFY=true + +# ── Container updates ────────────────────────────────────────────────────── +DAILY_CONTAINER_UPDATES=true +WEEKLY_REMAINING_UPDATES=true + +# ── Retry behaviour (shared by restart scripts) ──────────────────────────── +RETRY_COUNT=3 # retry attempts before marking failed +SLEEP=5 # seconds between retry attempts +``` + +--- + +## ━━━ PROCEDURES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +### ── Adding a Container to Monitoring ──────────────────────────────────────── + +Adding a container is purely additive — add the relevant lines to `master_host*.conf`. +No script changes. `detect_hosts()` picks up the new config on the next watchdog cycle. + +```bash +# master_host1.conf — example: adding "MyApp" to full Tier 1 + daily restarts + +# 1. Memory hard limit — size at ~150-200% of normal peak (check: docker stats MyApp) +HOST1_WATCHDOG_CONTAINERS=( + ...existing... + "MyApp:2048" # 2GB ceiling +) + +# 2. HTTP health check +HOST1_WATCHDOG_CONTAINER_URLS=( + ...existing... + "MyApp:http://localhost:8080/health" # or root URL if no /health endpoint +) + +# 3. Required — add only if absence breaks other services +HOST1_WATCHDOG_REQUIRED_CONTAINERS=( + ...existing... + "MyApp" +) + +# 4. Dependency — add if MyApp needs another container up first +HOST1_WATCHDOG_DEPENDENCIES=( + ...existing... + "MyApp:MyApp-Database" +) + +# 5. Daily restart — add if MyApp degrades over time +HOST1_DAILY_RESTART_CONTAINERS=( + ...existing... + "MyApp" # also adds it to the daily image update +) +``` + +Tier 2 picks up MyApp automatically — no configuration needed. It will be included in +the global health scan from the next cycle onward. + +To **exclude** MyApp from Tier 2 (e.g. it's a one-shot container that exits normally): +```bash +# master.conf +WATCHDOG_SCAN_IGNORE=( + "MyApp" # one-shot — exits cleanly, don't treat as crash +) +``` + +--- + +### ── Skip List Recovery ──────────────────────────────────────────────────────── + +Used when `docker_watchdog.sh` has skip-listed a container after exhausting restart +attempts. The watchdog stops touching it and sends a critical notification. Human +intervention required. + +```bash +# Step 1 — understand the situation +# Shows skip list contents, container states, restart history +watchdog_skip_list_manager.sh --status + +# Step 2 — fix the underlying problem first +# Check logs: docker logs ContainerName --tail 100 +# Check disk: df -h /mnt/user +# Check database: docker exec ContainerName sqlite3 /path/to.db ".tables" +# Fix before clearing — clearing without fixing just resets the counter + +# Step 3 — clear the container from the skip list + restart history +# Clearing history is important: the counter carries over otherwise and +# the container hits the limit again almost immediately on any startup trouble +watchdog_skip_list_manager.sh --clear ContainerName + +# Step 4 — start the container manually +# Confirms your fix worked before handing back to the watchdog +docker start ContainerName + +# Step 5 — monitoring resumes automatically +# Next watchdog cycle: container seen running → removed from skip list +# Restart history clean. Back to normal. +``` + +> ⚠️ **If `docker_watchdog.sh` is currently running when you clear the skip list**, it +> may re-add the container on its very next cycle if the container is still in a bad +> state. The script detects this and warns you. Fix the root cause **before** clearing. + +Other skip list actions: +```bash +watchdog_skip_list_manager.sh # status (default) +watchdog_skip_list_manager.sh --clear-all # clear everything +watchdog_skip_list_manager.sh --clear-all --force # non-interactive +watchdog_skip_list_manager.sh --clear-all --dry-run # preview what would clear +``` + +--- + +## ━━━ FLAG REFERENCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +All scripts support these standard flags: + +| Flag | What it does | +|------|-------------| +| `--dry-run` | Preview actions without making changes. Shows exactly what would happen. | +| `--status` | Show current config, container states, and relevant runtime info, then exit. | +| `--log` | Verbose output — full detail for every container checked and every decision made. | + +### `docker_watchdog.sh --status` shows: +Strike counts for all monitored containers, current skip list contents, whether grace +period is active and how long remains, whether RAM emergency deferral is active, last +cycle timing. + +### `docker_watchdog.sh --dry-run` shows: +A full watchdog cycle without restarting anything. Shows what the watchdog would do +based on current container states. Useful for verifying configuration before enabling +automatic restarts. + +### `docker_update.sh --remainder` +Switches to remainder mode — updates all running containers not in the managed daily/weekly +lists. Called by `weekly_sync_maintenance.sh`. Can be run manually to sweep containers +that haven't been updated recently. diff --git a/Docker_Essentials/README-Docker_Essentials.md b/Docker_Essentials/README-Docker_Essentials.md index cd3a895..2abee83 100644 --- a/Docker_Essentials/README-Docker_Essentials.md +++ b/Docker_Essentials/README-Docker_Essentials.md @@ -5,7 +5,8 @@ **Self-healing container lifecycle management for a 50+ container unRAID stack.** Health monitoring that catches problems as they happen. Scheduled restarts that prevent degradation before it becomes visible. Network configuration that survives reboots and -unRAID updates. Recovery tooling for when something genuinely breaks. +unRAID updates. Container image updates woven into the maintenance windows. Recovery +tooling for when something genuinely breaks. > **Why this folder exists:** Docker on unRAID does not heal itself. A container that > crashes stays crashed. A memory leak accumulates silently for days until the system @@ -141,8 +142,8 @@ is mid-attempt right now, or hasn't noticed yet. You have to manually check the list file on `/boot/config/`, check the restart history file, check the state file — none of which have obvious formats. -The fix: `watchdog_skip_list_manager.sh`. One command to see exactly what's on the -skip list, which containers are stopped vs running, how many restarts were attempted, +The fix: `watchdog_skip_list_manager.sh` (in `Tools/`). One command to see exactly what's +on the skip list, which containers are stopped vs running, how many restarts were attempted, and what the watchdog's current state is. One command to clear a specific container and its history after you've fixed the problem. No manual file editing required. @@ -157,7 +158,7 @@ Monitors/ ← observes, measures, reports Rsync/ ← moves data between servers ``` -Four distinct roles, each handled by dedicated scripts: +Five distinct responsibilities, each handled by dedicated scripts: --- @@ -185,6 +186,15 @@ windows so any downtime from restarts is absorbed by the window that's already h --- +### 🔄 Image Currency — `docker_update.sh` + `docker_update_remaining.sh` + +Keeps all container images current without manual intervention. Daily updates for the +auth/proxy stack (the containers that restart daily anyway — no extra downtime). Weekly +remainder pass for everything else — derives the target list automatically from `docker ps` +minus what was already updated, so there is no second list to maintain. + +--- + ### 🌐 Network Integrity — `docker_network_connect.sh` Ensures custom Docker networks exist and containers are connected to them at every @@ -193,11 +203,12 @@ array start. Silent when everything is correct. Notifies when it has to create s --- -### 🔧 Recovery Tooling — `watchdog_skip_list_manager.sh` +### 🧹 Downloader Hygiene — `downloaders_reset.sh` -Manual tool for the moments when automatic recovery hasn't worked. Gives a clear picture -of what the watchdog has already tried, lets you clear the skip list cleanly after fixing -the root cause, and warns you if clearing might immediately re-add the container. +Maintenance reset for all download clients (slskd, SABnzbd, qBittorrent) every 15 +minutes. Clears stuck searches, dead transfers, failed imports, and stale queue entries +that download clients accumulate but never clean up themselves. Never touches active +or in-progress downloads. --- @@ -232,980 +243,57 @@ indefinitely by a stale flag from a process that's no longer running. --- -## ━━━ SCRIPTS AT A GLANCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ━━━ SCRIPTS IN THIS FOLDER ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ | Script | Role | When It Runs | |--------|------|-------------| -| `docker_watchdog.sh` | Two-tier self-healing container monitor | Continuous background loop | +| `docker_watchdog.sh` | Two-tier self-healing container monitor | Continuous background loop — array start | | `docker_daily_restart.sh` | Nightly proactive restart of degradation-prone containers | 1am via `daily_sync_maintenance.sh` | | `docker_weekly_restart.sh` | Weekly restart of less-critical services | 2:30am Sunday via `weekly_sync_maintenance.sh` | +| `docker_update.sh` | Container image updates — daily list + weekly remainder mode | Daily before restart; weekly end of window | +| `docker_update_remaining.sh` | Image update + prune for all containers not in managed lists | End of weekly maintenance window | | `docker_network_connect.sh` | Network existence + container connection enforcement | Every array start | -| `watchdog_skip_list_manager.sh` | Skip list inspection and manual recovery | On demand | +| `docker_container_stop.sh` | Ordered container shutdown — graceful then forced | Called by `array_stopping.sh` | +| `downloaders_reset.sh` | Download client hygiene — slskd / SABnzbd / qBittorrent | Every 15min via `critical_sync_maintenance.sh` | + +> `watchdog_skip_list_manager.sh` — manual recovery tool for the skip list. Lives in `Tools/` because it's an operator utility, not a lifecycle script. --- -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🐳 docker_watchdog.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -The self-healing heart of the ecosystem. Two tiers of monitoring run every 15 minutes -as a background process. Tier 1 applies specific thresholds to explicitly configured -containers. Tier 2 scans everything else for generic health problems. Together they -catch the full range of container failures — from subtle memory leaks to outright crashes. - -```bash -# Started automatically at array start via array_start.sh -# Runs continuously until array stops (SIGTERM → clean shutdown) -# Interval: DOCKER_WATCHDOG_INTERVAL=900 (15 minutes) -``` - ---- - -### ── Tier 1 — Strict Per-Container Monitoring ──────────────────────────────── - -Applies **only** to containers you explicitly configure. These are the containers -you care most about — the ones that affect users when they fail. Configure them -once in `master_host*.conf` and they are monitored with specific, appropriate -thresholds forever. - ---- - -#### 💾 Memory Hard Limits - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# Memory hard limits in MB — immediate restart when exceeded, no strike system. -# Each entry: "ContainerName:LimitInMB" -# -# These are NOT soft targets — they are hard ceilings. A container that hits -# its limit gets restarted immediately on the current cycle. Memory leaks are -# not brief spikes. Waiting for a second confirmation just allows more leak. -# -# How to size limits: -# Check normal usage: docker stats ContainerName -# Set limit at ~150-200% of normal peak usage -# Emby peaks around 8-12GB during heavy transcoding — 18GB gives headroom -# without triggering on legitimate load spikes -# -HOST1_WATCHDOG_CONTAINERS=( - "Emby:18432" # 18GB — peaks ~12GB under heavy transcode load - "LidaTube:6144" # 6GB — YouTube downloader, can grow with large queues - "Tdarr:6144" # 6GB — video transcoder, memory-intensive by nature - "Code-Server:1024" # 1GB — IDE, should be light; 1GB is generous -) -``` - -A soft warning threshold fires at `SOFT_MEM_THRESHOLD=80` percent of the hard limit — -giving early visibility into a container approaching its ceiling before a restart is -triggered. Useful for catching gradual leaks before they become events. - ---- - -#### 📊 CPU Thresholds - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# CPU thresholds are normalised against the server's total core count. -# This makes the threshold meaningful regardless of hardware. -# -# Why normalised: -# A container using 85% of one core on a 16-core machine = 5.3% normalised -# → a nothing, don't touch it -# A container using 85% normalised on a 16-core machine = 13.6 cores worth -# → a runaway process, restart it -# -# CPU uses a STRIKE SYSTEM — not immediate restart like memory. -# Brief CPU spikes are completely normal (Tdarr encoding, Emby transcoding, -# SABnzbd unpacking). The strike system ignores spikes and acts on sustained usage. -# -# Strike 1: CPU above HARD_CPU_THRESHOLD this cycle → warn, increment strike -# Strike 2: CPU above threshold next cycle → restart, reset strike counter -# Recovery: CPU drops below threshold any cycle → reset strike counter to 0 -# - SOFT_CPU_THRESHOLD=50 # warn at 50% normalised — informational only - HARD_CPU_THRESHOLD=85 # strike at 85% normalised - CPU_FAIL_LIMIT=2 # consecutive strikes before restart -``` - ---- - -#### 🌐 HTTP Responsiveness - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# HTTP health checks hit the actual service endpoint on every watchdog cycle. -# "Container running" and "service responding" are not the same thing. -# -# A frozen Emby will respond to nothing but show as Up in Docker. -# A curl timeout catches this where Docker's own checks never would. -# -# Also uses a STRIKE SYSTEM — network hiccups and brief restarts happen. -# Two consecutive non-responses before acting prevents false positives -# from momentary connectivity issues. -# -# Format: "ContainerName:http://host:port/optional-path" -# The path can be a lightweight health endpoint or just the root URL -# -HOST1_WATCHDOG_CONTAINER_URLS=( - "Emby:http://localhost:8096" # Emby WebUI root — fast to respond - "NginxProxyManager:http://localhost:81" # NPM admin interface -) -# -# CURL_TIMEOUT=5 # seconds before a non-response counts as a failure -# RESP_FAIL_LIMIT=2 # consecutive failures before restart -``` - -> **Note on Docker HEALTHCHECK:** Docker has its own `HEALTHCHECK` mechanism but it -> requires the image to define a health check command — most self-hosted images don't. -> These HTTP checks work regardless of what the image defines. They check what actually -> matters: does the service respond to a request? - ---- - -#### ✅ Required Containers - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# Containers that must always be running. If found stopped, the watchdog -# attempts to restart them every cycle until they are running or hit the -# restart loop limit. -# -# These are the containers whose absence breaks everything else: -# NginxProxyManager — all external traffic routes through this -# Authelia — authentication for every protected service -# Mariadb-Authelia — Authelia's database — Authelia cannot start without it -# Redis-Authelia — Authelia's session store — same dependency -# -# Required containers use the STRIKE SYSTEM — one miss might be mid-restart. -# Persistent failure → skip list → critical notification. -# -HOST1_WATCHDOG_REQUIRED_CONTAINERS=( - "NginxProxyManager" # reverse proxy — external access depends on this - "Authelia" # SSO authentication — all protected services need it - "Mariadb-Authelia" # Authelia database — must be up before Authelia starts - "Redis-Authelia" # Authelia session cache — same startup dependency -) -``` - ---- - -### ── Tier 2 — Global Health Scan ───────────────────────────────────────────── - -Scans **every running container** when `WATCHDOG_SCAN_ALL=true`. No per-container -configuration required — this is the catch-all that protects everything not explicitly -configured in Tier 1. - -Containers in `WATCHDOG_SCAN_IGNORE` are excluded from Tier 2 but still covered by -Tier 1 if configured there. - ---- - -| Check | What Triggers It | What Happens | Why This Matters | -|-------|-----------------|--------------|-----------------| -| `WATCHDOG_RESTART_UNHEALTHY` | Docker HEALTHCHECK reports `unhealthy` | Restart | Catches containers with built-in health checks that are failing | -| `WATCHDOG_NOTIFY_OOM` | Kernel OOM-killed the container | Restart + notify | OOM kills are silent by default — you'd never know without this | -| `WATCHDOG_NOTIFY_CRASHLOOP` | Docker RestartCount climbing | Notify; above `WATCHDOG_CRASH_LIMIT` → restart → skip list | Distinguishes "just restarted once" from "has crashed 12 times" | -| `WATCHDOG_RESTART_DEAD` | Container in `dead` state | Remove + start | Dead containers can't be restarted — must be removed first | -| `WATCHDOG_RESTART_CRASHED` | Non-zero exit code, exited state | Restart | Catches clean-exit crashes that Docker's restart policy misses | - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# Tier 2 toggles — disable any check that produces false positives in your -# environment. Each is independent — disabling one doesn't affect the others. -# - WATCHDOG_SCAN_ALL=true # enable Tier 2 global scan - WATCHDOG_RESTART_UNHEALTHY=true # act on Docker HEALTHCHECK failures - WATCHDOG_NOTIFY_OOM=true # detect and notify kernel OOM kills - WATCHDOG_NOTIFY_CRASHLOOP=true # detect escalating restart counts - WATCHDOG_RESTART_DEAD=true # recover containers in dead state - WATCHDOG_RESTART_CRASHED=true # restart containers that exited non-zero - WATCHDOG_CRASH_LIMIT=5 # RestartCount above this → restart + skip list - -# Containers to exclude from Tier 2 global scan entirely. -# Use for containers you intentionally stop/start manually, or containers that -# have benign non-zero exits as part of their normal operation. - WATCHDOG_SCAN_IGNORE=( - "my-one-shot-container" # runs and exits normally — don't treat as crash - ) -``` - ---- - -### ── Cross-Cutting Intelligence ────────────────────────────────────────────── - -These mechanisms apply to **both tiers** on every watchdog cycle. They are what -separates intelligent monitoring from naive restart-on-failure. - ---- - -#### ⏱️ Startup Grace Period - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# After array start, containers take time to initialise — databases run recovery, -# services wait for dependencies, Emby scans its library. During this window, -# a container that isn't responding yet is not broken — it's just starting. -# -# Without a grace period, the watchdog fires false-positive restarts in the -# first minutes after every array start. With it, checks run and log normally -# but restart actions are suppressed until the grace period expires. -# -# The grace period clock starts from when the watchdog process itself starts — -# not from when the array starts — so it's accurate even if array start -# takes a few minutes. -# - WATCHDOG_STARTUP_GRACE=600 # 10 minutes — restarts suppressed, checks still log -``` - ---- - -#### 🔗 Dependency Ordering - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# When a container and its dependency are both down, restart the dependency first. -# Skip the dependent container entirely on this cycle. On the next cycle the -# dependency should be healthy, and the dependent can restart cleanly. -# -# Without this: Authelia starts, can't connect to MariaDB (still starting), -# exits immediately, strike 1. Next cycle: same thing, strike 2. Next cycle: -# restart, skip list, critical notification. MariaDB was fine the whole time. -# -# With this: MariaDB restarted first. Authelia skipped this cycle. -# Next cycle: MariaDB healthy → Authelia restarts cleanly. No false alarms. -# -# Format: "DependentContainer:dependency1 dependency2" -# Multiple dependencies space-separated. All must be running before dependent restarts. -# -HOST1_WATCHDOG_DEPENDENCIES=( - "Authelia:Mariadb-Authelia Redis-Authelia" # both db and cache must be up first - "Authelia-Secondary:Mariadb-Authelia Redis-Authelia-Secondary" - "NextCloud:Postgres-NextCloud" # NextCloud needs its postgres first -) -``` - ---- - -#### 🔒 Restart Loop Protection + Skip List - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# If the watchdog restarts the same container N times within a rolling time -# window and it keeps crashing, something is genuinely broken that restarts -# are not fixing. Continued restart attempts risk making it worse (database -# corruption from incomplete shutdowns, etc.). -# -# When a container hits the limit: -# 1. Added to the persistent skip list on /boot/config/ -# 2. Critical notification sent — this needs human attention -# 3. Watchdog stops touching it — completely hands off -# -# The skip list survives reboots — it lives on /boot/. This is intentional. -# If a container was in a bad enough state to be skip-listed, a reboot -# doesn't fix the underlying problem. It stays on the list until cleared. -# -# AUTO-CLEAR: The watchdog checks the skip list every cycle and removes any -# container it finds running. So if the container recovers on its own (e.g. -# Docker's own restart policy eventually succeeds after a longer backoff), -# the watchdog detects it running and resumes normal monitoring automatically. -# Manual clear only needed when the container is stuck stopped. -# - WATCHDOG_CONTAINER_RESTART_LIMIT=3 # restarts in the window before skip list - WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours - WATCHDOG_CONTAINER_RESTART_LOG="/boot/config/container_restart_history.db" - SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" -``` - ---- - -#### ⚡ Docker Daemon Health Check - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Every cycle begins with a daemon health check before any container operations. -# A hung Docker daemon makes every subsequent docker command hang — which would -# stall the entire watchdog indefinitely, leaving containers unmonitored. -# -# If the daemon doesn't respond within DOCKER_TIMEOUT seconds: -# 1. Attempt daemon restart via /etc/rc.d/rc.docker restart -# 2. Wait 15 seconds for recovery -# 3. Verify daemon is responding again -# 4. If recovered: log and continue the cycle normally -# 5. If still hung: log critical error, skip the rest of this cycle -# system_watchdog.sh will escalate from here (it has its own daemon check -# in Tier 1 Critical — bypass strikes, reboot if daemon stays down) -# -# DOCKER_TIMEOUT=10 # seconds — tight enough to detect hangs, not false positives -``` - ---- - -#### 📬 Notification Batching - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# All events from a single watchdog cycle are collected and sent as ONE -# notification at the end of the cycle. -# -# Why batching matters: -# On a 50+ container system, a single problem (e.g. shared database goes down) -# can cascade into 10+ dependent containers failing simultaneously. Without -# batching: 10 individual "container X failed" notifications arrive in rapid -# succession. With batching: one summary notification lists all affected -# containers. The first format is overwhelming. The second is actionable. -# - WATCHDOG_BATCH_NOTIFY=true -``` - ---- - -#### 🔇 Silent When Healthy - -``` -Runs 96 times per day. If it produced output every run, the logs would be -useless noise. The watchdog produces no output on clean cycles — only when -something needs attention or a periodic heartbeat fires. - -Heartbeat interval: SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1 - → "♥ docker_watchdog alive — HOST1 — ~4hr uptime" once per hour - → proof the watchdog is running without log spam -``` - ---- - -### ── State Files ───────────────────────────────────────────────────────────── - -| File | Configured As | Location | Resets | Purpose | -|------|--------------|----------|--------|---------| -| Strike counts | `WATCHDOG_STATE_FILE` | `/tmp/` | On reboot | Per-container CPU/HTTP strike counters | -| Skip list | `SYS_WATCHDOG_FAILED_FILE` | `/boot/config/` | Never (manual / auto-clear) | Containers that exhausted restart attempts | -| Restart history | `WATCHDOG_CONTAINER_RESTART_LOG` | `/boot/config/` | Auto-purge after window | Restart loop detection data | -| Shared state | `SYS_WATCHDOG_STATE_FILE` | `/tmp/` | On reboot | RAM emergency flag + cycle heartbeat from system_watchdog.sh | - -`/tmp/` files reset on every reboot — correct, because strike counts from before -a reboot are meaningless after it. `/boot/config/` files survive reboots — also -correct, because a container that was skip-listed before a reboot is still broken after it. - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Normal operation — started automatically by array_start.sh, runs until -# array stops. You do not need to run this manually under normal circumstances. -# ───────────────────────────────────────────────────────────────────────────── -docker_watchdog.sh - -# ───────────────────────────────────────────────────────────────────────────── -# Dry run — walk through a full watchdog cycle without restarting anything. -# Shows what the watchdog would do based on current container states. -# Useful for verifying configuration before enabling automatic restarts. -# ───────────────────────────────────────────────────────────────────────────── -docker_watchdog.sh --dry-run - -# ───────────────────────────────────────────────────────────────────────────── -# Status — show current watchdog state at a glance: -# • Strike counts for all monitored containers -# • Current skip list contents + which are running vs stopped -# • Whether grace period is active and how long remains -# • Whether RAM emergency deferral is active -# • Last cycle timing and daemon health -# ───────────────────────────────────────────────────────────────────────────── -docker_watchdog.sh --status - -# ───────────────────────────────────────────────────────────────────────────── -# Verbose — show full detail for every container checked, every decision made. -# Useful for debugging why a container is or isn't being restarted. -# ───────────────────────────────────────────────────────────────────────────── -docker_watchdog.sh --log -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## ♻️ docker_daily_restart.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Restarts configured containers every night at 1am. Called by `daily_sync_maintenance.sh` -via `DAILY_MAINTENANCE_SCRIPTS` — not scheduled directly. The maintenance window already -owns this timeslot, so any downtime from restarts is absorbed by a window that's already -happening. - ---- - -### ── Why Daily Restarts ─────────────────────────────────────────────────────── - -Not all container degradation triggers a watchdog response. Memory leaks that grow -slowly over days stay well below the hard limit for weeks. Connection tables that fill -up over a month never cross a clear threshold. Live TV schedulers that accumulate -internal state don't fail — they just get progressively slower. - -Daily restarts at 1am clear all of this with zero user impact. Proactive maintenance -at a time when no one is using the services. The containers in this list were specifically -chosen because they are known to degrade — not because they are unreliable. - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# Containers restarted every night at 1am. -# Each entry is just the container name — no configuration needed. -# -# Good candidates for daily restarts: -# - Reverse proxies (connection table management) -# - Authentication services (session cache clearing) -# - Live TV schedulers (accumulated scheduling state) -# - Download managers (connection pool maintenance) -# -HOST1_DAILY_RESTART_CONTAINERS=( - "NginxProxyManager" # connection table fills slowly over weeks - "Authelia" # session cache benefits from periodic clearing - "Dispatcharr" # Live TV scheduler accumulates state - "Dispatcharr-Basic" # secondary Live TV scheduler — same reason - "ErsatzTV-Emby" # channel schedule builder, stale entries accumulate -) -``` - ---- - -### ── Behaviour Rules ────────────────────────────────────────────────────────── - -``` -Running containers → docker restart (graceful stop + start — the correct approach) -Stopped containers → left stopped (was intentionally stopped — state is respected) -Missing containers → logged + skip (not found on this server — not an error) -``` - -The "was running → restart, was stopped → leave stopped" rule is consistent across -the entire ecosystem. Container state is always respected. The restart scripts -never bring back a container that was intentionally stopped. - ---- - -### ── Safeguards ────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# DEPENDENCY ORDERING — restarts happen in dependency-safe order. -# The same WATCHDOG_DEPENDENCIES configuration used by the watchdog applies -# here. If Authelia depends on Mariadb and Redis, those restart first. -# CONTAINER_DELAY seconds wait between dependency restart and dependent restart -# — gives the dependency time to fully initialise before the dependent tries -# to connect. -# -# RESTART VERIFICATION — after each restart, the script waits a settle period -# then checks if the container is still running. A container that starts and -# immediately crashes is marked as failed with a notification sent. The script -# does not silently pass a restart that didn't stick. -# -# DOCKER_TIMEOUT=30 — every docker command is wrapped in a 30 second timeout. -# A hung Docker daemon cannot cause this script to hang indefinitely. -# Timed-out commands are retried per RETRY_COUNT before marking as failed. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Normal — called by daily_sync_maintenance.sh, rarely run directly. -# Safe to run manually for ad hoc restarts when needed. -# ───────────────────────────────────────────────────────────────────────────── -docker_daily_restart.sh - -# ───────────────────────────────────────────────────────────────────────────── -# Dry run — show exactly which containers would be restarted and which would -# be skipped (with reason). Run this before scheduling to verify the list. -# ───────────────────────────────────────────────────────────────────────────── -docker_daily_restart.sh --dry-run - -# ───────────────────────────────────────────────────────────────────────────── -# Status — show configured restart list, current container states, -# and dependency ordering for this server. -# ───────────────────────────────────────────────────────────────────────────── -docker_daily_restart.sh --status - -# Verbose per-container output -docker_daily_restart.sh --log -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## ♻️ docker_weekly_restart.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Restarts configured containers once per week. Called by `weekly_sync_maintenance.sh` -via `WEEKLY_MAINTENANCE_SCRIPTS` — runs at 2:30am Sunday after the sync window has -completed and already restarted the critical containers (Emby, auth stack). - ---- - -### ── Context: After the Sync Window ───────────────────────────────────────── - -`weekly_sync_maintenance.sh` stops the critical container set (Emby, auth stack) for -a clean sync, then restarts them. This script runs **after** that restart — targeting a -**different** set of less-critical services that benefit from a weekly restart but do -not need to be stopped for the sync itself. - -These containers are already running when this script executes. The distinction between -daily and weekly is purely about how often each service needs a clean start to maintain -its best performance: - -``` -Daily: Connection-heavy infrastructure (proxy, auth, Live TV) - → degrades faster, benefits from more frequent resets - -Weekly: Productivity and media services (NextCloud, AdGuard, Immich) - → degrades slowly, monthly restart is overkill, daily is unnecessary -``` - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# Containers restarted every Sunday at 2:30am (after weekly sync completes). -# -HOST1_WEEKLY_RESTART_CONTAINERS=( - "NextCloud" # file sync — benefits from clean weekly start - "AdGuard-Home" # DNS — cache and stat accumulation - "Immich" # photo library — index/cache maintenance -) -``` - -Behaviour, dependency ordering, restart verification, and DOCKER_TIMEOUT are identical -to `docker_daily_restart.sh`. Same rules apply: running → restart, stopped → leave, -missing → skip. - -```bash -docker_weekly_restart.sh # normal run -docker_weekly_restart.sh --dry-run # preview -docker_weekly_restart.sh --status # show config and current states -docker_weekly_restart.sh --log # verbose -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🌐 docker_network_connect.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Ensures custom Docker networks exist and connects specified containers to them at every -array start. Idempotent — if everything is already correct it produces no output. -Run via `ARRAY_START_SCRIPTS` — starts early in the array start sequence, before the -watchdogs begin their first cycle. - ---- - -### ── What It Does ──────────────────────────────────────────────────────────── - -For each configured network, in order: - -``` -1. Does the network exist? - NO → Create it (bridge driver, Docker assigns subnet automatically) - → Send notification — this should not happen except after an update - → Log the subnet that was assigned - YES → Skip creation silently — correct state, nothing to do - -2. For each configured container: - Already connected? → Skip silently — correct state - Not connected? → Connect it - Container missing? → Warn and skip — container may not be running yet, - not treated as fatal, will succeed on next array start -``` - ---- - -### ── Why Network Recreation Matters ───────────────────────────────────────── - -Docker containers get their networks assigned at creation time via the unRAID template. -Changing the network assignment means deleting and recreating the container — which loses -any state not stored in the appdata volume. For containers created by other containers -(NextCloud AIO spawns its own stack, for example), you can't even touch the network -assignment through the unRAID UI. - -`high-availability` is the main shared network in this setup — most containers join it -so they can communicate internally without going through the reverse proxy. After a -unRAID update wipes custom networks, every container on `high-availability` suddenly -can't reach its peers. This script recreates the network and reconnects everything at -the next array start. - -The notification on creation is intentional. Network creation should only happen after -an update — if it's happening regularly, something is wrong with the network configuration -and you need to know. - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# Networks to ensure exist + containers to connect to each network. -# The mapping is many-to-many: every container connects to every network. -# -# Containers listed here do not need to be running — the script handles -# missing containers gracefully (warns + skips). They will be connected on -# the next array start if they come up later. -# -HOST1_NETWORK_CONNECT_NETWORKS=( - "high-availability" # main internal network — most containers should be on this -) - -HOST1_NETWORK_CONNECT_CONTAINERS=( - "memcached" # NextCloud's cache — needs to reach NextCloud AIO network - "Npm-CrowdSec" # CrowdSec bouncer — needs to reach NPM's network -) -``` - -> **Timing dependency:** Networks created by Docker Compose stacks exist only after -> those stacks start. If NextCloud AIO creates the `nextcloud-aio` network at startup -> and this script runs before NextCloud AIO starts, the network won't exist yet and -> the connection fails this run. It will succeed on the next array start once the -> network exists. This is a known limitation — schedule the Compose stacks early in -> the ARRAY_START_SCRIPTS order to minimise the window. - -```bash -docker_network_connect.sh # normal run (at array start) -docker_network_connect.sh --dry-run # show what would be created / connected -docker_network_connect.sh --status # show current network and connection state -docker_network_connect.sh --log # verbose per-network per-container output -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🔧 watchdog_skip_list_manager.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Manual recovery tool. Used when automatic healing hasn't worked and a container needs -human intervention. Gives a clear picture of what the watchdog has already tried and -provides a clean path to resume normal monitoring after the problem is fixed. - ---- - -### ── What the Skip List Is ─────────────────────────────────────────────────── - -When `docker_watchdog.sh` restarts the same container `WATCHDOG_CONTAINER_RESTART_LIMIT` -times within `WATCHDOG_CONTAINER_RESTART_WINDOW` hours, it concludes that restarts are -not fixing whatever is wrong. The container is added to the persistent skip list at -`SYS_WATCHDOG_FAILED_FILE` on `/boot/config/`. The watchdog stops touching it entirely. -A critical notification goes out. - -This is the correct response. A watchdog that keeps hammering a broken container is -not helpful — it's potentially destructive (database corruption, incomplete writes). -Stopping automated attempts and alerting the operator is the right escalation path. - -**Auto-clear:** The watchdog checks the skip list every cycle and removes any container -it finds running. If the container recovers on its own — Docker's built-in restart -policy eventually succeeds after a longer backoff, or someone manually starts it — the -watchdog detects it running and resumes normal monitoring without any intervention needed. -Manual clear is only necessary when the container is stuck stopped and cannot self-recover. - ---- - -### ── Standard Recovery Workflow ───────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Step 1 — Understand the situation -# Shows the skip list, which containers are running vs stopped, how many -# restarts were attempted, and the restart history for each. -# ───────────────────────────────────────────────────────────────────────────── -watchdog_skip_list_manager.sh --status - -# ───────────────────────────────────────────────────────────────────────────── -# Step 2 — Fix the underlying problem first -# Check container logs: docker logs ContainerName --tail 100 -# Check for disk issues: df -h /mnt/user -# Check for database issues: docker exec ContainerName sqlite3 /path/to.db ".tables" -# Fix whatever caused the repeated crashes before clearing the skip list. -# ───────────────────────────────────────────────────────────────────────────── - -# ───────────────────────────────────────────────────────────────────────────── -# Step 3 — Clear the container from the skip list + its restart history -# Clearing restart history is important — otherwise the counter carries over -# and the container hits the limit again almost immediately if it has any -# startup trouble. -# ───────────────────────────────────────────────────────────────────────────── -watchdog_skip_list_manager.sh --clear ContainerName - -# ───────────────────────────────────────────────────────────────────────────── -# Step 4 — Start the container manually -# Starting it yourself confirms your fix worked before handing it back -# to the watchdog. If it crashes immediately, you know the fix didn't work. -# ───────────────────────────────────────────────────────────────────────────── -docker start ContainerName - -# ───────────────────────────────────────────────────────────────────────────── -# Step 5 — Normal monitoring resumes automatically -# On the watchdog's next cycle it will see the container running and remove -# it from the skip list (if you cleared it manually, it's already gone). -# Restart history is clean. Back to normal. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Safety Warning ────────────────────────────────────────────────────────── - -``` -⚠️ If docker_watchdog.sh is currently running when you clear the skip list, - it may re-add the container on its very next cycle if the container is - still in a bad state. - - The script detects this and warns you: - "docker_watchdog.sh is currently RUNNING — cleared container may be - re-added on next cycle if still failing" - - Fix the root cause BEFORE clearing. Clearing the skip list without fixing - the underlying problem just resets the counter — the container will exhaust - its restart attempts again and return to the skip list. -``` - ---- - -### ── All Actions ───────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Show status — skip list contents, container states, restart history -# This is the default action — running without arguments shows status -# ───────────────────────────────────────────────────────────────────────────── -watchdog_skip_list_manager.sh -watchdog_skip_list_manager.sh --status # explicit - -# ───────────────────────────────────────────────────────────────────────────── -# Clear specific container — removes from skip list + clears restart history -# Requires confirmation (type YES) unless --force is passed -# ───────────────────────────────────────────────────────────────────────────── -watchdog_skip_list_manager.sh --clear ContainerName -watchdog_skip_list_manager.sh --clear ContainerName --force # no prompt - -# ───────────────────────────────────────────────────────────────────────────── -# Clear everything — full reset of skip list and restart history -# Use when multiple containers are affected or after a systemic problem is fixed -# ───────────────────────────────────────────────────────────────────────────── -watchdog_skip_list_manager.sh --clear-all -watchdog_skip_list_manager.sh --clear-all --force # non-interactive - -# ───────────────────────────────────────────────────────────────────────────── -# Dry run — show what would be cleared without actually clearing -# Works with --clear and --clear-all -# ───────────────────────────────────────────────────────────────────────────── -watchdog_skip_list_manager.sh --clear-all --dry-run -``` - ---- - -## ━━━ HOW THE SCRIPTS FIT TOGETHER ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ━━━ HOW THE SCRIPTS RELATE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ``` Array starts │ - ▼ -docker_network_connect.sh ─────── run once - │ ensure networks + connections - │ silent if correct, notify if creating + ├── docker_network_connect.sh ────── run once at start + │ ensure networks + connections exist + │ silent if correct, notify if creating │ - ▼ (continuous background) -docker_watchdog.sh ─────────────── every 15 minutes - │ Tier 1: memory, CPU, HTTP, required - │ Tier 2: global unhealthy/OOM/crash/dead scan - │ reads system_watchdog state (RAM emergency) - │ - │ (on skip list event) - ▼ -watchdog_skip_list_manager.sh ──── manual - inspect state, clear after fixing + └── docker_watchdog.sh ──────────── continuous background loop (every 15min) + Tier 1: memory, CPU, HTTP, required containers + Tier 2: global unhealthy / OOM / crash / dead scan + reads system_watchdog state (RAM emergency deferral) + │ + │ (on skip list event → operator uses) + └── Tools/watchdog_skip_list_manager.sh + inspect state, clear after fixing root cause -Scheduled maintenance windows: - │ - ▼ (1am daily) -daily_sync_maintenance.sh - └── docker_daily_restart.sh ── restart connection-heavy services - NPM, Authelia, Dispatcharr, ErsatzTV - │ - ▼ (2:30am Sunday) -weekly_sync_maintenance.sh - └── docker_weekly_restart.sh ─ restart less-critical services - NextCloud, AdGuard, Immich +Daily maintenance window (1am): + daily_sync_maintenance.sh + ├── docker_update.sh ──────────── pull latest images (DAILY_RESTART_CONTAINERS) + └── docker_daily_restart.sh ───── restart connection-heavy services + +Weekly maintenance window (2:30am Sunday): + weekly_sync_maintenance.sh + ├── docker_weekly_restart.sh ──── restart less-critical services + ├── docker_update.sh --remainder ─ update containers not in managed lists + └── docker_update_remaining.sh ── prune dangling images + +Critical maintenance window (every 15min): + critical_sync_maintenance.sh + └── downloaders_reset.sh ──────── clear stuck downloads / stale queue entries + +Array stopping: + array_stopping.sh + └── docker_container_stop.sh ──── ordered graceful shutdown ``` - ---- - -## ━━━ CONFIGURATION REFERENCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -All configuration lives in two places. `detect_hosts()` in `common.sh` aliases all -`HOST1_` and `HOST2_` prefixed variables to their unprefixed names so the scripts -always use the right values for the server they're running on. - ---- - -### 📋 master_host*.conf — Per-Host Configuration - -These vary between HOST1 and HOST2 because each server runs different containers -at different resource limits with different network requirements. - -```bash -# master_host1.conf (or master_host2.conf for HOST2) -# ───────────────────────────────────────────────────────────────────────────── - -# Tier 1 — memory limits ("ContainerName:LimitInMB") -HOST1_WATCHDOG_CONTAINERS=( - "Emby:18432" - "LidaTube:6144" -) - -# Tier 1 — HTTP health check endpoints ("ContainerName:http://host:port") -HOST1_WATCHDOG_CONTAINER_URLS=( - "Emby:http://localhost:8096" -) - -# Tier 1 — must always be running -HOST1_WATCHDOG_REQUIRED_CONTAINERS=( - "NginxProxyManager" - "Authelia" - "Mariadb-Authelia" - "Redis-Authelia" -) - -# Tier 1 + 2 — dependency ordering ("Dependent:dep1 dep2") -HOST1_WATCHDOG_DEPENDENCIES=( - "Authelia:Mariadb-Authelia Redis-Authelia" -) - -# Daily restart list -HOST1_DAILY_RESTART_CONTAINERS=( - "NginxProxyManager" - "Authelia" - "Dispatcharr" -) - -# Weekly restart list -HOST1_WEEKLY_RESTART_CONTAINERS=( - "NextCloud" - "AdGuard-Home" -) - -# Networks to ensure exist -HOST1_NETWORK_CONNECT_NETWORKS=( - "high-availability" -) - -# Containers to connect to every configured network -HOST1_NETWORK_CONNECT_CONTAINERS=( - "memcached" - "Npm-CrowdSec" -) -``` - ---- - -### 📋 master.conf — Shared Configuration - -These apply equally to both servers — thresholds, intervals, toggle switches. - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── - -# ── Watchdog Intervals and Timing ────────────────────────────────────────── - DOCKER_WATCHDOG_INTERVAL=900 # seconds between cycles (15 minutes) - WATCHDOG_STARTUP_GRACE=600 # seconds before restarts begin after boot - CONTAINER_DELAY=15 # seconds between dependency + dependent restart - -# ── Memory Thresholds ────────────────────────────────────────────────────── - SOFT_MEM_THRESHOLD=80 # warn at % of hard limit (no restart) - -# ── CPU Thresholds ───────────────────────────────────────────────────────── - SOFT_CPU_THRESHOLD=50 # warn threshold — normalised % of total cores - HARD_CPU_THRESHOLD=85 # strike threshold — normalised % of total cores - CPU_FAIL_LIMIT=2 # consecutive strikes before restart - -# ── HTTP Health Check ────────────────────────────────────────────────────── - CURL_TIMEOUT=5 # seconds before curl times out - RESP_FAIL_LIMIT=2 # consecutive failures before restart - -# ── Restart Loop Protection ──────────────────────────────────────────────── - WATCHDOG_CONTAINER_RESTART_LIMIT=3 # restarts before skip list - WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours - -# ── Tier 2 Global Scan ───────────────────────────────────────────────────── - WATCHDOG_SCAN_ALL=true # enable global scan - WATCHDOG_SCAN_IGNORE=() # containers excluded from Tier 2 - WATCHDOG_RESTART_UNHEALTHY=true - WATCHDOG_NOTIFY_OOM=true - WATCHDOG_NOTIFY_CRASHLOOP=true - WATCHDOG_CRASH_LIMIT=5 - WATCHDOG_RESTART_DEAD=true - WATCHDOG_RESTART_CRASHED=true - -# ── Notifications ────────────────────────────────────────────────────────── - WATCHDOG_BATCH_NOTIFY=true # collect events, send one summary -``` - ---- - -## ━━━ ADDING A CONTAINER ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Adding a new container to monitoring is additive — add the relevant lines to -`master_host1.conf` (and `master_host2.conf` if it runs there too). No changes to -any script needed. `detect_hosts()` picks up the new configuration on the next cycle. - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# Example: adding "MyApp" to full Tier 1 monitoring + daily restarts - -# 1. Memory hard limit — restart immediately if exceeded -HOST1_WATCHDOG_CONTAINERS=( - "Emby:18432" - "MyApp:2048" # 2GB — check with `docker stats MyApp` to size this correctly -) - -# 2. HTTP health check — restart after 2 consecutive failures -HOST1_WATCHDOG_CONTAINER_URLS=( - "Emby:http://localhost:8096" - "MyApp:http://localhost:8080/health" # or just the root if no /health endpoint -) - -# 3. Required — must always be running -HOST1_WATCHDOG_REQUIRED_CONTAINERS=( - "NginxProxyManager" - "Authelia" - "MyApp" # add here if it should always be running -) - -# 4. Dependency — if MyApp needs its database up first -HOST1_WATCHDOG_DEPENDENCIES=( - "Authelia:Mariadb-Authelia Redis-Authelia" - "MyApp:MyApp-Database" # database restarts first, then MyApp -) - -# 5. Daily restart — if MyApp degrades over time -HOST1_DAILY_RESTART_CONTAINERS=( - "NginxProxyManager" - "Authelia" - "MyApp" -) - -# ───────────────────────────────────────────────────────────────────────────── -# Tier 2 picks up MyApp automatically — no configuration needed. -# It will be included in the global unhealthy/OOM/crash/dead scan without -# any additional setup. Tier 2 is the catch-all for everything not in Tier 1. -# ───────────────────────────────────────────────────────────────────────────── - -# To EXCLUDE MyApp from Tier 2 (e.g. it intentionally exits between runs): -# master.conf -WATCHDOG_SCAN_IGNORE=( - "MyApp" # one-shot container — exits normally, not a crash -) -``` \ No newline at end of file diff --git a/Docker_Essentials/docker_container_stop.sh b/Docker_Essentials/docker_container_stop.sh index 6d75f1d..3006063 100755 --- a/Docker_Essentials/docker_container_stop.sh +++ b/Docker_Essentials/docker_container_stop.sh @@ -2,37 +2,116 @@ # ============================================================================================== # ============================= Docker Container Stop ========================================== # ============================================================================================== -# Stops all running Docker containers one at a time, verifying each is stopped before -# moving to the next. Called by array_stopping.sh as part of a planned shutdown sequence. # -# ── STOP SEQUENCE PER CONTAINER ────────────────────────────────────────────────────────────── -# 1. docker stop -t 30 (SIGTERM + 30s grace period — docker sends SIGKILL if needed) -# 2. Verify stopped — if still running, retry up to RETRY_COUNT times -# 3. docker kill (SIGKILL) if all retries exhausted -# 4. Final verify — error if still running after force-kill -# Never moves to the next container until the current one is confirmed stopped. +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Gracefully stops all running Docker containers in a verified sequential order. # -# ── WHY SEQUENTIAL ──────────────────────────────────────────────────────────────────────────── -# Containers may have dependencies — stopping one at a time avoids abruptly severing a -# service while its dependents are still running and trying to use it. +# Called by array_stopping.sh during planned shutdowns, maintenance windows, +# and controlled reboot operations. # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# Root check — docker requires root -# Per-container verify — confirmed stopped before proceeding to next -# Retry loop — RETRY_COUNT attempts before escalating to force-kill -# SIGTERM → SIGKILL — graceful then forced, never skips graceful -# DOCKER_TIMEOUT — all docker calls protected against hung daemon -# notify on failures — alert if any container cannot be stopped +# The script guarantees each container is fully stopped before moving to the +# next one — preventing dependency breakage, partial shutdown states, and +# abrupt service termination cascades. # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# RETRY_COUNT — retry attempts before force-kill (default 3) -# SLEEP — seconds between retries +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Shutdown is processed one container at a time: +# +# 1. docker stop -t 30 +# → sends SIGTERM with graceful shutdown window +# +# 2. Verify container state +# → confirm container fully stopped before continuing +# +# 3. Retry if still running +# → up to RETRY_COUNT attempts +# +# 4. Escalate to docker kill +# → SIGKILL only after graceful attempts exhausted +# +# 5. Final verification +# → failure notification if container survives SIGKILL +# +# The script NEVER advances to the next container until the current one +# is confirmed stopped or declared failed. +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Sequential Shutdown +# Containers may depend on upstream services still being available during +# shutdown. Sequential processing reduces dependency severance during stop. +# +# Graceful First, Forced Last +# SIGTERM is always attempted before SIGKILL. The script never force-kills +# first unless Docker itself escalates internally after timeout expiration. +# +# Verification Over Assumption +# Docker command success alone is not trusted. Container state is verified +# after every stop attempt. +# +# Fail Loudly +# Containers that cannot be stopped generate notifications and non-zero exit +# status so orchestrators know shutdown integrity was compromised. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root Enforcement +# Docker operations require root privileges. +# +# Docker Presence Check +# Verifies docker binary exists before execution. +# +# Timeout Protection +# All docker commands wrapped in timeout protection to prevent daemon hangs +# from stalling shutdown indefinitely. +# +# Retry Escalation +# Graceful retries occur before SIGKILL escalation. +# +# Per-Container Validation +# Every container state verified before progressing to the next. +# +# Deterministic Ordering +# Running container list sorted before processing for stable execution order. +# +# Failure Notification +# Containers surviving SIGKILL trigger operator notification. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# RETRY_COUNT +# Graceful retry attempts before force-kill escalation +# +# SLEEP +# Delay in seconds between retry attempts +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# docker_container_stop.sh +# Stop all running containers in verified sequential order +# +# docker_container_stop.sh --dry-run +# Preview shutdown actions without stopping containers +# +# docker_container_stop.sh --status +# Show currently running containers and configuration state +# +# docker_container_stop.sh --log +# Verbose per-container execution logging # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# docker_container_stop.sh — stop all running containers -# docker_container_stop.sh --dry-run — show which containers would be stopped -# docker_container_stop.sh --status — show running containers and exit -# docker_container_stop.sh --log — verbose output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Docker_Essentials/docker_daily_restart.sh b/Docker_Essentials/docker_daily_restart.sh index 4ca2723..3ed911d 100644 --- a/Docker_Essentials/docker_daily_restart.sh +++ b/Docker_Essentials/docker_daily_restart.sh @@ -2,52 +2,102 @@ # ============================================================================================== # ================================= Docker Daily Restart ======================================= # ============================================================================================== -# Restarts or starts all containers in HOST*_DAILY_RESTART_CONTAINERS. -# Called by daily_sync_maintenance.sh via DAILY_MAINTENANCE_SCRIPTS every night at 1am. -# Can also be run manually for ad hoc restarts. # -# ── WHY DAILY RESTARTS ──────────────────────────────────────────────────────────────────────── -# Some containers degrade over time without a restart: -# Dispatcharr — Live TV scheduler accumulates state and slows down -# NginxProxyManager — connection table grows, occasional stale proxy entries -# Authelia — session cache benefits from periodic clearing -# Daily restart is intentional maintenance, not just housekeeping. +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Restarts configured containers every night at 1am as proactive maintenance. # -# ── BEHAVIOUR ───────────────────────────────────────────────────────────────────────────────── -# Running containers → docker restart (graceful stop + start) -# Stopped containers → left stopped — was down intentionally, do not bring back up -# Missing containers → logged and skipped — not treated as fatal -# Each action uses RETRY_COUNT + SLEEP from master.conf for retry logic. +# Called by daily_sync_maintenance.sh via DAILY_MAINTENANCE_SCRIPTS. Runs inside +# the daily maintenance window — any service downtime is absorbed by a window +# that is already happening. Also drives docker_update.sh in normal mode: the +# same DAILY_RESTART_CONTAINERS list is used for both restarts and image pulls, +# so there is no second list to maintain. # -# The "was running → restart, was stopped → leave stopped" rule is consistent -# across the entire ecosystem — container state is always respected. +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# Dependency ordering — containers restart in dependency-safe order using -# HOST*_WATCHDOG_DEPENDENCIES from master_host*.conf. If Authelia depends on -# Mariadb + Redis, those restart first with CONTAINER_DELAY before Authelia starts. +# Proactive Maintenance +# Daily restarts target containers known to degrade over time without +# crossing a clear failure threshold — connection table growth, scheduler +# state accumulation, session cache bloat. The watchdog cannot detect this +# class of degradation. Scheduled restarts clear it before it becomes visible. # -# Restart verification — after each restart, container state is checked after a short -# settle period. If the container fails to stay running it is marked as failed and -# a notification is sent rather than silently passing. +# State Respect +# Running containers are restarted. Stopped containers are left stopped — they +# were intentionally halted and this script has no authority to override that +# decision. This rule is consistent across the entire ecosystem. # -# Timeout protection — all docker commands are wrapped in a 30 second timeout. -# A hung Docker daemon cannot cause this script to hang indefinitely. -# Timed-out commands are retried per RETRY_COUNT before marking as failed. +# Dependency-Safe Ordering +# Restarts follow the same dependency ordering used by docker_watchdog.sh. +# Services that other containers depend on restart first. A dependent is never +# restarted while its dependency is still coming up. # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── -# HOST*_DAILY_RESTART_CONTAINERS — list of containers to restart daily -# Set by detect_hosts() alias → DAILY_RESTART_CONTAINERS used by this script +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# RETRY_COUNT — retry attempts before giving up on a container -# SLEEP — seconds between retry attempts +# Dependency Ordering +# Containers restart in dependency-safe order using HOST*_WATCHDOG_DEPENDENCIES. +# CONTAINER_DELAY seconds between dependency restart and dependent restart gives +# the dependency time to fully initialise before dependents try to connect. +# +# Restart Verification +# After each restart, container state is checked after a settle period. A +# container that starts and immediately crashes is marked failed and a +# notification is sent — the script does not silently pass a restart that +# did not stick. +# +# Timeout Protection +# All docker commands wrapped in a 30 second timeout. A hung Docker daemon +# cannot cause this script to hang indefinitely. Timed-out commands retry +# per RETRY_COUNT before marking as failed. +# +# Lock Acquisition +# acquire_lock() prevents concurrent execution if a previous run is still active. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# +# HOST*_DAILY_RESTART_CONTAINERS +# Containers restarted nightly. Also used by docker_update.sh normal mode +# for image pulls — add a container once, it gets both. Aliased by +# detect_hosts() → DAILY_RESTART_CONTAINERS +# +# HOST*_WATCHDOG_DEPENDENCIES +# Dependency ordering shared with docker_watchdog.sh. Aliased by +# detect_hosts() → WATCHDOG_DEPENDENCIES +# +# master.conf +# +# RETRY_COUNT +# Retry attempts before giving up on a container +# +# SLEEP +# Seconds between retry attempts +# +# CONTAINER_DELAY +# Seconds to wait after restarting a dependency before starting its dependents +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# docker_daily_restart.sh +# Restart all containers in DAILY_RESTART_CONTAINERS +# +# docker_daily_restart.sh --dry-run +# Preview which containers would be restarted and which would be skipped +# +# docker_daily_restart.sh --status +# Show configured restart list, container states, and dependency ordering +# +# docker_daily_restart.sh --log +# Verbose per-container execution output # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# docker_daily_restart.sh — normal restart -# docker_daily_restart.sh --dry-run — preview without restarting -# docker_daily_restart.sh --log — verbose output -# docker_daily_restart.sh --status — show config and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Docker_Essentials/docker_network_connect.sh b/Docker_Essentials/docker_network_connect.sh index f032cfe..dc90d7f 100755 --- a/Docker_Essentials/docker_network_connect.sh +++ b/Docker_Essentials/docker_network_connect.sh @@ -2,46 +2,100 @@ # ============================================================================================== # ================================= Docker Network Connect ===================================== # ============================================================================================== -# Ensures custom Docker networks exist then connects specified containers to them. -# Run once at array start via ARRAY_START_SCRIPTS — idempotent, safe to re-run anytime. # -# ── WHAT IT DOES ────────────────────────────────────────────────────────────────────────────── -# For each network in NETWORK_CONNECT_NETWORKS: -# 1. Check if network exists -# → missing → create it (bridge driver, Docker assigns subnet automatically) -# notifies on creation — unexpected, usually means unRAID wiped networks -# → exists → skip creation silently -# 2. Connect each container in NETWORK_CONNECT_CONTAINERS to the network -# → already connected → skip cleanly -# → not connected → connect it -# → container not found → warn and skip (not an error — may not be running yet) +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Ensures custom Docker networks exist and connects configured containers to +# them at every array start. # -# ── USE CASE ────────────────────────────────────────────────────────────────────────────────── -# high-availability is the main custom network — shared by most containers. -# After a unRAID update wipes custom networks → recreated automatically at next array start. -# Containers on their own networks (NextCloud AIO etc.) can be added to -# NETWORK_CONNECT_CONTAINERS so they also join high-availability without touching their -# primary network configuration. +# Called via ARRAY_START_SCRIPTS — runs early in the array start sequence, +# before watchdogs begin their first cycle. Idempotent: safe to re-run at any +# time. Silent when everything is already correct. Notifies when a network had +# to be created — that only happens after a unRAID update wipes custom networks, +# and it is worth knowing when it does. # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# Docker daemon check — verifies daemon is responsive before any network operations -# Command validation — validates unRAID notify script before use -# Timeout protection — all docker commands wrapped in timeout — daemon hangs cannot stall -# Empty array guards — warns and exits cleanly if arrays are unconfigured -# Silent by default — only warnings and errors produce output (v3.4 standard) -# network creation always warns — unexpected, means networks were wiped -# Idempotent — safe to run multiple times, skips what is already correct +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── -# HOST*_NETWORK_CONNECT_NETWORKS — networks to ensure exist -# HOST*_NETWORK_CONNECT_CONTAINERS — containers to connect to each network -# Aliased by detect_hosts() — script uses unprefixed names +# For each configured network: +# +# 1. Does the network exist? +# NO → create it (bridge driver, Docker assigns subnet automatically) +# → send notification — creation is unexpected outside post-update recovery +# YES → skip silently +# +# 2. For each configured container: +# Already connected → skip silently +# Not connected → connect it +# Container missing → warn and skip — may not be running yet, not fatal +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Idempotent by Design +# Running this script multiple times produces the same result as running it +# once. Skips anything already in the correct state without error or noise. +# +# Silent When Correct +# Produces no output on a clean run. The absence of output is confirmation +# that everything is already correct. +# +# Notify on Creation +# Network creation is always notified because it should only happen after a +# unRAID update. If it happens regularly something is misconfigured and the +# operator needs to know. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Docker Daemon Check +# Verifies daemon is responsive before any network operations. Network +# commands against a hung daemon hang indefinitely. +# +# Timeout Protection +# All docker commands wrapped in timeout. Daemon hangs cannot stall the +# array start sequence. +# +# Empty Array Guards +# Warns and exits cleanly if NETWORK_CONNECT_NETWORKS or +# NETWORK_CONNECT_CONTAINERS are unconfigured. +# +# Command Validation +# Validates unRAID notify script before use. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# +# HOST*_NETWORK_CONNECT_NETWORKS +# Networks to ensure exist at array start. Aliased by detect_hosts() → +# NETWORK_CONNECT_NETWORKS +# +# HOST*_NETWORK_CONNECT_CONTAINERS +# Containers to connect to every configured network. Aliased by +# detect_hosts() → NETWORK_CONNECT_CONTAINERS +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# docker_network_connect.sh +# Ensure all configured networks exist and containers are connected +# +# docker_network_connect.sh --dry-run +# Preview what would be created or connected without making changes +# +# docker_network_connect.sh --status +# Show current network state and container connection status +# +# docker_network_connect.sh --log +# Verbose per-network per-container output # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# docker_network_connect.sh — normal run -# docker_network_connect.sh --dry-run — preview without making changes -# docker_network_connect.sh --log — verbose output -# docker_network_connect.sh --status — show current network state and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Docker_Essentials/docker_update.sh b/Docker_Essentials/docker_update.sh index 9740b81..001ffb7 100644 --- a/Docker_Essentials/docker_update.sh +++ b/Docker_Essentials/docker_update.sh @@ -2,53 +2,117 @@ # ============================================================================================== # ================================= Docker Update ============================================== # ============================================================================================== -# Two modes — normal (daily) and remainder (weekly). # -# ── NORMAL MODE (daily) ─────────────────────────────────────────────────────────────────────── -# Pulls the latest image for each container in HOST*_DAILY_RESTART_CONTAINERS. -# Called by daily_sync_maintenance.sh before docker_daily_restart.sh — containers stay -# running during the pull, so there is no extra downtime. +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Pulls the latest images for configured containers. Two modes: normal (daily) +# and remainder (weekly). # -# ── REMAINDER MODE (weekly) ─────────────────────────────────────────────────────────────────── -# Called by weekly_sync_maintenance.sh as the last step. -# Updates all currently running containers that are NOT in: -# DAILY_RESTART_CONTAINERS — already updated daily -# emby + critical-data profiles — already updated inline by the weekly sync window -# FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER* — owned by the remote server's update cycle +# Normal mode is called by daily_sync_maintenance.sh before docker_daily_restart.sh. +# Containers stay running during the pull — no extra downtime beyond what the +# nightly restart already causes. # -# Fallback containers are excluded because this server only runs them during a failover. -# The remote server is the version owner — if remainder updates them independently and a -# handback writeback occurs, the remote's older version may not handle the newer data. -# This catches everything local-only (Organizr, AdGuard, etc.) once a week. +# Remainder mode is called by weekly_sync_maintenance.sh as the final update step. +# It catches everything that normal mode and the weekly sync window did not already +# update — derived automatically from docker ps, nothing to configure. # -# ── WHY SAME LIST AS DAILY RESTART (normal mode) ───────────────────────────────────────────── -# Containers that restart daily (auth stack: Authelia, NPM, Mariadb, Redis, etc.) are -# exactly the containers that benefit from staying current. Reusing DAILY_RESTART_CONTAINERS -# means no second list to maintain — add/remove a container once and both update + restart -# reflect the change automatically. +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── WHAT THIS DOES ──────────────────────────────────────────────────────────────────────────── -# docker pull — fetches the latest digest from the registry -# Old vs new image ID comparison — distinguishes "updated" from "already current" -# Containers keep running — pull does not affect the live container -# docker_daily_restart.sh runs after (normal mode) — containers restart on the fresh image +# Normal mode (daily): +# Targets DAILY_RESTART_CONTAINERS — same list used by docker_daily_restart.sh. +# Pull → compare old vs new image ID → mark updated or already current. +# docker_daily_restart.sh runs after — containers restart onto the fresh image. # -# ── TOGGLE ──────────────────────────────────────────────────────────────────────────────────── -# DAILY_CONTAINER_UPDATES=false in master.conf — skips normal mode, exits cleanly -# docker_daily_restart.sh still runs regardless — update and restart are independent -# Remainder mode has no toggle — exclude it from WEEKLY_MAINTENANCE_SCRIPTS to disable +# Remainder mode (weekly): +# Targets all currently running containers NOT in: +# DAILY_RESTART_CONTAINERS — already updated daily +# emby + critical-data profiles — updated inline by the weekly sync window +# FALLBACK_*_TIER* — owned by the remote server's update cycle +# Pull → compare → restart if updated → prune dangling images. # -# ── CONFIGURATION ───────────────────────────────────────────────────────────────────────────── -# master.conf: DAILY_CONTAINER_UPDATES — enable/disable normal mode (default: true) -# master_host*.conf: HOST*_DAILY_RESTART_CONTAINERS — containers to update (normal mode) -# master.conf: PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data] — remainder exclusions +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Single List +# Normal mode reuses DAILY_RESTART_CONTAINERS rather than maintaining a +# separate update list. Adding or removing a container from the restart list +# automatically updates the image pull list — one change, both places. +# +# Version Ownership +# Fallback containers are excluded from remainder mode. This server only runs +# them during a failover. The remote server owns their version — if remainder +# updates them independently and a handback occurs, the remote's older image +# may not handle data written by the newer version. +# +# State Respect +# Stopped containers are never targeted. Pulling while stopped adds no value +# and a stopped container was likely halted intentionally. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root Enforcement +# Docker operations require root privileges. +# +# DAILY_CONTAINER_UPDATES Toggle +# Normal mode exits cleanly when disabled. docker_daily_restart.sh still runs +# regardless — update and restart are independent operations. +# +# Fallback Exclusion +# Remainder mode excludes containers owned by the remote server's update cycle +# to prevent version divergence across the failover boundary. +# +# Running-Only Filter +# Stopped containers excluded from remainder mode — intentionally down. +# +# Image ID Comparison +# Containers not restarted unless their image actually changed. Pulls that +# result in "already up to date" produce no restart. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# DAILY_CONTAINER_UPDATES +# Enable or disable normal mode. docker_daily_restart.sh runs regardless. +# (default: true) +# +# PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data] +# Container names for emby and critical-data profiles — excluded from +# remainder mode (already updated by the weekly sync window) +# +# master_host*.conf +# +# HOST*_DAILY_RESTART_CONTAINERS +# Containers updated in normal mode. Aliased by detect_hosts() → +# DAILY_RESTART_CONTAINERS +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# docker_update.sh +# Normal mode — pull latest images for DAILY_RESTART_CONTAINERS +# +# docker_update.sh --remainder +# Remainder mode — pull all running containers not in managed lists, +# restart those that received updates, prune dangling images +# +# docker_update.sh --dry-run +# Preview which containers would be pulled without making changes +# +# docker_update.sh --status +# Show configuration and container list for current mode +# +# docker_update.sh --log +# Verbose per-container pull and comparison output # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# docker_update.sh — normal mode: update DAILY_RESTART_CONTAINERS -# docker_update.sh --remainder — remainder mode: update all except daily + weekly sync containers -# docker_update.sh --dry-run — show what would be pulled -# docker_update.sh --log — verbose output -# docker_update.sh --status — show config and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Docker_Essentials/docker_update_remaining.sh b/Docker_Essentials/docker_update_remaining.sh index 2f3f8ce..026e900 100644 --- a/Docker_Essentials/docker_update_remaining.sh +++ b/Docker_Essentials/docker_update_remaining.sh @@ -2,43 +2,73 @@ # ============================================================================================== # ============================= Docker Update — Remaining ====================================== # ============================================================================================== -# Pulls the latest image for every running container NOT already covered by the daily or -# weekly update/restart cycles. Restarts containers that received a new image, then prunes -# dangling images. Runs at the end of the weekly maintenance window. # -# ── WHAT THIS COVERS ────────────────────────────────────────────────────────────────────────── -# Daily update: DAILY_RESTART_CONTAINERS — auth stack, NPM, Dispatcharr, etc. -# Weekly update: WEEKLY_RESTART_CONTAINERS — NextCloud, AdGuard, Immich, etc. -# This script: everything else running on the system (media stack, utilities, etc.) +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Weekly sweep that pulls the latest image for every running container not +# already covered by the daily or weekly managed update cycles. Restarts +# containers that received a new image, then prunes dangling images. # -# Together the three scripts ensure every deployed container receives at least one image -# pull per week, with no container list to maintain here — it derives the remainder -# automatically from `docker ps` minus the two managed lists. +# Called by weekly_sync_maintenance.sh as the final step in the weekly window. +# Derives its target list automatically from docker ps minus the two managed +# lists — there is nothing to configure for this script. # -# ── WHAT THIS DOES ──────────────────────────────────────────────────────────────────────────── -# 1. Pull latest image for each remaining running container -# 2. Restart containers whose image ID changed (new update landed) -# 3. Prune dangling images left behind by the updates -# Containers already up to date are not restarted. +# Together with docker_update.sh (normal + remainder modes), every deployed +# container receives at least one image pull per week without any per-container +# configuration required here. # -# ── EXCLUSION LOGIC ─────────────────────────────────────────────────────────────────────────── -# Exclusion set = DAILY_RESTART_CONTAINERS + WEEKLY_RESTART_CONTAINERS (aliased by detect_hosts) -# Only running containers are targeted — stopped containers are intentionally excluded -# (stopped = likely paused intentionally; pulling while stopped adds no value). +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── TOGGLE ──────────────────────────────────────────────────────────────────────────────────── -# WEEKLY_REMAINING_UPDATES=false in master.conf — skips all pulls, exits cleanly +# Root Enforcement +# Docker operations require root privileges. # -# ── CONFIGURATION ───────────────────────────────────────────────────────────────────────────── -# master.conf: WEEKLY_REMAINING_UPDATES — enable/disable (default: true) -# master_host*.conf: HOST*_DAILY_RESTART_CONTAINERS — excluded from this script -# HOST*_WEEKLY_RESTART_CONTAINERS — excluded from this script +# WEEKLY_REMAINING_UPDATES Toggle +# Exits cleanly when disabled via master.conf. +# +# Running-Only Filter +# Stopped containers excluded — intentionally down, pulling adds no value. +# +# Image ID Comparison +# Containers not restarted unless their image actually changed. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# WEEKLY_REMAINING_UPDATES +# Enable or disable this script. (default: true) +# To disable without the toggle: remove from WEEKLY_MAINTENANCE_SCRIPTS. +# +# master_host*.conf +# +# HOST*_DAILY_RESTART_CONTAINERS +# Excluded from this script — already updated daily. Aliased by +# detect_hosts() → DAILY_RESTART_CONTAINERS +# +# HOST*_WEEKLY_RESTART_CONTAINERS +# Excluded from this script — already updated by weekly sync window. +# Aliased by detect_hosts() → WEEKLY_RESTART_CONTAINERS +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# docker_update_remaining.sh +# Pull all remaining running containers, restart those updated, prune images +# +# docker_update_remaining.sh --dry-run +# Preview which containers would be pulled and restarted +# +# docker_update_remaining.sh --status +# Show exclusion lists and current remaining container count +# +# docker_update_remaining.sh --log +# Verbose per-container pull and restart output # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# docker_update_remaining.sh — normal run -# docker_update_remaining.sh --dry-run — show which containers would be pulled/restarted -# docker_update_remaining.sh --log — verbose output -# docker_update_remaining.sh --status — show config and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Docker_Essentials/docker_watchdog.sh b/Docker_Essentials/docker_watchdog.sh index a0360e8..180eacd 100755 --- a/Docker_Essentials/docker_watchdog.sh +++ b/Docker_Essentials/docker_watchdog.sh @@ -2,81 +2,217 @@ # ============================================================================================== # ================================= Docker Watchdog ============================================ # ============================================================================================== -# Two-tier self-healing container monitoring system. -# Runs continuously as a background process — started by array_started.sh at array start. -# Shuts down cleanly on SIGTERM/SIGINT when array stops. # -# ── TIER 1 — STRICT MONITORING ──────────────────────────────────────────────────────────────── -# Applies only to explicitly configured containers (HOST*_WATCHDOG_CONTAINERS etc.) +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Two-tier self-healing container monitoring system. Runs as a continuous +# background daemon started by array_started.sh at array start. Shuts down +# cleanly on SIGTERM/SIGINT when the array stops. # -# Memory hard limits — immediate restart if container exceeds configured MB ceiling -# Memory soft threshold — warn at SOFT_MEM_THRESHOLD % of hard limit (no restart) +# Every DOCKER_WATCHDOG_INTERVAL seconds the watchdog runs a full cycle: +# Tier 1 applies specific thresholds to explicitly configured containers. +# Tier 2 scans everything else for generic health problems. Silent on clean +# cycles, loud when something needs attention. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Tier 1 — Strict Per-Container Monitoring +# Applies only to containers explicitly configured in master_host*.conf. +# +# Memory hard limits — immediate restart if container exceeds MB ceiling +# Memory soft threshold — warn at SOFT_MEM_THRESHOLD % of limit (no restart) # CPU thresholds — strike system: warn at SOFT_CPU_THRESHOLD, restart after # CPU_FAIL_LIMIT consecutive strikes at HARD_CPU_THRESHOLD -# HTTP responsiveness — strike system: restart after RESP_FAIL_LIMIT consecutive failures +# HTTP responsiveness — strike system: restart after RESP_FAIL_LIMIT consecutive +# failures against the configured endpoint # Required containers — must always be running; strike system before restart; -# skip list after WATCHDOG_CONTAINER_RESTART_LIMIT in window; -# auto-clears when container recovers +# skip list after WATCHDOG_CONTAINER_RESTART_LIMIT in window # -# ── TIER 2 — GLOBAL HEALTH SCAN ─────────────────────────────────────────────────────────────── -# Scans ALL running containers when WATCHDOG_SCAN_ALL=true. -# Containers in WATCHDOG_SCAN_IGNORE are excluded from Tier 2. +# Tier 2 — Global Health Scan +# Scans ALL running containers when WATCHDOG_SCAN_ALL=true. +# Containers in WATCHDOG_SCAN_IGNORE are excluded. # -# Unhealthy status — Docker HEALTHCHECK unhealthy → safe_restart() -# OOM killed — kernel OOM killed → safe_restart() + notify -# OOM state tracked per-session to prevent restart loop -# Crash loop detection — RestartCount climbing → notify; above WATCHDOG_CRASH_LIMIT -# → safe_restart() → skip list if restart limit hit -# Dead containers — safe_restart() via remove + start -# Unexpected exits — non-zero exit code → safe_restart() +# Unhealthy status — Docker HEALTHCHECK unhealthy → restart +# OOM killed — kernel OOM kill detected → restart + notify +# Crash loop — RestartCount climbing → notify; above WATCHDOG_CRASH_LIMIT +# → restart → skip list if restart limit hit +# Dead containers — remove + start (dead state cannot be restarted directly) +# Unexpected exits — non-zero exit code → restart +# +# Cross-Cutting Intelligence +# Applies to both tiers on every cycle. # -# ── CROSS-CUTTING INTELLIGENCE ──────────────────────────────────────────────────────────────── # Startup grace period — no restarts for WATCHDOG_STARTUP_GRACE seconds after boot -# Dependency ordering — waits for dependencies before restarting a dependent container -# Restart loop protect — skip list after WATCHDOG_CONTAINER_RESTART_LIMIT in rolling window -# Skip list auto-clear — clears when container is seen running again +# Dependency ordering — dependency restarted first, dependent skipped this cycle +# Restart loop protect — skip list after WATCHDOG_CONTAINER_RESTART_LIMIT in window +# Skip list auto-clear — removed when container seen running again # Notification batching — one summary per cycle, not one ping per event # Parity awareness — skips restart actions during parity check -# Timeout protection — all docker commands wrapped in timeout — daemon hangs cannot -# stall the watchdog and leave containers unmonitored -# Docker daemon check — first check every cycle; hung daemon → strike system → -# restart daemon via /etc/rc.d/rc.docker → verify recovery -# system_watchdog.sh handles escalation if restart fails -# Quiet when healthy — only logs when something needs attention (plus heartbeat) +# Timeout protection — all docker commands wrapped in timeout +# Docker daemon check — each cycle begins with daemon health check; hung daemon → +# restart via rc.docker → system_watchdog.sh escalates if needed +# RAM emergency defer — reads SYS_WATCHDOG_STATE_FILE; stands down while +# system_watchdog.sh is managing a RAM emergency # -# ── STATE FILES ─────────────────────────────────────────────────────────────────────────────── -# WATCHDOG_STATE_FILE — strike counts (/tmp — resets on reboot, correct) -# SYS_WATCHDOG_FAILED_FILE — skip list (/boot — survives reboots, intentional) +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Tiered Monitoring +# Not all containers need the same monitoring strategy. Tier 1 gives explicit +# control over the containers that matter most. Tier 2 is the catch-all that +# requires no configuration and protects everything else. +# +# Strike vs Immediate +# CPU spikes and HTTP failures are transient — brief spikes are normal during +# transcoding or library scans. Memory leaks are not transient. CPU and HTTP +# use a strike system to distinguish sustained problems from momentary ones. +# Memory triggers immediate restart because a container at its ceiling is +# actively leaking, not spiking. +# +# Loop Protection Over Persistence +# A watchdog that keeps restarting a broken container is not helpful — it risks +# making a database corruption worse. After WATCHDOG_CONTAINER_RESTART_LIMIT +# attempts the container is skip-listed and the operator is notified. Automated +# recovery stops. Human investigation begins. +# +# Dependency-Safe Ordering +# When a container and its dependency are both down, restart the dependency +# first and skip the dependent this cycle. Prevents false-alarm skip-listing +# of containers whose only failure was starting before their dependency was ready. +# +# Silent When Healthy +# Runs 96 times per day. Producing output on every clean cycle would make +# logs useless. Output only when something needs attention or a heartbeat fires. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Startup Grace Period +# Restart actions suppressed for WATCHDOG_STARTUP_GRACE seconds after the +# watchdog starts. Checks still run and log — only restarts are suppressed. +# Prevents false-positive restarts while containers are still initialising. +# +# Restart Loop Protection +# WATCHDOG_CONTAINER_RESTART_LIMIT restarts within WATCHDOG_CONTAINER_RESTART_WINDOW +# hours triggers skip-listing and a critical notification. Skip list persists on +# /boot/config/ — survives reboots intentionally. Auto-clears when container +# is seen running again. +# +# Docker Daemon Health Check +# First operation every cycle. Daemon not responding within DOCKER_TIMEOUT → +# restart via /etc/rc.d/rc.docker → verify recovery. If still hung: log +# critical, skip cycle. system_watchdog.sh handles further escalation. +# +# RAM Emergency Deferral +# Reads SYS_WATCHDOG_STATE_FILE each cycle. If system_watchdog.sh has set +# mem_shutdown_active=true, all restart logic defers until the flag clears. +# Stale state guard: if file is >2 hours old with flag still set, +# system_watchdog.sh has likely stopped — watchdog resumes normal operation. +# +# Timeout Protection +# All docker commands wrapped in timeout. Daemon hangs cannot stall the +# watchdog and leave containers unmonitored between cycles. +# +# Notification Batching +# Events collected across a full cycle and sent as a single summary. +# Prevents notification floods when a shared dependency failure cascades. +# +# ============================================================================================== +# STATE FILES +# ============================================================================================== +# +# WATCHDOG_STATE_FILE — strike counts (default: /tmp — resets on reboot) +# SYS_WATCHDOG_FAILED_FILE — skip list (default: /boot/config — survives reboots) # WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection +# SYS_WATCHDOG_STATE_FILE — shared state with system_watchdog.sh (RAM emergency flag) # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── -# HOST*_WATCHDOG_CONTAINERS — memory hard limits per container -# HOST*_WATCHDOG_CONTAINER_URLS — HTTP health check URLs -# HOST*_WATCHDOG_REQUIRED_CONTAINERS — must always be running -# HOST*_WATCHDOG_SCAN_IGNORE — skip in Tier 2 scan -# HOST*_WATCHDOG_DEPENDENCIES — dependency ordering for restart decisions -# All aliased by detect_hosts() — script uses unprefixed names +# /tmp files reset on reboot — correct, pre-reboot strike counts are meaningless after it. +# /boot/config files survive reboots — correct, a skip-listed container is still broken after one. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# +# HOST*_WATCHDOG_CONTAINERS +# Memory hard limits per container. Format: "ContainerName:LimitInMB" +# Aliased by detect_hosts() → WATCHDOG_CONTAINERS +# +# HOST*_WATCHDOG_CONTAINER_URLS +# HTTP health check endpoints. Format: "ContainerName:http://host:port" +# Aliased by detect_hosts() → WATCHDOG_CONTAINER_URLS +# +# HOST*_WATCHDOG_REQUIRED_CONTAINERS +# Containers that must always be running. Aliased by detect_hosts() → +# WATCHDOG_REQUIRED_CONTAINERS +# +# HOST*_WATCHDOG_SCAN_IGNORE +# Containers excluded from Tier 2 global scan. Aliased by detect_hosts() → +# WATCHDOG_SCAN_IGNORE +# +# HOST*_WATCHDOG_DEPENDENCIES +# Dependency ordering. Format: "Dependent:dep1 dep2". Aliased by +# detect_hosts() → WATCHDOG_DEPENDENCIES +# +# master.conf # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# SOFT_CPU_THRESHOLD / HARD_CPU_THRESHOLD / CPU_FAIL_LIMIT -# SOFT_MEM_THRESHOLD -# RESP_FAIL_LIMIT / CURL_TIMEOUT # DOCKER_WATCHDOG_INTERVAL -# DOCKER_WATCHDOG_HEARTBEAT / DOCKER_WATCHDOG_HEARTBEAT_HOURS +# Seconds between full watchdog cycles (default: 900) +# +# WATCHDOG_STARTUP_GRACE +# Seconds before restart actions begin after watchdog starts (default: 600) +# +# SOFT_MEM_THRESHOLD +# Warn at this % of hard memory limit — no restart (default: 80) +# +# SOFT_CPU_THRESHOLD / HARD_CPU_THRESHOLD / CPU_FAIL_LIMIT +# CPU monitoring thresholds and strike limit +# +# CURL_TIMEOUT / RESP_FAIL_LIMIT +# HTTP health check timeout and consecutive failure limit +# # WATCHDOG_SCAN_ALL +# Enable Tier 2 global health scan (default: true) +# # WATCHDOG_RESTART_UNHEALTHY / WATCHDOG_RESTART_DEAD / WATCHDOG_RESTART_CRASHED -# WATCHDOG_NOTIFY_OOM / WATCHDOG_NOTIFY_CRASHLOOP -# WATCHDOG_CRASH_LIMIT -# WATCHDOG_STARTUP_GRACE +# Tier 2 action toggles +# +# WATCHDOG_NOTIFY_OOM / WATCHDOG_NOTIFY_CRASHLOOP / WATCHDOG_CRASH_LIMIT +# OOM and crash loop detection toggles and threshold +# # WATCHDOG_CONTAINER_RESTART_LIMIT / WATCHDOG_CONTAINER_RESTART_WINDOW +# Restart loop protection: attempt limit and rolling window in hours +# # WATCHDOG_BATCH_NOTIFY -# WATCHDOG_STATE_FILE / SYS_WATCHDOG_FAILED_FILE / WATCHDOG_CONTAINER_RESTART_LOG +# Collect cycle events and send as one notification (default: true) +# +# DOCKER_WATCHDOG_HEARTBEAT_HOURS +# Hours between alive heartbeat log entries +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# docker_watchdog.sh +# Start continuous monitoring loop — normally launched by array_started.sh +# +# docker_watchdog.sh --dry-run +# Run a full watchdog cycle without restarting anything. Shows what would +# happen based on current container states. Use to verify configuration. +# +# docker_watchdog.sh --status +# Show strike counts, skip list contents, grace period status, RAM emergency +# deferral state, and last cycle timing. Then exit. +# +# docker_watchdog.sh --log +# Verbose output — full detail for every container checked and every decision. +# Use to debug why a container is or is not being restarted. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# docker_watchdog.sh — normal start (continuous loop) -# docker_watchdog.sh --dry-run — preview without restarting -# docker_watchdog.sh --status — show config and exit -# docker_watchdog.sh --log — verbose output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Docker_Essentials/docker_weekly_restart.sh b/Docker_Essentials/docker_weekly_restart.sh index da7df64..a188675 100644 --- a/Docker_Essentials/docker_weekly_restart.sh +++ b/Docker_Essentials/docker_weekly_restart.sh @@ -2,52 +2,79 @@ # ============================================================================================== # ================================= Docker Weekly Restart ====================================== # ============================================================================================== -# Restarts all running containers in HOST*_WEEKLY_RESTART_CONTAINERS. -# Called by weekly_sync_maintenance.sh via WEEKLY_MAINTENANCE_SCRIPTS every Sunday at 2:30am. -# Can also be run manually for ad hoc weekly restarts. # -# ── CONTEXT ─────────────────────────────────────────────────────────────────────────────────── -# weekly_sync_maintenance.sh stops containers before syncing and restarts them after. -# This script runs AFTER that restart — targeting a different set of less critical services -# that benefit from a weekly restart but don't need to be stopped for the sync itself. -# These containers are typically already running when this script executes. +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Restarts configured containers every Sunday at 2:30am as proactive maintenance. # -# ── BEHAVIOUR ───────────────────────────────────────────────────────────────────────────────── -# Running containers → docker restart (graceful stop + start) -# Stopped containers → left stopped — was down intentionally, do not bring back up -# Missing containers → logged and skipped — not treated as fatal -# Each action uses RETRY_COUNT + SLEEP from master.conf for retry logic. +# Called by weekly_sync_maintenance.sh via WEEKLY_MAINTENANCE_SCRIPTS. Runs after +# the sync window has completed and already restarted its own critical containers +# (Emby, auth stack). Targets a separate set of less-critical services that benefit +# from a weekly clean start but do not need to be stopped for the sync itself. # -# The "was running → restart, was stopped → leave stopped" rule is consistent -# across the entire ecosystem — container state is always respected. +# Same behavioural rules as docker_daily_restart.sh: running → restart, +# stopped → leave, missing → skip. Container state is always respected. # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# Dependency ordering — containers restart in dependency-safe order using -# HOST*_WATCHDOG_DEPENDENCIES from master_host*.conf. Dependencies restart -# first with CONTAINER_DELAY before their dependents. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# Restart verification — after each restart, container state is checked after a short -# settle period. If the container fails to stay running it is marked as failed and -# a notification is sent rather than silently passing. +# Dependency Ordering +# Containers restart in dependency-safe order using HOST*_WATCHDOG_DEPENDENCIES. +# CONTAINER_DELAY seconds between dependency restart and dependent restart. # -# Timeout protection — all docker commands are wrapped in a 30 second timeout. -# A hung Docker daemon cannot cause this script to hang indefinitely. +# Restart Verification +# Container state checked after a settle period. A container that crashes +# immediately after restart is marked failed with a notification sent. # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── -# HOST*_WEEKLY_RESTART_CONTAINERS — list of containers to restart weekly -# HOST*_WATCHDOG_DEPENDENCIES — dependency ordering for restart sequence -# Set by detect_hosts() alias → WEEKLY_RESTART_CONTAINERS used by this script +# Timeout Protection +# All docker commands wrapped in a 30 second timeout. A hung Docker daemon +# cannot cause this script to hang indefinitely. # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# RETRY_COUNT — retry attempts before giving up on a container -# SLEEP — seconds between retry attempts -# CONTAINER_DELAY — seconds to wait between dependency and dependent restart +# Lock Acquisition +# acquire_lock() prevents concurrent execution. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# +# HOST*_WEEKLY_RESTART_CONTAINERS +# Containers restarted weekly. Aliased by detect_hosts() → +# WEEKLY_RESTART_CONTAINERS +# +# HOST*_WATCHDOG_DEPENDENCIES +# Dependency ordering shared with docker_watchdog.sh. Aliased by +# detect_hosts() → WATCHDOG_DEPENDENCIES +# +# master.conf +# +# RETRY_COUNT +# Retry attempts before giving up on a container +# +# SLEEP +# Seconds between retry attempts +# +# CONTAINER_DELAY +# Seconds to wait after restarting a dependency before starting its dependents +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# docker_weekly_restart.sh +# Restart all containers in WEEKLY_RESTART_CONTAINERS +# +# docker_weekly_restart.sh --dry-run +# Preview which containers would be restarted and which would be skipped +# +# docker_weekly_restart.sh --status +# Show configured restart list, container states, and dependency ordering +# +# docker_weekly_restart.sh --log +# Verbose per-container execution output # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# docker_weekly_restart.sh — normal restart -# docker_weekly_restart.sh --dry-run — preview without restarting -# docker_weekly_restart.sh --log — verbose output -# docker_weekly_restart.sh --status — show config and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Docker_Essentials/downloaders_reset.sh b/Docker_Essentials/downloaders_reset.sh index 4cd2948..13d2c1b 100644 --- a/Docker_Essentials/downloaders_reset.sh +++ b/Docker_Essentials/downloaders_reset.sh @@ -176,6 +176,56 @@ fi [[ -n "$SABNZBD_URL" ]] && log "SABnzbd active on $MY_ID" [[ -n "$QBIT_URL" ]] && log "qBittorrent active on $MY_ID" +# ============================================================================================== +# ━━━ slskd — Connection Check ━━━ +# ============================================================================================== +# slskd's internal watchdog doesn't always recover from disconnection. Check before +# running API-dependent sections; attempt reconnect if down. + +SLSKD_CONNECTED=false + +if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then + echo "" + echo "━━━ $ICON_SYNC slskd — Connection Check ━━━" + + _slskd_is_connected() { + local state + state=$(curl -sf --max-time 10 \ + -H "X-Api-Key: $SLSKD_API_KEY" \ + "$SLSKD_URL/api/v0/application" 2>/dev/null | \ + jq -r '.server.isConnected // false' 2>/dev/null) + [[ "$state" == "true" ]] + } + + if _slskd_is_connected; then + log "slskd connected to Soulseek ✅" + SLSKD_CONNECTED=true + else + warn "slskd disconnected — triggering reconnect" + curl -sf --max-time 10 -X PUT \ + -H "X-Api-Key: $SLSKD_API_KEY" \ + -H "Content-Type: application/json" \ + "$SLSKD_URL/api/v0/server" \ + -d '{"address":"server.slsknet.org","port":2242}' \ + >/dev/null 2>&1 + + _ELAPSED=0 + while [[ "$_ELAPSED" -lt 60 ]]; do + sleep 10 + _ELAPSED=$(( _ELAPSED + 10 )) + if _slskd_is_connected; then + log "slskd reconnected after ${_ELAPSED}s ✅" + SLSKD_CONNECTED=true + break + fi + log " waiting... (${_ELAPSED}s / 60s)" + done + + [[ "$SLSKD_CONNECTED" != true ]] && \ + warn "slskd still disconnected after 60s — skipping API-dependent sections" + fi +fi + # ============================================================================================== # ━━━ slskd — Stuck Searches ━━━ # ============================================================================================== @@ -183,7 +233,7 @@ fi # Prevents 409 Conflict error on next Soularr startup when it tries to # create a search with the same ID that already exists in a terminal state. -if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then +if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]] && [[ "$SLSKD_CONNECTED" == true ]]; then echo "" echo "━━━ 🔍 slskd — Stuck Searches ━━━" @@ -236,7 +286,7 @@ fi # Prevents Soularr 404 loop when polling a user whose transfer no longer exists. # Safety: NEVER removes transfers that are InProgress or Queued — active downloads protected. -if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then +if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]] && [[ "$SLSKD_CONNECTED" == true ]]; then echo "" echo "━━━ 🔍 slskd — Dead Transfer Records ━━━" diff --git a/Fallback/Manual-Fallback.md b/Fallback/Manual-Fallback.md new file mode 100644 index 0000000..3b1931a --- /dev/null +++ b/Fallback/Manual-Fallback.md @@ -0,0 +1,613 @@ +# ━━━━━ FALLBACK — Manual ━━━━━ + +Config reference, procedures, operational workflows. +For overview see README-Fallback.md. For per-script detail see script headers. + +--- + +## ━━━ CONFIGURATION — master.conf ━━━ + +``` +FALLBACK_ENABLED=true +``` +Enable or disable the entire fallback system. `false` = exit cleanly on startup — no +monitoring, no container actions. Set false when a server is being rebuilt or fallback +is temporarily suspended. Must be explicitly enabled when both servers are ready. +**(default: false)** + +--- + +``` +FALLBACK_CHECK_INTERVAL=30 +``` +Seconds between connectivity checks (ping remote + ping internet). Shorter = faster +detection and more pings. With `FALLBACK_HANDBACK_STRIKES=3` at 30s intervals: +fallback detected in ≤30s, handback requires 90s of continuous remote-up before +beginning. **(default: 30)** + +--- + +``` +FALLBACK_HANDBACK_STRIKES=3 +``` +Consecutive remote-up checks required before handback begins. Prevents false triggers +from brief network recovery during an ongoing outage. 3 strikes × 30s = 90s continuous +up required before handback sequence starts. **(default: 3)** + +--- + +``` +FALLBACK_STATE_FILE=/boot/config/fallback_state.db +``` +Path to the persistent state file. Lives on `/boot/` intentionally — survives reboots. +If the server was in FALLBACK state when it rebooted, it resumes FALLBACK on restart +rather than assuming everything is normal. + +--- + +``` +FALLBACK_RSYNC_ENABLED=true +``` +Gate for writeback rsync jobs during handback. Set `false` to hand back without syncing +any data — useful when the primary still has reliable last-known-good state and writeback +would be counterproductive. **(default: true)** + +--- + +``` +EXTERNAL_IP=8.8.8.8 +``` +IP pinged to verify internet connectivity. **(default: 8.8.8.8)** + +--- + +``` +FALLBACK_TEST_BLOCK_WAIT=60 +``` +Seconds fallback_test.sh waits in Phase 3 for fallback.sh to detect the simulated +outage. Must be greater than `FALLBACK_CHECK_INTERVAL` plus a buffer. At 30s interval: +use ≥60s. **(default: 60)** + +--- + +``` +FALLBACK_TEST_HANDBACK_WAIT=300 +``` +Seconds fallback_test.sh waits in Phase 6 for fallback.sh to complete the full handback +sequence. Must cover: strike confirmation window + pre-flight time + rsync duration + +container start time. At 3 strikes × 30s + ~2min rsync + ~1min container start: use ≥240s. +**(default: 300)** + +--- + +## ━━━ CONFIGURATION — master_host*.conf ━━━ + +``` +HOST*_DDNS_CONTAINERS=("...") +``` +DDNS containers managed by this host — the containers that update this server's DNS +records. Stopped when internet is lost. Started as the very last step of handback after +all primary containers are confirmed running. + +```bash +HOST1_DDNS_CONTAINERS=("Gmer4Lfe.com-DDNS") +HOST2_DDNS_CONTAINERS=("Gmer4Lfe.us-DDNS") +``` + +--- + +``` +FALLBACK_HOST*_STOP_ON_NO_NET=(...) +``` +Containers to stop when this host loses internet. Services that are meaningless without +internet connectivity. Most hosts leave this empty. + +--- + +``` +FALLBACK_HOST*_COVERS_HOST*_TIER1=(...) +FALLBACK_HOST*_COVERS_HOST*_TIER2=(...) +FALLBACK_HOST*_COVERS_HOST*_TIER3=(...) +FALLBACK_HOST*_COVERS_HOST*_TIER4=(...) +``` +Containers this host starts for the remote host when the remote is down. TIER1 starts +immediately. TIER2–4 activate after the corresponding delay thresholds. + +Variable pattern: `FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER${N}` + +The DDNS container for the remote's domain must be the first entry in TIER1 — DNS +coverage before anything else. + +--- + +``` +HOST*_TIER2_DELAY=240 +HOST*_TIER3_DELAY=720 +HOST*_TIER4_DELAY=1440 +``` +Minutes after FALLBACK state entry before activating each tier. Variable references the +**remote** host's ID — delays for a HOST1 outage use `HOST1_TIER*_DELAY`. + +```bash +HOST1_TIER2_DELAY=240 # 4 hours — covers most ISP and power issues +HOST1_TIER3_DELAY=720 # 12 hours — genuine extended outage +HOST1_TIER4_DELAY=1440 # 24 hours — arrs only worth starting at this threshold +``` + +--- + +``` +HOST*_TIER1_WRITEBACK_DELAY=60 +``` +Minimum outage duration in minutes before Tier 1 writeback runs on handback. Outages +shorter than this skip writeback entirely — the primary's last-known-good state is more +reliable than a short period of activity on the covering server. + +```bash +HOST1_TIER1_WRITEBACK_DELAY=60 # skip writeback for outages under 60 minutes +``` + +--- + +``` +FALLBACK_HOST*_WRITEBACK_TIER1=(...) +FALLBACK_HOST*_WRITEBACK_TIER2=(...) +FALLBACK_HOST*_WRITEBACK_TIER3=(...) +``` +Paths rsynced back to the remote on handback, per tier. Writeback runs only if the +outage exceeded the tier's activation delay. + +Variable pattern: `FALLBACK_${REMOTE_ID}_WRITEBACK_TIER${N}` + +**Tier 4 writeback** auto-uses the remote's `HOST*_DAILY_SYNC_SHARES` — the same list +daily_sync_maintenance.sh uses, in the opposite direction. No separate TIER4 list needed. + +--- + +## ━━━ FULL CONFIGURATION REFERENCE BLOCKS ━━━ + +### master.conf + +```bash +FALLBACK_ENABLED=true +FALLBACK_CHECK_INTERVAL=30 +FALLBACK_HANDBACK_STRIKES=3 +FALLBACK_STATE_FILE=/boot/config/fallback_state.db +FALLBACK_RSYNC_ENABLED=true +EXTERNAL_IP=8.8.8.8 +FALLBACK_TEST_BLOCK_WAIT=60 +FALLBACK_TEST_HANDBACK_WAIT=300 +``` + +--- + +### master_host2.conf — HOST2 covering HOST1 + +```bash +HOST2_DDNS_CONTAINERS=("Gmer4Lfe.us-DDNS") + +FALLBACK_HOST2_STOP_ON_NO_NET=() + +# Tier 1 — immediate (vital services + Live TV) +FALLBACK_HOST2_COVERS_HOST1_TIER1=( + "Gmer4Lfe.com-DDNS" # ALWAYS FIRST — DNS coverage before anything else + "Emby" # media server — people are watching + "NginxProxyManager" # reverse proxy — all external access routes through this + "Lldap-Gmer4Lfe" # user directory — already warm, verify and keep + "Mariadb-Authelia" # auth database — already warm, verify and keep + "Redis-Authelia" # auth session cache — already warm, verify and keep + "Authelia" # SSO — already warm, serving users already + "VaultWarden" # passwords — people lock themselves out without this + "Dispatcharr" # Live TV scheduler — people are watching right now + "Dispatcharr-Basic" + "Dispatcharr-Iptv-Users" + "ErsatzTV-Emby" # channel schedule builder +) + +# Tier 2 — after 4 hours (shared productivity services) +FALLBACK_HOST2_COVERS_HOST1_TIER2=( + "Postgres-NextCloud" # must start before NextCloud + "NextCloud" + "PostgreSQL-Immich" # must start before Immich + "Immich-Gmer4Lfe" + "Jellyseerr" +) + +# Tier 3 — after 12 hours (secondary services) +FALLBACK_HOST2_COVERS_HOST1_TIER3=( + "Organizrv2-Gmer4Lfe" + "AdGuard-Home" + "UptimeKuma" + "Gitea" + "Collabora-CODE" +) + +# Tier 4 — after 24 hours (arrs + downloaders) +FALLBACK_HOST2_COVERS_HOST1_TIER4=( + "Sonarr-Gmer4Lfe" + "Radarr-Gmer4Lfe" + "Lidarr-Gmer4Lfe" + "Prowlarr-Gmer4Lfe" + "SABnzbd-Gmer4Lfe" + "qBittorrent-Gmer4Lfe" + "LidaTube" + "Pinchflat" +) + +# Tier delays for HOST1 outage +HOST1_TIER2_DELAY=240 +HOST1_TIER3_DELAY=720 +HOST1_TIER4_DELAY=1440 +HOST1_TIER1_WRITEBACK_DELAY=60 + +# Writeback paths — synced back to HOST1 on handback +FALLBACK_HOST1_WRITEBACK_TIER1=( + "/mnt/user/appdata-Failover/Critical-Data" # auth stack — Authelia + NPM + certs + "/mnt/user/Media_Server/Emby" # Emby userdata — watch history, playstates +) + +FALLBACK_HOST1_WRITEBACK_TIER2=( + "/mnt/user/appdata-Failover/Important-Data" # NextCloud + Postgres + Immich +) + +FALLBACK_HOST1_WRITEBACK_TIER3=( + "/mnt/user/appdata-Failover/Gmer4Lfe" # secondary appdata accumulated changes +) + +# Tier 4 writeback uses HOST1_DAILY_SYNC_SHARES automatically — no list needed here. +``` + +--- + +### master_host1.conf — HOST1 covering HOST2 + +```bash +HOST1_DDNS_CONTAINERS=("Gmer4Lfe.com-DDNS") + +FALLBACK_HOST1_STOP_ON_NO_NET=() + +# Tier 1 — immediate (HOST2's vital services) +FALLBACK_HOST1_COVERS_HOST2_TIER1=( + "Gmer4Lfe.us-DDNS" # ALWAYS FIRST + # HOST2's Tier 1 services — fill per HOST2's stack +) + +FALLBACK_HOST1_COVERS_HOST2_TIER2=(...) +FALLBACK_HOST1_COVERS_HOST2_TIER3=(...) +FALLBACK_HOST1_COVERS_HOST2_TIER4=(...) + +# Tier delays for HOST2 outage +HOST2_TIER2_DELAY=240 +HOST2_TIER3_DELAY=720 +HOST2_TIER4_DELAY=1440 +HOST2_TIER1_WRITEBACK_DELAY=60 + +FALLBACK_HOST2_WRITEBACK_TIER1=( + # HOST2's critical data paths +) + +# Tier 4 writeback uses HOST2_DAILY_SYNC_SHARES automatically. +``` + +--- + +## ━━━ STATE FILE REFERENCE ━━━ + +Location: `/boot/config/fallback_state.db` (survives reboots) + +``` +state=NORMAL # NORMAL | FALLBACK | NO_INTERNET | DARK +fallback_start=0 # epoch timestamp when FALLBACK began (0 = not in FALLBACK) +handback_strikes=0 # consecutive remote-up checks accumulated toward handback +tier2_started=false # whether Tier 2 containers started this event +tier3_started=false # whether Tier 3 containers started +tier4_started=false # whether Tier 4 containers started +``` + +View state: `cat /boot/config/fallback_state.db` +Check state: `fallback.sh --status` + +The file is managed exclusively by fallback.sh. Do not edit it while fallback.sh is +running — the next cycle will overwrite your changes. Use the Manual State Reset +procedure (below) when fallback.sh is not running. + +--- + +## ━━━ INITIAL SETUP ━━━ + +Before fallback.sh can work on both servers: + +### 1. Tailscale Connected on Both Servers + +```bash +# Verify from HOST1 +tailscale ip -4 unRAID-Jayred365 +tailscale ping unRAID-Jayred365 + +# Verify from HOST2 +tailscale ip -4 unRAID-Gmer4Lfe +tailscale ping unRAID-Gmer4Lfe +``` + +### 2. SSH Keys Configured — No Password Prompt + +```bash +# From HOST1 — should print HOST2's hostname without password prompt +ssh -i /root/.ssh/Gmer4Lfe-rsync-key root@[HOST2-tailscale-ip] "hostname" + +# From HOST2 — should print HOST1's hostname +ssh -i /root/.ssh/Jayred365-rsync-key root@[HOST1-tailscale-ip] "hostname" +``` + +### 3. Container Names Match + +Fallback containers must exist (but stopped) on the covering server, with volume mounts +pointing at mirrored share paths. + +```bash +# Verify on HOST2 that HOST1's container exists (stopped is expected) +ssh root@[HOST2-tailscale-ip] "docker inspect Emby --format '{{.State.Status}}'" +# Expected: created or exited — NOT "no such container" +``` + +### 4. Critical Data Mirrored + +These shares must exist on HOST2 with current data from HOST1 before failover is needed: + +``` +/mnt/user/appdata-Failover/Critical-Data # auth stack — NPM, LLDAP, Authelia, certs +/mnt/user/appdata-Failover/Important-Data # NextCloud + Postgres + Immich +/mnt/user/Media_Server/Emby # Emby userdata — watch history, playstates +/mnt/user/appdata-Failover/Gmer4Lfe # server appdata +``` + +```bash +# Verify data is current — check modification times +ssh root@[HOST2-tailscale-ip] "ls -la /mnt/user/appdata-Failover/Critical-Data/" +``` + +Sync is maintained continuously by `daily_sync_maintenance.sh` critical-data profile. + +### 5. DDNS TTL Set to 1 Minute + +Set in your DDNS provider settings. Higher TTL means users continue hitting the old IP +for longer after failover. At 5-minute TTL, users can be hitting a downed server for up +to 5 minutes before DNS switches. + +### 6. Both Servers Running fallback.sh + +Fallback only works in one direction if only one server is running the script. Both +servers must be running it continuously for mutual coverage. + +```bash +# Verify fallback.sh is running +pgrep -f "fallback.sh" + +# Check the state file +cat /boot/config/fallback_state.db +``` + +Start via User Scripts plugin on both servers. + +--- + +## ━━━ MONITORING ━━━ + +**Sunday coffee report** (`sunday_morning_coffee_report.sh`) — Fallback section shows +current state, outage duration if not NORMAL, Tailscale reachability, and whether +fallback.sh is running. + +**Weekly health digest** (`weekly_health_digest.sh`) — reads the state file. If +`DIGEST_SMART_ON_FAILOVER=true` and state is not NORMAL, it sends a notification even +in smart mode — a non-NORMAL state at digest time needs attention. + +**Direct check:** + +```bash +fallback.sh --status # full state snapshot +cat /boot/config/fallback_state.db # raw state file +``` + +--- + +## ━━━ PROCEDURES ━━━ + +### Running the Failover Test + +> This starts and stops real containers on both servers. Users will experience a brief +> service interruption. Always run `--dry-run` first. + +```bash +# Step 1 — verify all phases without touching anything +fallback_test.sh --dry-run + +# Step 2 — schedule maintenance window, then run live +fallback_test.sh + +# Step 3 — check state after test completes +fallback.sh --status +cat /boot/config/fallback_state.db +``` + +If the test doesn't complete cleanly, the state file may be left in FALLBACK. The +iptables safety trap in fallback_test.sh removes the DROP rule on any exit, so remote +connectivity is always restored. Use the Manual State Reset procedure below if the state +file is stuck. + +### Manual State Reset + +Use when the state file is stuck in a non-NORMAL state after testing, a failed handback, +or killing fallback.sh directly (not via User Scripts Abort). + +**Before resetting, verify the situation is actually safe to reset:** + +```bash +# 1. Right containers running on the right server +docker ps | grep -E "Emby|VaultWarden|NginxProxyManager" + +# 2. DDNS pointing at the correct server +nslookup Gmer4Lfe.com 8.8.8.8 +nslookup Gmer4Lfe.us 8.8.8.8 + +# 3. Both servers visible on Tailscale +tailscale ping [remote-tailscale-ip] + +# 4. No actual fallback in progress (remote is genuinely up and stable) +ping -c 5 [remote-tailscale-ip] +``` + +**Stop fallback.sh first (via User Scripts Abort), then reset:** + +```bash +# View current state +cat /boot/config/fallback_state.db + +# Write a clean NORMAL state +cat > /boot/config/fallback_state.db << 'EOF' +state=NORMAL +fallback_start=0 +handback_strikes=0 +tier2_started=false +tier3_started=false +tier4_started=false +EOF + +# Verify the write +cat /boot/config/fallback_state.db +``` + +Restart fallback.sh via User Scripts plugin. It will resume from NORMAL on its next cycle. + +> **Warning:** Do NOT reset during an actual fallback event. fallback.sh will think +> everything is normal and stop covering the remote — services go offline until the next +> detection cycle catches it again. + +### Adding a Container to a Tier + +1. Create the container on the covering server (stopped), with volume mounts pointing at + the mirrored share path (e.g. `/mnt/user/Movies` must exist on the covering server) +2. Add the container name to `FALLBACK_HOST*_COVERS_HOST*_TIER*` in master_host*.conf + in the appropriate tier position (dependency ordering — databases before apps) +3. Verify: `fallback.sh --status` shows the container in the expected tier list +4. Run `fallback_test.sh --dry-run` to confirm the full configuration is valid + +--- + +## ━━━ TROUBLESHOOTING ━━━ + +### Fallback Not Triggering + +``` +Is fallback.sh running on the covering server? + → User Scripts plugin → check status of the fallback script + → pgrep -f "fallback.sh" + +Is Tailscale connected? + → tailscale status (should show the remote peer) + +Can this server reach the remote Tailscale IP? + → ping [remote-tailscale-ip] + +What does fallback.sh report? + → fallback.sh --status + → cat /boot/config/fallback_state.db +``` + +### Handback Not Completing + +``` +Is the primary's array fully started? + → ls /mnt/user (should show share directories) + +Is Docker responding on the primary? + → docker ps (should return a list, not hang) + +Is rootfs nearly full? (pre-flight checks this) + → df / + +Is rsync running and stuck? + → pgrep rsync + → A stalled rsync blocks handback. Let fallback.sh retry next cycle. +``` + +### DDNS Not Cutting Over + +``` +Is the DDNS container running on the covering server? + → docker ps | grep DDNS + +What TTL is the DNS record set to? + → nslookup Gmer4Lfe.com 8.8.8.8 (check TTL in response) + → High TTL = slow propagation + +Is the DDNS provider accepting updates? + → docker logs [ddns-container] --tail 50 +``` + +### Split Brain — Both DDNS Running + +``` +This should not happen if the handback sequence completed correctly. +If it has happened: + +1. Check both servers for running DDNS containers + HOST1: docker ps | grep DDNS + HOST2: docker ps | grep DDNS + +2. Stop the duplicate (the one on the covering server) + docker stop [duplicate-ddns-container] + +3. Understand the state before resetting + fallback.sh --status + cat /boot/config/fallback_state.db + +4. Perform Manual State Reset above on the server in a bad state + +5. Restart fallback.sh via User Scripts plugin +``` + +--- + +## ━━━ FLAG REFERENCE ━━━ + +### fallback.sh + +`fallback.sh` +Normal start — continuous loop. Start via User Scripts plugin or array_start.sh. **Do NOT +stop by killing the process** — state file may be left inconsistent. Stop via User Scripts +Abort only. + +`fallback.sh --dry-run` +Walk through one full cycle showing what would happen based on current network state. No +containers started or stopped. No DDNS changes. Use to verify configuration before relying +on it. + +`fallback.sh --status` +Show current state, identity, DDNS containers, check interval, handback strikes, and +(if in FALLBACK) outage duration and tier activation status. Use this first to understand +current state before any manual intervention. + +`fallback.sh --log` +Verbose output on every decision in every cycle — ping results, state evaluation, tier +checks. Use when debugging why the state machine is or is not acting as expected. + +--- + +### fallback_test.sh + +`fallback_test.sh --dry-run` +Walk through all 7 phases with full output. No iptables rules added. No container +starts or stops. **Always run this before a live test** — confirms timing configuration +is correct and phases would pass before committing to real changes. + +`fallback_test.sh` +Full live test — real iptables DROP rule, real container lifecycle. Users will experience +a brief service interruption. Run during a maintenance window. The iptables safety trap +removes the DROP rule on any exit (normal completion, crash, ctrl-c). + +`fallback_test.sh --status` +Show current fallback state, test timing configuration, and the Tier 1 containers that +would be tested. No test run. + +`fallback_test.sh --log` +Verbose output on every check in every phase. diff --git a/Fallback/README-Failover.md b/Fallback/README-Failover.md deleted file mode 100644 index a3381d0..0000000 --- a/Fallback/README-Failover.md +++ /dev/null @@ -1,1165 +0,0 @@ -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# 🔀 FAILOVER -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -**Mutual automatic failover between two fully independent unRAID servers.** -When one server goes down the other starts its containers, cuts over DNS, and keeps -users online. When it comes back everything hands back in the correct sequence — DDNS -first off, rsync, containers, DDNS last on — so users only hit the returning server -after it is actually ready to serve them. - -> **Built from scratch. Refined through a year of production testing.** The DDNS -> sequencing and handback order were the hardest parts to get right. Both directions -> are exercised regularly with `failover_test.sh`. The logic is documented here in -> full so it is never lost and never has to be rediscovered. - ---- - -## ━━━ THE PROBLEM THAT BUILT THIS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Running a self-hosted media and productivity stack means being the operator. When -something goes down — power cut, ISP failure, hardware fault — it is your problem. -Cloud services have entire infrastructure teams and redundant data centers. A home -unRAID server has one power outlet, one ISP, and one point of failure. - -The specific problems that led to building this: - ---- - -### 🔴 A Single Point of Failure for an Entire Household - -HOST1 runs Emby, NginxProxyManager, Authelia, NextCloud, VaultWarden, and every -other service the household uses daily. When HOST1 goes down — even briefly — all of -those services go down with it. Everyone in the household loses access simultaneously. -The one place everything lives is the one thing that can fail. - -The fix: a second server with mirrored critical data and the ability to start any of -HOST1's containers. The second server covers the first automatically. From a user's -perspective, a brief interruption and then everything is back — not hours of downtime. - ---- - -### 🔴 DNS Cutting Over Before the Server Was Ready - -Early attempts at failover started the containers on the covering server and then -updated DNS. The problem: DNS propagation happens in under a minute. Users hit the -new IP before Emby had finished starting, before Authelia had loaded its sessions, -before NPM had loaded its proxy configurations. Emby reconnection attempts failed. -Users who were mid-stream got dropped and couldn't reconnect. The auth wall was up -before the service behind it was. - -The fix: warm standby for the auth stack. NPM, LLDAP, and Authelia run actively on -both servers at all times — not cold-started at failover. When DNS cuts over, auth -is already running and ready. The 30-60 second dead zone after failover disappeared. - ---- - -### 🔴 Split Brain DNS During Handback - -When HOST1 returned after being down, the obvious thing to do was: start HOST1's -containers, then switch DNS back. The problem: between "start containers" and "DNS -switches" there is a window where HOST1's DDNS container and HOST2's DDNS container -are both running, both updating the same domain record with different IPs. Users -during that window get routed randomly between two servers — some to the primary that -has fresh data, some to the covering server that might be mid-cleanup. Authentication -sessions don't transfer between servers. This is split brain and it causes exactly -the intermittent failures that are hardest to diagnose. - -The fix: stop DDNS on the covering server first, before anything else moves. Nothing -starts on HOST1 until HOST2's DDNS is confirmed stopped. DNS is always owned by -exactly one server. There is no window where two DDNS containers are updating the -same record. - ---- - -### 🔴 Rsync Running Into Active Container I/O - -Early versions synced data back to HOST1 while containers were still running on HOST2 -— the logic being "minimise downtime, sync while active." The problem: rsync competing -with active container I/O is slower. Files changing mid-transfer can cause -inconsistency. Database writes during sync risk dirty state on the returned server. -The shorter-downtime approach paradoxically created a higher risk of problems on -handback. - -The fix: stop containers before syncing. The window where containers are down is only -the rsync duration — typically a few minutes. Clean static source at full bandwidth. -Predictable, consistent handback state every time. - ---- - -### 🔴 No Way to Validate the System Before Needing It - -A failover system that has never been tested is not a failover system — it is a hope. -The only honest test is to actually pull the plug and see what happens. But doing that -on a live production server means real users experience real downtime, and if something -is misconfigured the downtime is extended while you debug it. - -The fix: `failover_test.sh`. A controlled simulation that uses an iptables rule to -make the remote server appear unreachable — triggering the full failover sequence -without actually taking anything offline. The test validates every phase: detection, -container start, handback, container stop, DNS return. A safety trap removes the -iptables rule on any exit — crash, error, ctrl-c, or clean completion. The remote is -always reachable after the test regardless of what happened during it. - ---- - -## ━━━ THE SETUP ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -``` -HOST1 — unRAID-Gmer4Lfe - Hardware: Threadripper 1950X, 128GB RAM, ZFS cache pools - Location: Primary site - DDNS: Gmer4Lfe.com - Role: Primary — full service stack + source of truth for Movies/Shows/Music - -HOST2 — unRAID-Jayred365 - Hardware: Intel i5 10th gen, 64GB RAM — completely different hardware - Location: Remote — 50 miles away - DDNS: Gmer4Lfe.us - Role: Secondary — own stack + covers HOST1 + mirrors critical data -``` - -**Hardware does not need to match.** Everything is accessed through `/mnt/user/` — -unRAID's fused share layer. HOST1 has a Threadripper with ZFS pools. HOST2 has a -completely different CPU, fewer drives, different layout. Failover containers on HOST2 -mount `/mnt/user/Movies` and see mirrored data because the share names match. The -hardware underneath is irrelevant. - -**What must match between servers:** - -``` -Share names /mnt/user/Movies must exist on both servers (mirrored data) -Container names "Emby" on HOST2 must be the container HOST2 starts for HOST1 -Network names Docker custom networks must have matching names for NPM routing -``` - ---- - -### ── Split Source of Truth — No Conflicts ──────────────────────────────────── - -Both servers run arr instances simultaneously with zero conflict — because they manage -completely different shares: - -``` -HOST1 owns source of truth for: HOST2 owns source of truth for: - /mnt/user/Movies (Radarr) /mnt/user/Anime_Movies (his Radarr) - /mnt/user/Tv_Shows (Sonarr) /mnt/user/Anime_Shows (his Sonarr) - /mnt/user/Music (Lidarr) - -Each server mirrors the other's shares: - HOST1 ← pulls Anime from HOST2 - HOST2 ← pulls Movies/Shows/Music from HOST1 -``` - -**The rule:** never run two instances of the same arr against the same share -simultaneously. Different arrs managing different shares is fine. Scheduling keeps -them clean even within the same window: - -```bash -# master.conf scheduling notes -# ───────────────────────────────────────────────────────────────────────────── -# HOST2 arrs: midnight → noon managing and downloading anime -# HOST1 Tdarr: 12:30 → 23:00 transcoding anime from HOST2's share -# -# They operate in non-overlapping windows even on the same share. -# HOST1 is processing content that HOST2 already downloaded. -``` - ---- - -### ── Auth Stack — Warm on Both Servers ─────────────────────────────────────── - -NPM, LLDAP, and Authelia run actively on both servers at all times. HOST2 needs them -running to serve his users through his domain every day — this is not a failover-only -configuration. HOST1 is source of truth: all changes (proxy rules, user accounts, -certs, Authelia policies) mirror to HOST2 every 15 minutes via critical sync. - -``` -Running warm on both servers always: - NginxProxyManager ← serving both domains continuously - LLDAP ← authenticating all users continuously - Authelia ← protecting all services continuously - Certs ← mirrored, valid, already loaded - -Started from stopped only at failover: - Emby ← media server (Tier 1 — immediate) - Remote DDNS ← DNS updater for the covered domain (Tier 1 — immediate) - Dispatcharr x3 ← Live TV schedulers (Tier 1 — people are watching) - ...and tiered services beyond Tier 1 -``` - -Running warm eliminates the 30-60 second dead zone where auth is coming up after DNS -has already cut over. By the time users hit the covering server, auth is already ready. - ---- - -## ━━━ HOW IT WORKS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Both servers run `failover.sh` independently as a continuous background process. They -do not coordinate with each other. There is no SSH signaling between servers, no shared -state file, no election algorithm. Each server makes all of its decisions based entirely -on two pings from its own network perspective: - -```bash -# Every FAILOVER_CHECK_INTERVAL seconds (default 120s): -ping -c 1 -W 3 $REMOTE_TAILSCALE_IP # Is the other server reachable? -ping -c 1 -W 3 $EXTERNAL_IP # Do I have internet? (default: 8.8.8.8) -``` - -The combination of those two answers determines the current state. That is the entire -input to the decision logic. Pure, observable facts from each server's own vantage point. - ---- - -## ━━━ STATES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -### ✅ NORMAL - -``` -Remote ping: PASS Internet ping: PASS -``` - -Both servers running normally. Each server runs its own containers. Own DDNS ON — -pointing DNS at this server's IP. Silent operation — no output on clean cycles. - ---- - -### 🔴 FAILOVER - -``` -Remote ping: FAIL Internet ping: PASS -``` - -The remote server is unreachable but this server has internet. The remote's containers -need to be started locally — users are hitting DNS records that point to a server that's -down. Failover is **additive** — own containers keep running, remote containers are -added on top. DDNS for the remote domain starts immediately (Tier 1) so DNS cuts over -within TTL window (1 minute). - ---- - -### 🟡 NO_INTERNET - -``` -Internet ping: FAIL (remote state irrelevant) -``` - -This server has lost internet connectivity. Stop own DDNS immediately — updating DNS -records without internet access sends conflicting updates and could route users to an -unreachable server. Do not start remote containers — there is no internet to serve -them on. Wait for recovery. - ---- - -### ⚫ DARK - -``` -Remote ping: FAIL Internet ping: FAIL -``` - -Both pings fail. Cannot determine if the remote is truly down or if the same outage -affecting internet connectivity is making it unreachable. Conservative approach: same -actions as NO_INTERNET — stop own DDNS, wait. Do not start remote containers. - ---- - -## ━━━ DDNS — THE CRITICAL PART ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -> **This took a year to get right. Do not change the sequencing.** - -The DDNS rules are absolute. The script is the sole authority over when any DDNS -container starts. Network state returning is not permission to start DDNS. Only the -completion of the full handback sequence grants that permission. - ---- - -### ── The Rules ──────────────────────────────────────────────────────────────── - -``` -ONE DOMAIN → ONE DDNS ACTIVE → AT ALL TIMES - -Gmer4Lfe.com → HOST1's DDNS normally → HOST2's DDNS during HOST1 outage -Gmer4Lfe.us → HOST2's DDNS normally → HOST1's DDNS during HOST2 outage - -Own DDNS: ON when this server has internet. OFF when internet is lost. -Remote DDNS: ON as Tier 1 failover action. OFF as first handback action. -Auto-start: NEVER — DDNS never starts automatically on internet return. -``` - ---- - -### ── Why Auto-Start Is Forbidden ───────────────────────────────────────────── - -If HOST1 lost internet and its DDNS auto-started when internet returned, there is a -window where both HOST1 and HOST2 are running the same DDNS — each updating the same -domain record with different IPs. DNS TTL is 1 minute. During that window users get -routed randomly between two servers. Authentication sessions do not transfer between -servers. This is split brain. - -Split brain from double-DDNS is exactly the class of failure that produces the most -confusing symptoms — intermittent auth failures, users sometimes getting through and -sometimes not, no clear error state anywhere. The fix is absolute: the script controls -when DDNS starts. Network state coming back is not permission. - ---- - -### ── Full DDNS Lifecycle — HOST1 Outage ───────────────────────────────────── - -``` -Normal operation: - HOST1 DDNS (Gmer4Lfe.com) → ON always on while HOST1 has internet - HOST2 DDNS (Gmer4Lfe.us) → ON always on while HOST2 has internet - -HOST1 loses internet: - HOST1 DDNS → OFF stopped immediately — no internet, no updates - HOST2 DDNS → ON unaffected, HOST2 still has internet - -HOST2 detects HOST1 is down (FAILOVER state): - HOST1 DDNS (on HOST2) → ON Tier 1 action — DNS now points at HOST2's IP - Both HOST2 DDNS containers → ON own domain + covering HOST1's domain - -HOST1 returns (handback sequence): - HOST1 DDNS (on HOST2) → OFF FIRST action before anything else - [rsync writeback runs] - [HOST1 containers start] - HOST1 DDNS (on HOST1) → ON LAST action — only after containers confirmed up - -Normal operation restored: - HOST1 DDNS (Gmer4Lfe.com) → ON back on HOST1 - HOST2 DDNS (Gmer4Lfe.us) → ON back to normal -``` - -DDNS stops on HOST2 before rsync starts. DDNS starts on HOST1 after containers are -confirmed running. There is a brief window where neither DDNS is updating the record — -this is intentional. DNS TTL caches the last value. During the rsync + container start -window, cached DNS still routes users to HOST2 where the containers are still running. -By the time the cache expires, HOST1's DDNS has started and the record points at HOST1. - ---- - -## ━━━ TIERED FAILOVER ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Not every service needs to start immediately. Starting the full stack on a secondary -server wastes resources for short outages — most power blips and brief ISP issues -resolve in minutes. The tier system starts only what is needed for the actual outage -duration. - ---- - -### ── Tier 1 — Immediate (0 minutes) ───────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# These start the moment FAILOVER state is entered. -# Vital services + services people are actively using right now. -# -FAILOVER_HOST2_RUNS_FOR_HOST1_TIER1=( - "Gmer4Lfe.com-DDNS" # ALWAYS FIRST — DNS coverage before anything else - "Emby" # media server — people are watching - "NginxProxyManager" # reverse proxy — all external access routes through this - "Lldap-Gmer4Lfe" # user directory — already warm, verify and keep - "Mariadb-Authelia" # auth database — already warm, verify and keep - "Redis-Authelia" # auth session cache — already warm, verify and keep - "Authelia" # SSO — already warm, serving users already - "VaultWarden" # passwords — needed immediately, people lock themselves out - "Dispatcharr" # Live TV scheduler — people are watching right now - "Dispatcharr-Basic" # secondary scheduler - "Dispatcharr-Iptv-Users" # tertiary scheduler - "ErsatzTV-Emby" # channel schedule builder -) -# -# Live TV is Tier 1 because you cannot tell a household mid-game that their -# live TV will be back in 2 hours. The disruption is unacceptable. -# Passwords are Tier 1 because people lock themselves out of everything else -# if VaultWarden is unavailable — it becomes the blocking failure. -``` - ---- - -### ── Tier 2 — After HOST*_TIER2_DELAY (default: 4 hours) ──────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# Shared productivity services. Not immediately needed for media access but -# actively used throughout the day. 4 hours is long enough to resolve most -# ISP and power issues without starting services unnecessarily. -# -FAILOVER_HOST2_RUNS_FOR_HOST1_TIER2=( - "NextCloud" # file sync and sharing - "Postgres-NextCloud" # NextCloud's database — must start before NextCloud - "Immich-Gmer4Lfe" # photo library - "PostgreSQL-Immich" # Immich's database — same dependency - "Jellyseerr" # media request management -) - HOST1_TIER2_DELAY=240 # minutes — 4 hours -``` - ---- - -### ── Tier 3 — After HOST*_TIER3_DELAY (default: 12 hours) ────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# Secondary services — useful but not critical for daily operation. -# 12 hours covers most genuine outage scenarios while avoiding unnecessary -# resource usage on HOST2 for short-to-medium disruptions. -# -FAILOVER_HOST2_RUNS_FOR_HOST1_TIER3=( - "Organizrv2-Gmer4Lfe" # dashboard - "AdGuard-Home" # DNS filtering - "UptimeKuma" # uptime monitoring - "Gitea" # git server - "Collabora-CODE" # document editing for NextCloud -) - HOST1_TIER3_DELAY=720 # minutes — 12 hours -``` - ---- - -### ── Tier 4 — After HOST*_TIER4_DELAY (default: 24 hours) ────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# Full workflow continuity — arrs and downloaders. -# These generate significant I/O and have writeback requirements on handback. -# 24 hours is the threshold where it becomes worthwhile to start them. -# -# Arrs_Stack only starts if HOST1 has genuinely been down for 24+ hours. -# Below this threshold: arrs never started on HOST2 → nothing to sync back. -# Above this threshold: meaningful downloads accumulated → full Tier 4 writeback. -# -FAILOVER_HOST2_RUNS_FOR_HOST1_TIER4=( - "Sonarr-Gmer4Lfe" # TV show management - "Radarr-Gmer4Lfe" # movie management - "Lidarr-Gmer4Lfe" # music management - "Prowlarr-Gmer4Lfe" # indexer management - "SABnzbd-Gmer4Lfe" # usenet downloader - "qBittorrent-Gmer4Lfe" # torrent downloader - "LidaTube" # YouTube music - "Pinchflat" # YouTube video -) - HOST1_TIER4_DELAY=1440 # minutes — 24 hours -``` - ---- - -## ━━━ HANDBACK SEQUENCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -When the remote server returns after a FAILOVER event, handback must happen in exactly -this order. Every step has a reason. Do not reorder. - ---- - -### ── Step 1 — Strike Confirmation ──────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# Network blips happen. A server can be reachable for one ping cycle and then -# drop again. If handback triggered on the first successful ping, a brief -# network recovery during an ongoing outage would cause a failed handback — -# containers stop on the covering server, rsync starts, remote goes down again -# mid-transfer. That is worse than not handing back at all. -# -# Strike confirmation requires N consecutive successful remote pings before -# handback begins. With FAILOVER_CHECK_INTERVAL=120s and HANDBACK_STRIKES=2: -# → remote must be up continuously for 4 minutes before handback starts -# → eliminates false triggers from brief network recovery -# - FAILOVER_HANDBACK_STRIKES=2 # consecutive remote-up checks required - FAILOVER_CHECK_INTERVAL=120 # seconds between checks -# -# 2 strikes × 120s = 4 minute confirmation window before handback begins -``` - ---- - -### ── Step 2 — Pre-flight Checks ────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Before starting the handback sequence, verify the remote is actually ready. -# Reachable on Tailscale does not mean the array is started and Docker is -# running — the server might still be booting. -# -# Pre-flight checks (any failure → abort this cycle, retry next): -# ✓ Version parity — both servers on compatible unRAID versions -# ✓ Remote array — /mnt/user is mounted on remote -# ✓ Remote Docker — daemon is responding within DOCKER_TIMEOUT -# ✓ Remote rootfs — not nearly full (headroom for rsync temp files) -# -# If any pre-flight fails: abort handback, reset strike counter, retry. -# Better to wait another cycle than to start a handback that will fail midway. -``` - ---- - -### ── Step 3 — Stop Remote DDNS FIRST ───────────────────────────────────────── - -``` -This is the most critical ordering step in the entire sequence. - -Remote DDNS stops before any containers stop, before rsync starts, -before HOST1 containers start, before HOST1 DDNS starts. - -During the gap between remote DDNS stopping and HOST1 DDNS starting: - DNS TTL caches the last value → users still routed to HOST2 - HOST2 containers still running → users are still served - No split brain window exists because one DDNS stopped before - the other started. The transition is clean. -``` - ---- - -### ── Step 4 — Stop Remote Containers ──────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Containers stop on the covering server before rsync begins. -# Clean static source. No competing I/O. No database writes during transfer. -# Full bandwidth available to rsync. -# -# The user impact window (containers down) is the rsync duration only — typically -# minutes for critical data. Media files are never synced back — they were never -# moved. Downloads are never synced back — start fresh is cleaner. -# -# Containers stop in the correct dependency order: -# Dependents before dependencies (Authelia before MariaDB) -# Same WATCHDOG_DEPENDENCIES ordering used by docker_watchdog.sh -``` - ---- - -### ── Step 5 — Tiered Rsync Writeback ───────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# What gets written back depends on how long the outage lasted. -# Short outages skip writeback entirely — the primary's last-known-good state -# is more reliable than a few minutes of activity on the covering server. -# -# Under 60min → skip all writeback (primary state is cleaner) -# Under 4hr → skip Tier 2+ writeback -# Under 12hr → skip Tier 3+ writeback -# Under 24hr → skip Tier 4 writeback (arrs never started, nothing to sync) -# Over 24hr → full tiered writeback (meaningful delta accumulated) -# -# Real-world outage profile: -# Power blip 2-10min → no writeback, clean restart (most common) -# ISP issue 10-60min → no writeback, clean restart -# Real outage 3-6hr → Tier 1+2 writeback, skip 3+4 -# Extended 24hr+ → full tiered writeback - -FAILOVER_HOST1_WRITEBACK_TIER1=( - "/mnt/user/appdata-Failover/Critical-Data" # auth stack — Authelia + NPM + certs - "/mnt/user/Media_Server/Emby" # Emby userdata — watch history, playstates -) - -FAILOVER_HOST1_WRITEBACK_TIER2=( - "/mnt/user/appdata-Failover/Important-Data" # NextCloud + Postgres + Immich -) - -FAILOVER_HOST1_WRITEBACK_TIER3=( - "/mnt/user/appdata-Failover/Gmer4Lfe" # secondary appdata accumulated changes -) - -# Tier 4 writeback uses HOST2_DAILY_SYNC_SHARES automatically — same list -# daily_sync_maintenance.sh uses, in the opposite direction. -# HOST2 was running HOST1's arrs against mirrored media — that content needs -# to sync back before HOST1's own arrs restart. -# No duplicate list needed — uses the same config, reversed direction. - -# What never gets written back: -# Own media shares (Movies, Tv_Shows, Music) — already on HOST1, never moved -# Downloads — start fresh is cleaner than partial download state -``` - ---- - -### ── Step 6 — Start Remote Containers ─────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Containers start on HOST1 in dependency-safe order. -# Same WATCHDOG_DEPENDENCIES ordering — databases before applications. -# CONTAINER_DELAY seconds between dependency start and dependent start. -# Each container verified running after a settle period before the next starts. -# A container that starts and immediately crashes is caught and reported. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Step 7 — Start Remote DDNS LAST ───────────────────────────────────────── - -``` -DNS cuts back to HOST1 ONLY after all containers are confirmed running. - -Users hit HOST1 only after HOST1 is actually ready to serve them. - -The handback sequence completes. State file resets to NORMAL. -Tier flags cleared. Next cycle confirms everything is healthy. -``` - ---- - -## ━━━ MUTUAL FAILOVER — BOTH DIRECTIONS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -The same `failover.sh` script handles both directions without any code changes. -`detect_hosts()` in `common.sh` determines which server is local and which is remote -at runtime, then selects the correct container arrays and tier delays from `master.conf`. - -```bash -# master.conf — symmetric configuration for both directions -# ───────────────────────────────────────────────────────────────────────────── - -# HOST2 covers HOST1 (HOST1 goes down): -FAILOVER_HOST2_RUNS_FOR_HOST1_TIER1=(...) # what HOST2 starts for HOST1 -FAILOVER_HOST2_RUNS_FOR_HOST1_TIER2=(...) -FAILOVER_HOST2_RUNS_FOR_HOST1_TIER3=(...) -FAILOVER_HOST2_RUNS_FOR_HOST1_TIER4=(...) -HOST1_TIER2_DELAY=240 # delays for HOST1 outage — from HOST1's perspective -HOST1_TIER3_DELAY=720 -HOST1_TIER4_DELAY=1440 - -# HOST1 covers HOST2 (HOST2 goes down): -FAILOVER_HOST1_RUNS_FOR_HOST2_TIER1=(...) # what HOST1 starts for HOST2 -FAILOVER_HOST1_RUNS_FOR_HOST2_TIER2=(...) -HOST2_TIER2_DELAY=240 # delays for HOST2 outage — from HOST2's perspective -HOST2_TIER3_DELAY=720 -HOST2_TIER4_DELAY=1440 - -# Writeback — what each server syncs back on handback -FAILOVER_HOST1_WRITEBACK_TIER1=(...) # what HOST2 syncs back to HOST1 -FAILOVER_HOST2_WRITEBACK_TIER1=(...) # what HOST1 syncs back to HOST2 -``` - -Both servers run identical scripts with identical configuration. The configuration -controls what each server does. The script reads `MY_ID` from `detect_hosts()` and -selects the correct arrays. No hostname comparisons anywhere in the script. - ---- - -## ━━━ INDEPENDENCE — ALWAYS ONE RSYNC STOP AWAY ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -HOST2 is designed to be fully independent if needed. If HOST2 ever wants to fully -separate from HOST1: stop HOST1 pushing data. Any changes HOST2 makes to his own -data stick permanently. His server becomes fully independent immediately — no script -changes, no migration, no data movement required. - -The ecosystem supports this by design. Everything HOST2 runs is self-contained. -The failover and rsync scripts are configuration-driven — stopping the rsync job -is the entire separation process. - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🔀 failover.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -The main state machine. Runs continuously as a background process on both servers, -started at array start by `array_start.sh`. Makes all decisions autonomously from -two pings. Never requires human intervention during normal failover and handback. - -```bash -# Started automatically at array start via array_start.sh -# Runs continuously until array stops (SIGTERM → clean shutdown) -# Interval: FAILOVER_CHECK_INTERVAL=120 (2 minutes) -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Normal operation — started by array_start.sh, runs continuously. -# You do not need to run this manually under normal circumstances. -# ───────────────────────────────────────────────────────────────────────────── -failover.sh - -# ───────────────────────────────────────────────────────────────────────────── -# Status — show current state at a glance: -# • Current state (NORMAL / FAILOVER / NO_INTERNET / DARK) -# • Which tier containers are active -# • Outage duration if in FAILOVER -# • Handback strike count if remote has returned -# • DDNS containers currently running on this server -# ───────────────────────────────────────────────────────────────────────────── -failover.sh --status - -# ───────────────────────────────────────────────────────────────────────────── -# Dry run — walk through one full cycle without starting or stopping anything. -# Shows what the script would do based on current network state. -# Useful for verifying configuration before relying on it. -# ───────────────────────────────────────────────────────────────────────────── -failover.sh --dry-run - -# ───────────────────────────────────────────────────────────────────────────── -# Verbose — full detail on every decision made in each cycle. -# ───────────────────────────────────────────────────────────────────────────── -failover.sh --log -``` - -> **To stop:** Click Abort in the User Scripts plugin. Do NOT kill the process directly -> — the state file may be left inconsistent. If the state gets stuck, use -> `failover_state_reset.sh` to recover. - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🧪 failover_test.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Controlled simulation of the full failover lifecycle. Validates the entire sequence -without waiting for a real outage. Contains no failover logic itself — exercises -the real `failover.sh` through an iptables block. Any change to `failover.sh` -is automatically reflected in the test. - -> **⚠️ This starts and stops real containers on both servers. Users will experience -> a brief service interruption. Run during a maintenance window or quiet period. -> Use `--dry-run` first to walk through all phases without making any changes.** - ---- - -### ── Test Phases ───────────────────────────────────────────────────────────── - -``` -Phase 1 — Pre-flight - Verify both servers reachable via Tailscale - Verify Docker daemons responding on both servers - Verify version parity between servers - Verify failover.sh exists and is running on remote - Verify current state is NORMAL on both servers - → Any failure: abort test before touching anything - -Phase 2 — Block Remote - Add iptables rule dropping all traffic to remote Tailscale IP - From this server's perspective: remote is now unreachable - failover.sh on this server will see the remote as down on next check - -Phase 3 — Failover Detection - Wait FAILOVER_TEST_BLOCK_WAIT seconds for failover.sh to detect outage - Verify this server entered FAILOVER state - Verify Tier 1 containers started on this server - -Phase 4 — Container Verification - Check each Tier 1 container is actually running - Report any that failed to start - -Phase 5 — Restore Remote - Remove iptables rule — remote becomes reachable again - (Safety trap also removes rule on any exit — crash, ctrl-c, error) - -Phase 6 — Handback Wait - Wait FAILOVER_TEST_HANDBACK_WAIT seconds for failover.sh to: - Confirm remote up (strike confirmation window) - Complete pre-flight checks - Stop remote DDNS - Stop remote containers - Run rsync writeback (if applicable) - Start local containers - Start local DDNS - Return to NORMAL - -Phase 7 — Handback Verification - Verify state returned to NORMAL - Verify Tier 1 containers stopped on this server (handed back) - Verify DDNS is correct on both servers - -Phase 8 — Report - Full pass/fail per phase with timing - Clear indication of what failed if any phase did not pass -``` - ---- - -### ── Safeguards ────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# IPTABLES SAFETY TRAP — the most important safeguard. -# The iptables rule that blocks remote connectivity is added in Phase 2. -# It is removed via a trap on ANY exit condition: -# Normal completion → rule removed -# Script crashes → rule removed -# Error in any phase → rule removed -# ctrl-c → rule removed -# -# Remote connectivity is always restored regardless of test outcome. -# You cannot accidentally leave the remote permanently blocked. -# -# FAILOVER_ENABLED gate — aborts if FAILOVER_ENABLED=false in master.conf. -# Running a failover test when failover is disabled would be confusing and -# potentially destructive. The gate prevents this. -# -# Version parity check — pre-flight verifies both servers are on compatible -# unRAID versions before any iptables rules are added. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Timing Configuration ──────────────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# These must be set correctly for the test to pass. -# Too short: test fails because failover.sh hasn't had time to complete its work. -# Too long: test runs unnecessarily long. -# - FAILOVER_TEST_BLOCK_WAIT=150 # must be > FAILOVER_CHECK_INTERVAL + buffer - # failover.sh checks every 120s — 150s gives - # one full cycle plus 30s margin - - FAILOVER_TEST_HANDBACK_WAIT=360 # must cover: strike confirmation window - # + pre-flight time - # + rsync duration (critical data only) - # + container start time - # 2 strikes × 120s + ~2min rsync + ~1min start - # = ~6 minutes = 360 seconds -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Dry run — ALWAYS run this first. -# Walks through all 8 phases with full output but no iptables changes and -# no container starts/stops. Verifies configuration and timing before -# committing to a live test. -# ───────────────────────────────────────────────────────────────────────────── -failover_test.sh --dry-run - -# ───────────────────────────────────────────────────────────────────────────── -# Live test — run during maintenance window. -# Full sequence with real iptables rules and real container lifecycle. -# Users will experience a brief service interruption — schedule accordingly. -# ───────────────────────────────────────────────────────────────────────────── -failover_test.sh - -# ───────────────────────────────────────────────────────────────────────────── -# Status — show current failover state and test configuration without running. -# ───────────────────────────────────────────────────────────────────────────── -failover_test.sh --status - -# Verbose — full detail on every check in every phase -failover_test.sh --log -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🔧 failover_state_reset.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Manual recovery tool. Resets the state file to NORMAL and clears all tier flags. -Use when the state file is stuck in a non-NORMAL state after testing, a failed -handback, or killing `failover.sh` mid-cycle. - -> **Does NOT start or stop containers — state file only.** After reset, `failover.sh` -> will resume from NORMAL on its next cycle. Verify that containers are actually in -> the right state before resetting — the state file should reflect reality. - ---- - -### ── When to Use This ──────────────────────────────────────────────────────── - -``` -After failover_test.sh didn't complete cleanly - → state file left in FAILOVER but containers are actually back to normal - -After a failed handback - → state shows FAILOVER but remote is back up and containers are split - -After killing failover.sh directly (not via User Scripts Abort) - → state is unknown, cycle was interrupted mid-operation - -After a dev/debug session - → state left in a non-NORMAL state from testing -``` - ---- - -### ── Safety Checks ─────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# VERIFY BEFORE RESETTING: -# ✓ Right containers running on the right server -# ✓ DDNS pointing at the correct server (check with nslookup Gmer4Lfe.com) -# ✓ No actual failover in progress (remote is genuinely up) -# ✓ Both servers can see each other (tailscale ping) -# -# Resetting during an actual failover causes failover.sh to think everything -# is normal and stop covering the remote — services go offline until the next -# detection cycle catches it again. -# -# The script warns if failover.sh is currently running when you attempt reset. -# It does not block you — but heed the warning. If failover.sh is running and -# you reset mid-cycle, the next cycle will start from a clean NORMAL state -# regardless of what was in progress. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Status — show current state file contents without changing anything. -# Also shows whether failover.sh is currently running. -# Use this first to understand the situation before resetting. -# ───────────────────────────────────────────────────────────────────────────── -failover_state_reset.sh --status - -# ───────────────────────────────────────────────────────────────────────────── -# Interactive reset — prompts for "YES" before proceeding. -# Default mode — use when running manually at a terminal. -# ───────────────────────────────────────────────────────────────────────────── -failover_state_reset.sh - -# ───────────────────────────────────────────────────────────────────────────── -# Dry run — show current state and what would be written, without writing. -# ───────────────────────────────────────────────────────────────────────────── -failover_state_reset.sh --dry-run - -# ───────────────────────────────────────────────────────────────────────────── -# Force — non-interactive, no confirmation prompt. -# For use in scripts or when running without a terminal. -# ───────────────────────────────────────────────────────────────────────────── -failover_state_reset.sh --force -``` - ---- - -## ━━━ STATE FILE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -```bash -# /boot/config/failover_state.db -# ───────────────────────────────────────────────────────────────────────────── -# Lives on /boot/ — survives reboots intentionally. -# If a server was in FAILOVER state when it rebooted, it should resume -# FAILOVER on restart — not assume everything is NORMAL again. -# -state=NORMAL # current state machine state -failover_start=0 # epoch timestamp when FAILOVER began (0 = not in FAILOVER) -handback_strikes=0 # consecutive remote-up checks accumulated toward handback -tier2_started=false # whether Tier 2 containers have been started this event -tier3_started=false # whether Tier 3 containers have been started -tier4_started=false # whether Tier 4 containers have been started -last_reset=2026-04-14 03:00:00 -reset_by=HOST1 # which server last reset the state (set by failover_state_reset.sh) -``` - ---- - -## ━━━ INITIAL SETUP REQUIREMENTS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Before `failover.sh` can work on both servers: - ---- - -### 1. Tailscale Connected on Both Servers - -```bash -# Verify from HOST1 — should return HOST2's Tailscale IP -tailscale ip -4 unRAID-Jayred365 - -# Verify from HOST2 — should return HOST1's Tailscale IP -tailscale ip -4 unRAID-Gmer4Lfe - -# Test reachability from HOST1 -tailscale ping unRAID-Jayred365 -``` - ---- - -### 2. SSH Keys Configured — No Password Prompt - -```bash -# From HOST1 — should print HOST2's hostname without asking for password -ssh -i /root/.ssh/Gmer4Lfe-rsync-key root@[HOST2-tailscale-ip] "hostname" - -# From HOST2 — should print HOST1's hostname without asking for password -ssh -i /root/.ssh/Jayred365-rsync-key root@[HOST1-tailscale-ip] "hostname" -``` - ---- - -### 3. Container Names Match - -Failover containers must be created (but stopped) on the server that will run them. -If HOST2 will start `Emby` for HOST1, the `Emby` container must exist on HOST2 with -its volume mounts pointing at the mirrored share paths. - -```bash -# Verify the container exists on HOST2 (stopped is expected) -ssh root@[HOST2-ip] "docker inspect Emby --format '{{.State.Status}}'" -# Expected: created or exited — not "no such container" -``` - ---- - -### 4. Critical Data Mirrored Before Failover is Needed - -```bash -# These shares must exist on HOST2 with current data from HOST1: -/mnt/user/appdata-Failover/Critical-Data # auth stack -/mnt/user/appdata-Failover/Important-Data # NextCloud + Postgres -/mnt/user/Media_Server/Emby # Emby userdata -/mnt/user/appdata-Failover/Gmer4Lfe # server appdata - -# Verify data is current — check modification times -ssh root@[HOST2-ip] "ls -la /mnt/user/appdata-Failover/Critical-Data/" -``` - ---- - -### 5. DDNS TTL Set to 1 Minute - -In your DDNS provider settings. Higher TTL means users continue hitting the old IP -for longer after failover. 1 minute is the minimum most providers allow. At 5 minute -TTL, users can be hitting a downed server for up to 5 minutes before DNS switches. - ---- - -### 6. Both Servers Running failover.sh - -Failover only works in one direction if only one server is running the script. -For mutual coverage, both servers must be running it continuously. - -```bash -# Verify failover.sh is running on both servers -pgrep -f "failover.sh" - -# Check the state file directly -cat /boot/config/failover_state.db -``` - ---- - -## ━━━ MONITORING ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -**Sunday coffee report** (`sunday_morning_coffee_report.sh`) includes a Failover section -showing current state, outage duration if not NORMAL, Tailscale reachability, and -whether the failover.sh process is running. - -**Weekly health digest** (`weekly_health_digest.sh`) reads the state file. If -`DIGEST_SMART_ON_FAILOVER=true` and state is not NORMAL, it sends a notification -even in smart mode — a non-NORMAL state at digest time needs attention. - -**Direct status check:** - -```bash -failover.sh --status # full state snapshot from current server -``` - ---- - -## ━━━ TROUBLESHOOTING ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - ---- - -### 🔴 Failover Not Triggering - -``` -Is failover.sh actually running on the covering server? - → User Scripts plugin → check status of the failover script - -Is Tailscale connected on the covering server? - → tailscale status (should show the remote peer) - -Can the covering server reach the remote's Tailscale IP? - → ping [remote-tailscale-ip] - -What state is the covering server in? - → failover.sh --status (should show FAILOVER if remote is down) - → cat /boot/config/failover_state.db -``` - ---- - -### 🔴 Handback Not Completing - -``` -Is HOST1's array fully started? - → ls /mnt/user (should show share directories) - -Is Docker responding on HOST1? - → docker ps (should return a list, not hang) - -Is HOST1's rootfs below the warn threshold? - → df / (rootfs nearly full blocks handback pre-flight) - -Is rsync running and stuck? - → pgrep rsync (a stalled rsync blocks the handback sequence) - → rsync_stop.sh to clear it, then let failover.sh retry -``` - ---- - -### 🔴 DDNS Not Cutting Over - -``` -Is the DDNS container actually running on the covering server? - → docker ps | grep DDNS - -What TTL is your DNS record set to? - → nslookup Gmer4Lfe.com 8.8.8.8 (check TTL in response) - → High TTL means slow propagation - -Is your DDNS provider accepting updates? - → Check DDNS container logs: docker logs [ddns-container] --tail 50 -``` - ---- - -### 🔴 Split Brain (Both DDNS Running) - -``` -This should not happen if the handback sequence completed correctly. -If it has happened: - -1. Check both servers — which DDNS containers are running where - HOST1: docker ps | grep DDNS - HOST2: docker ps | grep DDNS - -2. Manually stop the duplicate - docker stop [duplicate-ddns-container] - -3. Reset the state file on the server in a bad state - failover_state_reset.sh --status (understand the situation) - failover_state_reset.sh --force (after verifying it is safe) - -4. Restart failover.sh (it will have been stopped by the state reset) - Start via User Scripts plugin -``` - ---- - -### 🔴 State File Stuck in FAILOVER After Testing - -``` -Normal — this is what failover_state_reset.sh is for. - -1. Verify everything is actually back to normal: - - Right containers on right server - - DDNS correct (nslookup Gmer4Lfe.com) - - Both servers visible on Tailscale - -2. Reset the state file: - failover_state_reset.sh --status (confirm what you're resetting) - failover_state_reset.sh (interactive — prompts YES) - -failover.sh will resume from NORMAL on its next cycle. -``` \ No newline at end of file diff --git a/Fallback/README-Fallback.md b/Fallback/README-Fallback.md new file mode 100644 index 0000000..2dec0c2 --- /dev/null +++ b/Fallback/README-Fallback.md @@ -0,0 +1,275 @@ +# ━━━━━ FALLBACK ━━━━━ + +Mutual automatic failover between two independent unRAID servers. When one goes down the +other starts its containers, cuts over DNS, and keeps users online. When it comes back +everything hands back in the correct sequence — covering DDNS stops, containers stop, rsync +writeback runs, containers start on the primary, primary DDNS starts last — so users hit the +returning server only after it's actually ready. + +> **Built from scratch. Refined through a year of production testing.** The DDNS sequencing +> and handback order were the hardest parts to get right. Both directions are exercised +> regularly with `fallback_test.sh`. + +--- + +## ━━━ THE PROBLEM THAT BUILT THIS ━━━ + +Running a self-hosted stack means being the operator. These are the specific problems +that drove this build: + +**A Single Point of Failure for an Entire Household** +HOST1 runs Emby, NPM, Authelia, NextCloud, VaultWarden, and every service the household +uses daily. When HOST1 goes down — even briefly — all of those services go down with it. +The fix: a second server with mirrored critical data that covers the first automatically. +From a user's perspective, a brief interruption and then everything is back. + +**DNS Cutting Over Before the Server Was Ready** +Early attempts started containers on the covering server then updated DNS. Problem: DNS +propagated in under a minute. Users hit the new IP before Emby had finished starting, +before Authelia had loaded its sessions, before NPM had loaded its proxy configurations. +The fix: warm standby for the auth stack. NPM, LLDAP, and Authelia run actively on both +servers at all times. When DNS cuts over, auth is already running and ready. + +**Split Brain DNS During Handback** +When HOST1 returned, the obvious sequence was: start HOST1 containers, then switch DNS +back. Problem: between "start containers" and "DNS switches" both servers' DDNS containers +were running, both updating the same domain with different IPs. Users got routed randomly +between servers — intermittent auth failures, no clear error state anywhere. +The fix: stop DDNS on the covering server first, before anything else moves. There is +never a window where two DDNS containers update the same record. + +**Rsync Running Into Active Container I/O** +Syncing data back while containers were still running — to minimise downtime — produced +slower transfers, potential file inconsistency, and database dirty state risk. +The fix: stop containers before syncing. The outage window is only the rsync duration — +typically minutes. Clean static source at full bandwidth, predictable state every time. + +**No Way to Validate the System Before Needing It** +A failover system that has never been tested is not a failover system — it is a hope. +The fix: `fallback_test.sh` — a controlled simulation using an iptables DROP rule to make +the remote appear unreachable, triggering the full sequence without taking anything offline. +A safety trap removes the rule on any exit — crash, error, ctrl-c, or clean completion. + +--- + +## ━━━ THE TWO-SERVER SETUP ━━━ + +``` +HOST1 — unRAID-Gmer4Lfe + Hardware: Threadripper 1950X, 128GB RAM, ZFS cache pools + Location: Primary site + DDNS: Gmer4Lfe.com + Role: Primary — full service stack + source of truth for Movies/Shows/Music + +HOST2 — unRAID-Jayred365 + Hardware: Intel i5 10th gen, 64GB RAM + Location: Remote — 50 miles away + DDNS: Gmer4Lfe.us + Role: Secondary — own stack + covers HOST1 + mirrors critical data +``` + +**Hardware does not need to match.** Everything is accessed through `/mnt/user/` — unRAID's +fused share layer. HOST1 has a Threadripper with ZFS pools. HOST2 has completely different +hardware. Fallback containers on HOST2 mount `/mnt/user/Movies` and see mirrored data +because the share names match. The hardware underneath is irrelevant. + +**What must match between servers:** + +``` +Share names /mnt/user/Movies must exist on both servers (mirrored data) +Container names "Emby" on HOST2 must be the container HOST2 starts for HOST1 +Network names Docker custom networks must match for NPM routing to work +``` + +### Split Source of Truth — No Conflicts + +Both servers run arr instances simultaneously with zero conflict — they manage completely +different shares: + +``` +HOST1 owns: Movies (Radarr), Tv_Shows (Sonarr), Music (Lidarr) +HOST2 owns: Anime_Movies (his Radarr), Anime_Shows (his Sonarr) + +Each server mirrors the other's shares continuously via rsync. +``` + +The rule: never run two arr instances against the same share simultaneously. Different arrs +managing different shares is fine. When HOST2 runs HOST1's arrs during a Tier 4 fallback, +HOST1's Tdarr is not running (HOST1 is down) — no conflict. + +### Auth Stack — Warm on Both Servers + +NPM, LLDAP, and Authelia run actively on both servers at all times. HOST2 needs them running +to serve his own users daily — this is not a fallback-only configuration. HOST1 is source of +truth: all changes mirror to HOST2 every 15 minutes via critical sync. + +When DNS cuts over, auth is already running on the covering server. The 30-60 second dead +zone where auth is coming up after DNS has already switched does not exist. + +--- + +## ━━━ HOW IT WORKS ━━━ + +Both servers run `fallback.sh` independently as a continuous background process. Each server +makes all decisions from two pings every `FALLBACK_CHECK_INTERVAL` seconds: + +```bash +ping REMOTE_TAILSCALE_IP # is the other server reachable? +ping EXTERNAL_IP # do I have internet? (default: 8.8.8.8) +``` + +No SSH signaling between servers. No shared state file. No election algorithm. Each server +acts entirely from its own network perspective. + +**States:** + +| State | Remote | Internet | Action | +|-------|--------|----------|--------| +| NORMAL | up | up | Silent — own containers, own DDNS on | +| FALLBACK | down | up | Start tier containers, cut DDNS over | +| NO_INTERNET | — | down | Stop own DDNS immediately, wait | +| DARK | down | down | Same as NO_INTERNET — cannot determine cause | + +--- + +## ━━━ DDNS — THE CRITICAL PART ━━━ + +> **This took a year to get right. Do not change the sequencing.** + +`fallback.sh` is the sole authority over when any DDNS container starts. Network state +returning is not permission to start DDNS. Only completion of the full handback sequence +grants that permission. + +``` +ONE DOMAIN → ONE DDNS ACTIVE → AT ALL TIMES + +Gmer4Lfe.com → HOST1's DDNS normally → HOST2's DDNS during HOST1 outage +Gmer4Lfe.us → HOST2's DDNS normally → HOST1's DDNS during HOST2 outage + +Own DDNS: ON when this server has internet. OFF when internet is lost. +Remote DDNS: ON as Tier 1 fallback action. OFF as FIRST handback action. +Auto-start: NEVER — DDNS never starts automatically on internet return. +``` + +**Why auto-start is forbidden:** If HOST1 lost internet and its DDNS auto-started when +internet returned, there is a window where both servers are updating the same domain with +different IPs. Users get routed randomly — some to the primary with fresh data, some to +the covering server. Authentication sessions don't transfer between servers. This is split +brain and produces the most confusing symptoms: intermittent auth failures with no clear +error state anywhere. + +The brief gap where neither DDNS is updating the record is intentional. DNS TTL caches the +last value. During the rsync + container start window, cached DNS still routes users to the +covering server where containers are still running. By the time the cache expires, the +primary's DDNS has started and the record points at the right server. + +--- + +## ━━━ TIERED FALLBACK ━━━ + +Starting the full stack for a 5-minute power blip wastes resources — most brief outages +resolve before Tier 2 would even activate. Tiers start only what is needed for the actual +outage duration. + +| Tier | Delay | Coverage | Why This Timing | +|------|-------|----------|-----------------| +| Tier 1 | Immediate | Vital services + Live TV | People are watching — cannot wait 4 hours | +| Tier 2 | 4hr (HOST*_TIER2_DELAY) | NextCloud, Immich, Jellyseerr | 4hr covers most ISP and power events | +| Tier 3 | 12hr (HOST*_TIER3_DELAY) | Dashboard, AdGuard, Git, Collabora | Secondary — useful but not daily-critical | +| Tier 4 | 24hr (HOST*_TIER4_DELAY) | Arrs + downloaders | Significant I/O — only worth starting at 24hr | + +Tier 1 always includes the remote domain's DDNS container as the first entry — DNS +coverage happens before any other container starts. + +For the actual container lists and tier delay values, see `Manual-Fallback.md`. + +--- + +## ━━━ HANDBACK SEQUENCE ━━━ + +When the remote server returns after a FALLBACK event. Every step has a reason. +Do not reorder. + +1. **Strike confirmation** — FALLBACK_HANDBACK_STRIKES consecutive remote-up checks before + handback begins. Prevents false triggers from brief network recovery. + +2. **Pre-flight checks** — version parity, remote array mounted, remote Docker daemon up. + Any failure aborts and retries next cycle. + +3. **Staged reverse handback: Tier 4 → 3 → 2** — Emby and vital services stay on the + covering server serving users throughout this phase. Each tier: stop local containers + → rsync writeback (if outage exceeded tier threshold) → start on remote. + +4. **DDNS handoff** — stop remote DDNS immediately before Tier 1 goes down. This prevents + split brain during the Tier 1 rsync window. + +5. **Tier 1 handback** — stop local vital services, rsync writeback, start on remote. + +6. **Start remote DDNS last** — DNS cuts back to the primary only after all containers + are confirmed running. + +7. **Return to NORMAL** — state file reset, own DDNS restored if it was stopped. + +--- + +## ━━━ MUTUAL FALLBACK — BOTH DIRECTIONS ━━━ + +The same `fallback.sh` handles both directions without any code changes. `detect_hosts()` +determines which server is local and which is remote at runtime, then selects the correct +container arrays and tier delays from config via MY_ID. + +``` +HOST2 covers HOST1: FALLBACK_HOST2_COVERS_HOST1_TIER* (in master_host2.conf) +HOST1 covers HOST2: FALLBACK_HOST1_COVERS_HOST2_TIER* (in master_host1.conf) +``` + +Both servers run identical scripts. MY_ID selects the correct arrays. No hostname +comparisons anywhere in the script code. + +--- + +## ━━━ INDEPENDENCE — ALWAYS ONE RSYNC STOP AWAY ━━━ + +HOST2 is designed to be fully independent if needed. If HOST2 ever wants to separate from +HOST1: stop HOST1 pushing data. Any changes HOST2 makes to his own data stick permanently. +His server becomes fully independent immediately — no script changes, no migration, no data +movement required. The fallback and rsync scripts are configuration-driven. + +--- + +## ━━━ SCRIPTS IN THIS FOLDER ━━━ + +| Script | Role | When It Runs | +|--------|------|--------------| +| `fallback.sh` | Continuous state machine — monitors remote, manages fallback and handback | Continuously (started by `array_start.sh`) | +| `fallback_test.sh` | 7-phase test harness — validates the entire fallback lifecycle via iptables simulation | On demand — maintenance window only | + +--- + +## ━━━ HOW THE SCRIPTS RELATE ━━━ + +``` +array_start.sh + │ + └── starts fallback.sh (continuous loop) + │ + ├── Every FALLBACK_CHECK_INTERVAL seconds: + │ ping remote, ping internet + │ → determine state → act on containers + DDNS + │ + └── Test path: + │ + fallback_test.sh + │ + ├── Phase 1: pre-flight — both servers ready + ├── Phase 2: iptables DROP rule → remote appears down + ├── Phase 3: wait for fallback.sh to detect → FALLBACK state + ├── Phase 4: verify Tier 1 containers started locally + ├── Phase 5: remove DROP rule → remote reachable again + ├── Phase 6: wait for fallback.sh to complete handback → NORMAL + └── Phase 7: verify Tier 1 containers stopped locally +``` + +`fallback_test.sh` contains no fallback logic. It exercises the real `fallback.sh` through +connectivity manipulation. Any change to `fallback.sh` is automatically reflected in the +test result. diff --git a/Fallback/fallback_test.sh b/Fallback/fallback_test.sh index 8f10a15..28765e8 100644 --- a/Fallback/fallback_test.sh +++ b/Fallback/fallback_test.sh @@ -1,54 +1,101 @@ #!/bin/bash # ============================================================================================== -# ================================= Failover Test ============================================== +# ================================= Fallback Test ============================================== # ============================================================================================== -# Controlled simulation of the failover lifecycle — validates the entire failover sequence -# without waiting for a real outage. # -# ── WHAT THIS SCRIPT IS ─────────────────────────────────────────────────────────────────────── -# A test harness only — contains no failover logic. -# All failover logic lives in fallback.sh and is exercised by this test. -# Any changes to fallback.sh are automatically reflected here. +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Controlled simulation of the fallback lifecycle. Validates the entire sequence +# without waiting for a real outage. Contains no fallback logic — exercises the +# real fallback.sh via an iptables DROP rule on the remote Tailscale IP. # -# ── TEST SEQUENCE ───────────────────────────────────────────────────────────────────────────── -# Phase 1 — Pre-flight verify both servers reachable, daemons healthy, -# version parity, fallback.sh exists, state is NORMAL -# Phase 2 — Block Remote iptables rule drops all traffic to remote IP -# Phase 3 — Fallback Detection wait for fallback.sh to detect outage and enter FALLBACK -# Phase 4 — Container Start verify Tier 1 failover containers started locally -# Phase 5 — Restore remove iptables rule, remote becomes reachable -# Phase 6 — Handback wait for fallback.sh to complete handback to NORMAL -# Phase 7 — Container Handback verify Tier 1 containers stopped locally after handback -# Phase 8 — Report full pass/fail summary per phase +# Run during a maintenance window. Users will experience a brief service +# interruption. Use --dry-run to walk through all phases without real changes. # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# FALLBACK_ENABLED gate — aborts if fallback monitoring is disabled -# iptables safety trap — rule ALWAYS removed on exit (crash, error, ctrl-c, normal) -# remote connectivity always restored regardless of outcome -# Version parity check — pre-flight verifies both servers on compatible unRAID versions -# Remote Docker daemon — pre-flight verifies remote daemon is responsive -# DOCKER_TIMEOUT — all docker calls protected against daemon hangs -# MY_ID-based routing — tier containers selected via MY_ID not hostname comparison -# Command validation — iptables and notify validated before use -# Dry-run safe — full sequence walkthrough without touching iptables or containers +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── WARNING ─────────────────────────────────────────────────────────────────────────────────── -# ⚠️ This script starts and stops REAL containers on both servers. -# Run during a maintenance window — users will experience a brief service interruption. -# Use --dry-run to walk through the sequence without any real changes. +# Phase 1 — Pre-flight Both servers reachable, Docker daemons healthy, +# version parity, fallback.sh exists, state NORMAL +# Phase 2 — Block Remote iptables DROP rule added — remote appears unreachable +# Phase 3 — Fallback Detection Wait FALLBACK_TEST_BLOCK_WAIT for fallback.sh to +# detect the outage and enter FALLBACK state +# Phase 4 — Container Start Verify Tier 1 containers started locally +# Phase 5 — Restore iptables rule removed — remote reachable again +# Phase 6 — Handback Wait FALLBACK_TEST_HANDBACK_WAIT for fallback.sh to +# complete full handback and return to NORMAL +# Phase 7 — Container Handback Verify Tier 1 containers stopped locally +# Report — Full pass/fail per phase with timing # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# FALLBACK_TEST_BLOCK_WAIT — seconds to wait for fallback.sh to detect outage -# FALLBACK_TEST_HANDBACK_WAIT — seconds to wait for fallback.sh to complete handback -# FALLBACK_CHECK_INTERVAL — check interval of the running fallback.sh (informational) -# FALLBACK_HANDBACK_STRIKES — strikes required before handback (informational) -# FALLBACK_STATE_FILE — state file path to read current state +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Test Harness Only +# Contains zero fallback logic. All fallback is exercised through fallback.sh. +# Any change to fallback.sh is automatically reflected in the test result. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# iptables Safety Trap +# The DROP rule is removed via trap on ANY exit — normal completion, crash, error, +# ctrl-c. Remote connectivity is always restored regardless of test outcome. +# You cannot accidentally leave the remote permanently blocked. +# +# FALLBACK_ENABLED Gate +# Aborts if FALLBACK_ENABLED=false. Testing a disabled fallback system is +# misleading and potentially destructive. +# +# State Must Be NORMAL +# Pre-flight fails if state is not NORMAL. Running a test during an actual +# fallback event would interfere with the real event. +# +# Version Parity Check +# Pre-flight verifies unRAID version parity before any iptables rules are +# added. A mismatch makes the test result unreliable. +# +# Remote Docker Daemon Check +# Pre-flight confirms remote Docker daemon is responsive before Phase 2. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# FALLBACK_TEST_BLOCK_WAIT +# Seconds to wait in Phase 3 for fallback.sh to detect the outage. +# Must be > FALLBACK_CHECK_INTERVAL + buffer. At 30s interval: use ≥60s. +# (default: 60) +# +# FALLBACK_TEST_HANDBACK_WAIT +# Seconds to wait in Phase 6 for fallback.sh to complete handback. +# Must cover: FALLBACK_HANDBACK_STRIKES × FALLBACK_CHECK_INTERVAL + rsync +# duration + container start time. At 3 strikes × 30s + ~2min rsync + +# ~1min container start: use ≥240s. (default: 300) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# fallback_test.sh --dry-run +# Walk through all 7 phases with output but no iptables changes and no +# container starts/stops. ALWAYS run this before a live test. +# +# fallback_test.sh +# Full live test — real iptables DROP rule, real container lifecycle. +# Users will experience a brief service interruption. Run during a +# maintenance window. +# +# fallback_test.sh --status +# Show current fallback state and test timing configuration. No test run. +# +# fallback_test.sh --log +# Verbose output on every check in every phase. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# fallback_test.sh — run full test sequence -# fallback_test.sh --dry-run — walk through all phases without changes -# fallback_test.sh --status — show current fallback state and test config -# fallback_test.sh --log — verbose output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Initial_run/partnership_onboard.sh b/Initial_run/partnership_onboard.sh deleted file mode 100755 index 7d08bfb..0000000 --- a/Initial_run/partnership_onboard.sh +++ /dev/null @@ -1,181 +0,0 @@ -#!/bin/bash -# ============================================================================================== -# ============================= Partnership Onboard Setup ====================================== -# ============================================================================================== -# Orchestrates the full partnership setup sequence: -# 1. SSH key setup — generate keypair, install on remote, update conf -# 2. Partnership onboard — configure auth WebUIs, write state, FolderView3 folder -# -# Run this once to join a new partner server. Both servers run their own copy. -# After setup: critical_sync_maintenance.sh carries the relationship via --check. -# -# ── PRE-REQUISITES ──────────────────────────────────────────────────────────────────────────── -# Both servers must be on the same Tailscale tailnet -# Remote server must have password auth enabled for root (SSH key install step) -# PARTNERSHIP_OWNER_HOST set correctly in master.conf (HOST1 = owner by default) -# -# ── WHAT THIS SCRIPT DOES ───────────────────────────────────────────────────────────────────── -# Step 1: ssh_setup.sh — generates {hostname}_rsync_automation keypair if missing, -# installs on remote via ssh-copy-id, updates master_host*.conf -# Step 2: partnership_manager.sh --onboard -# — verifies both servers, reconfigures auth WebUIs → owner IP, -# writes ACTIVE state, creates FolderView3 folder (if enabled) -# Step 3: arr_sync.sh — bidirectional library bootstrap: partner gets our full -# Lidarr/Sonarr/Radarr library, we get theirs. Both sides -# start tracking the merged library from day one. -# Non-fatal — partnership is valid even if arrs aren't live yet. -# -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# Initial_run/partnership_onboard.sh — full setup -# Initial_run/partnership_onboard.sh --dry-run — preview without changes -# Initial_run/partnership_onboard.sh --log — verbose output -# Initial_run/partnership_onboard.sh --skip-ssh — skip ssh_setup.sh (key already set up) -# Initial_run/partnership_onboard.sh --skip-arr-sync — skip arr sync (arrs not live yet) -# ============================================================================================== - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SCRIPTS_ROOT="$SCRIPT_DIR/.." - -source "$SCRIPTS_ROOT/load_config.sh" - -# ── Parse --skip-ssh before parse_args ──────────────────────────────────────────────────────── -SKIP_SSH=false -SKIP_ARR_SYNC=false -FILTERED_ARGS=() -for arg in "$@"; do - case "$arg" in - --skip-ssh) SKIP_SSH=true ;; - --skip-arr-sync) SKIP_ARR_SYNC=true ;; - *) FILTERED_ARGS+=("$arg") ;; - esac -done - -parse_args "${FILTERED_ARGS[@]}" - -# ============================================================================================== -# ━━━ Setup ━━━ -# ============================================================================================== -if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root" - exit 1 -fi - -detect_hosts - -START=$(date +%s) - -echo "" -echo "━━━ $ICON_FALLBACK Partnership Onboard Setup — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -echo "" -echo " This server: $MY_ID ($LOCAL_SERVER_NAME)" -echo " Partner: $REMOTE_ID ($REMOTE_SERVER_NAME)" -echo "" - -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made" - -EXTRA_FLAGS=() -[[ "$DRY_RUN" == true ]] && EXTRA_FLAGS+=("--dry-run") -[[ "$LOG_MODE" == true ]] && EXTRA_FLAGS+=("--log") - -# ============================================================================================== -# ━━━ Step 1: SSH Key Setup ━━━ -# ============================================================================================== -echo "" -echo "━━━ Step 1/3 — SSH Key Setup ━━━" - -if [[ "$SKIP_SSH" == true ]]; then - warn "Skipping SSH setup (--skip-ssh)" -else - if bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then - log "SSH key setup complete ✅" - else - error "SSH key setup failed — aborting" - error "Fix SSH key issue then re-run, or use --skip-ssh if key is already set up" - exit 1 - fi -fi - -# ============================================================================================== -# ━━━ Step 2: Partnership Onboard ━━━ -# ============================================================================================== -echo "" -echo "━━━ Step 2/3 — Partnership Onboard ━━━" - -if bash "$SCRIPTS_ROOT/Partnership/partnership_manager.sh" --onboard "${EXTRA_FLAGS[@]}"; then - log "Partnership onboard complete ✅" - ONBOARD_OK=true -else - error "Partnership onboard failed" - ONBOARD_OK=false -fi - -# ============================================================================================== -# ━━━ Step 3: Arr Library Bootstrap ━━━ -# ============================================================================================== -echo "" -echo "━━━ Step 3/3 — Arr Library Bootstrap ━━━" - -ARR_SYNC_OK=true -if [[ "$ONBOARD_OK" == false ]]; then - warn "Skipping arr sync — onboard did not complete" - ARR_SYNC_OK=false -elif [[ "$SKIP_ARR_SYNC" == true ]]; then - warn "Skipping arr sync (--skip-arr-sync)" - ARR_SYNC_OK=false -else - ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh" - if [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then - warn "arr_sync.sh not found — skipping library bootstrap" - warn "Run Media/arr_sync.sh manually once arrs are live" - ARR_SYNC_OK=false - else - log "Syncing arr libraries with $REMOTE_SERVER_NAME..." - if bash "$ARR_SYNC_SCRIPT" "${EXTRA_FLAGS[@]}"; then - log "Arr library bootstrap complete ✅" - else - warn "Arr sync completed with errors — partnership is still valid" - warn "Re-run Media/arr_sync.sh once all arr containers are live" - ARR_SYNC_OK=false - fi - fi -fi - -# ============================================================================================== -# ━━━ Summary ━━━ -# ============================================================================================== -END=$(date +%s) -echo "" -echo "━━━━━ $ICON_SUMMARY PARTNERSHIP SETUP SUMMARY ━━━━━" -echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" -echo "$ICON_NET Partner: $REMOTE_SERVER_NAME" -echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" -echo "" - -_ssh_status() { [[ "$SKIP_SSH" == true ]] && echo "skipped" || echo "✅"; } -_arr_status() { - if [[ "$SKIP_ARR_SYNC" == true ]]; then echo "skipped (--skip-arr-sync)" - elif [[ "$ONBOARD_OK" == false ]]; then echo "skipped (onboard failed)" - elif [[ "$ARR_SYNC_OK" == true ]]; then echo "✅" - else echo "⚠️ errors — re-run arr_sync.sh once arrs are live" - fi -} - -echo " Step 1 — SSH key: $(_ssh_status)" -echo " Step 2 — Onboard: $( [[ "$ONBOARD_OK" == true ]] && echo "✅" || echo "❌" )" -echo " Step 3 — Arr bootstrap: $(_arr_status)" -echo "" - -if [[ "$ONBOARD_OK" == true ]]; then - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — no changes made" - else - log "$ICON_DONE DONE — partnership established ✅" - log "Next: verify with 'Partnership/partnership_manager.sh --status'" - fi -else - error "Setup incomplete — resolve onboard errors above and re-run" -fi - -echo "━━━━━━━━━━━━━━━━━━━━━━━" -[[ "$ONBOARD_OK" == false ]] && exit 1 -exit 0 diff --git a/Kernel/README-Kernel.md b/Kernel/README-Kernel.md new file mode 100644 index 0000000..26ea96f --- /dev/null +++ b/Kernel/README-Kernel.md @@ -0,0 +1,91 @@ +# ━━━━━ KERNEL ━━━━━ + +Shared intelligence layer for media automation. Sourced by consumer scripts — +not run directly. Contains stateful scoring systems, adaptive logic, and +cross-domain engines that would pollute common.sh if placed there. + +--- + +## ━━━ WHAT THIS FOLDER IS ━━━ + +**Not scripts. Not utilities. Engines.** + +The Kernel holds logic that is: + +- **Stateful** — maintains scores, decay state, or adaptive thresholds across calls +- **Domain-aware** — understands the difference between music, TV, and movie decisions +- **Shared across consumers** — one engine, multiple discovery scripts consuming it + +This is distinct from: + +| Location | Contains | +|----------|----------| +| `common.sh` | Reusable utility functions (logging, locking, arg parsing) | +| `master.conf` | Shared ecosystem configuration and tuning values | +| `Kernel/` | Stateful scoring systems and adaptive decision logic | + +The rule: if the logic makes *decisions*, it lives here. If it's a helper or +config value, it lives in common.sh or master.conf. + +--- + +## ━━━ WHY A SEPARATE KERNEL ━━━ + +Media automation decisions are not uniform. A Lidarr discovery script should be +highly selective. A Sonarr intake script should be family-aware and balanced. A +Radarr script should be broadly flexible. If scoring logic lives in each script, +it diverges and drifts independently. + +Centralizing the decision layer: + +- Keeps consumers thin — they define weights and thresholds, not scoring logic +- Prevents cross-domain bias pollution — music strictness does not contaminate TV +- Allows the scoring model to improve without touching every consumer script + +--- + +## ━━━ WHAT BELONGS HERE ━━━ + +An engine belongs in Kernel if it: + +- Implements a scoring or evaluation model +- Maintains or reads adaptive state +- Is consumed by more than one script (or is designed to be) +- Contains logic that would create tight coupling if duplicated + +Utilities — locking, notifications, arg parsing, API calls — belong in common.sh. + +--- + +## ━━━ FILES IN THIS FOLDER ━━━ + +| File | Role | Status | +|------|------|--------| +| `decision_engine.sh` | Behavior-driven scoring kernel — scoring, decay, deduplication, threshold evaluation | WIP — currently paired with `playback_aware_lidarr_discovery.sh` | + +**Planned:** + +| File | Role | +|------|------| +| `transcoding_engine.sh` | Transcoding decision logic (quality, codec selection, cost-benefit evaluation) | + +--- + +## ━━━ HOW CONSUMERS USE THE KERNEL ━━━ + +Source the engine at the top of the consumer script: + +```bash +source "$ROOT_DIR/Kernel/decision_engine.sh" +``` + +Then call engine functions directly, passing consumer-defined weights and thresholds: + +```bash +score=$(score_candidate "$user_score" "$popularity" "$recency" "$quality") +score=$(apply_temporal_decay "$score" "$age_days") +decision=$(make_decision "$score" "$MY_THRESHOLD") +``` + +The engine returns values — consumer scripts decide what to do with them. The +engine has no side effects: no file writes, no API calls, no deletions. diff --git a/Kernel/decision_engine.sh b/Kernel/decision_engine.sh index 278a545..638db21 100644 --- a/Kernel/decision_engine.sh +++ b/Kernel/decision_engine.sh @@ -1,128 +1,64 @@ #!/bin/bash # ============================================================================================== -# ================================= DECISION ENGINE ============================================ +# ================================= Decision Engine ============================================ # ============================================================================================== -# Central behavior-driven decision kernel used by media automation systems. # -# This engine does NOT download media. -# This engine does NOT search indexers. -# This engine does NOT manage applications directly. +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Behavior-driven scoring kernel for media automation. Sourced by consumer +# scripts — not run directly. Evaluates candidates, applies weighted scoring, +# temporal decay, and deduplication, then returns a verdict. # -# Instead: -# It evaluates candidates. -# Scores them against ecosystem behavior. -# Applies adaptive filtering rules. -# Returns decisions to consumer scripts. +# Does NOT download media, search indexers, or manage applications. Has no +# side effects — no file writes, no API calls, no deletions. +# +# Currently paired with: Media/playback_aware_lidarr_discovery.sh # # ============================================================================================== -# ── DESIGN PHILOSOPHY ───────────────────────────────────────────────────────────────────────── +# DESIGN PRINCIPLES +# ============================================================================================== # -# The ecosystem is built around: +# Domain-Agnostic Core +# The engine itself has no knowledge of music vs TV vs movies. Consumers +# define thresholds, weights, and strictness profiles — the engine just +# scores and decides. This prevents cross-domain bias pollution: music +# strictness cannot contaminate TV intake logic. # -# Family-aware decisions -# Time-aware weighting -# Behavior-driven adaptation -# Domain-specific strictness +# Lidarr consumers → highly selective, quality-first discovery +# Sonarr consumers → balanced, family-aware episodic intake +# Radarr consumers → broader flexibility with intelligent filtering # -# Each media domain consumes the engine differently: +# Temporal Decay +# Old behavioral signals lose influence over time (one decay unit per 30 +# days). Prevents permanent genre lock-in, historical bias accumulation, +# and dead-user score dominance. # -# Lidarr → highly selective, quality-first discovery -# Sonarr → balanced family-aware episodic intake -# Radarr → broader flexibility with intelligent filtering -# -# The engine itself remains domain-agnostic. -# Consumers define their own thresholds, weights, and strictness profiles. -# -# This separation prevents: -# -# Cross-domain bias pollution -# Unified-feed degeneration -# Overfitting to a single user's habits -# Low-quality recommendation drift over time -# -# Result: -# -# Music stays curated and intentional -# TV stays balanced across users -# Movies remain adaptive without chaos +# Consumer Owns the Decision +# The engine returns ACCEPT/REJECT/SCORE. What happens next is entirely +# the consumer's concern — the engine never acts on its own verdict. # # ============================================================================================== -# ── RESPONSIBILITIES ────────────────────────────────────────────────────────────────────────── -# -# The decision engine is responsible for: -# -# Candidate scoring -# User weighting -# Temporal decay -# Popularity normalization -# Duplicate prevention -# Strictness enforcement -# Threshold evaluation -# Final decision output -# -# The engine returns: -# -# ACCEPT -# REJECT -# SCORE -# REASON -# -# Consumer scripts decide what to do with the result. -# +# FUNCTIONS # ============================================================================================== -# ── ECOSYSTEM ROLE ──────────────────────────────────────────────────────────────────────────── # -# Kernel Position: +# score_candidate user_score popularity_score recency_score quality_score +# Returns TOTAL_SCORE (integer sum). Consumer defines actual weight values. # -# Kernel/ -# ├── decision_engine.sh -# ├── transcoding_engine.sh -# ├── future_engine_modules... +# evaluate_threshold score minimum +# Returns 0 (pass) or 1 (fail). Used as: if evaluate_threshold ... # -# Shared reusable logic belongs in: +# apply_temporal_decay score age_days +# Returns adjusted score. Subtracts (age_days / 30), floor at 0. # -# common.sh +# is_duplicate_candidate candidate history_file +# Returns 0 (duplicate found in history_file) or 1 (not found). # -# Shared ecosystem configuration belongs in: -# -# master.conf -# -# Host-specific secrets/configuration belong in: -# -# master_host*.conf -# -# The kernel contains: -# -# Stateful logic -# Adaptive systems -# Scoring systems -# Cross-domain intelligence -# -# ============================================================================================== -# ── VERSION ─────────────────────────────────────────────────────────────────────────────────── -# -# v1.0 -# Initial decision kernel architecture -# Built first for Lidarr discovery orchestration +# make_decision score threshold +# Returns "ACCEPT" or "REJECT". Calls evaluate_threshold internally. # # ============================================================================================== -# ============================================================================================== -# ── SCORE CANDIDATE ─────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Calculates weighted score for a media candidate. -# -# Inputs: -# USER_SCORE -# POPULARITY_SCORE -# RECENCY_SCORE -# QUALITY_SCORE -# -# Output: -# TOTAL_SCORE -# -# Consumer scripts define actual weighting values. - +# ── score_candidate ─────────────────────────────────────────────────────────── score_candidate() { local user_score="${1:-0}" @@ -140,14 +76,7 @@ score_candidate() { echo "$TOTAL_SCORE" } -# ============================================================================================== -# ── THRESHOLD CHECK ─────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Determines if candidate passes scoring threshold. -# -# Usage: -# evaluate_threshold "$score" "$minimum" - +# ── evaluate_threshold ─────────────────────────────────────────────────────── evaluate_threshold() { local score="$1" @@ -160,19 +89,7 @@ evaluate_threshold() { return 1 } -# ============================================================================================== -# ── TEMPORAL DECAY ──────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Reduces influence of old behavior over time. -# -# Prevents: -# Permanent genre lock-in -# Historical bias accumulation -# Dead-user dominance -# -# Usage: -# apply_temporal_decay current_score age_days - +# ── apply_temporal_decay ───────────────────────────────────────────────────── apply_temporal_decay() { local score="$1" @@ -187,20 +104,7 @@ apply_temporal_decay() { echo "$adjusted" } -# ============================================================================================== -# ── DUPLICATE PROTECTION ────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Prevents repetitive acquisitions. -# -# Consumer defines: -# cooldown periods -# replay windows -# duplicate tolerance -# -# Returns: -# 0 = duplicate -# 1 = unique - +# ── is_duplicate_candidate ─────────────────────────────────────────────────── is_duplicate_candidate() { local candidate="$1" @@ -209,15 +113,7 @@ is_duplicate_candidate() { grep -qi "^${candidate}$" "$history_file" 2>/dev/null } -# ============================================================================================== -# ── FINAL DECISION ──────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Produces final engine verdict. -# -# Outputs: -# ACCEPT -# REJECT - +# ── make_decision ───────────────────────────────────────────────────────────── make_decision() { local score="$1" diff --git a/Media/Manual-Media.md b/Media/Manual-Media.md new file mode 100644 index 0000000..449264f --- /dev/null +++ b/Media/Manual-Media.md @@ -0,0 +1,636 @@ +# ━━━━━ MEDIA — Manual ━━━━━ + +Config reference, procedures, operational workflows. +For overview see README-Media.md. For per-script detail see script headers. + +--- + +## ━━━ PERMISSIONS MODEL ━━━ + +``` +Directories: 755 nobody:users + Owner (nobody) — rwx enter, list, create files + Group (users) — r-x enter and list + Others — r-x Samba guests can browse + No world-write — prevents accidental deletion by unauthenticated access + +Files: 664 nobody:users + Owner (nobody) — rw read + write + Group (users) — rw arrs can import, rename, delete + Others — r Samba guests can read + No execute bit — media files are never executable +``` + +**Two separate passes — not a single recursive chmod.** Directories need the execute bit +to enter. Files must never have the execute bit. A single `chmod -R 664` would break +directory entry. The script runs `find -type d` and `find -type f` separately. + +**If this script corrects many files on every run**, a container has wrong PUID/PGID. +Correct values on unRAID: `PUID=99 (nobody)` `PGID=100 (users)`. Add to each container's +environment in its Docker template. Common culprits: SABnzbd, qBittorrent, slskd. +Once fixed, this script corrects 0 files per run — it becomes a pure daily failsafe. + +--- + +## ━━━ ARR CLEANUP — FILE CLASSIFICATION ━━━ + +Every file found on disk during an arr cleanup run falls into exactly one category: + +``` +TRACKED → arr API returned this exact path → leave it alone +PROTECTED → matches ARR_PROTECTED_PATTERNS → never delete +ORPHAN → media extension, not tracked, old enough → delete +JUNK → not a media extension, not protected → delete (any age) +RECENT → not tracked, under ARR_ORPHAN_AGE days → skip (may be mid-import) +``` + +**Why protected patterns are critical:** arrs generate artwork (`*.jpg`), metadata +(`*.nfo`), and subtitles/lyrics that do NOT appear in the tracked file API response. +Without protection, these would be classified as orphans and deleted — removing cover art +from every album, every movie poster, every TV show thumbnail. Requires a full rescan +to recover. Never remove artwork extensions from protected patterns. + +--- + +## ━━━ ARR CLEANUP — SAFETY LAYERS ━━━ + +All 7 layers must pass before any file is touched. There is no way to push through a +failed safety check without the explicit override flag. + +``` +1. Container running + healthy — a stopped container has an empty API +2. API reachable — no API = no tracked file list = everything looks orphaned +3. API version matches — major version must match tested version in master.conf +4. Item count > 0 — no artists/series/movies = something is wrong with DB +5. Tracked file count > 0 — empty response = everything would be deleted +6. Tracked count >= MIN_TRACKED_PCT — dramatic drop from last run = abort and alert +7. Deletion size < MAX_DELETE_GB — last line of defense against misconfigured root path +``` + +Layer 7 is the catastrophic failure prevention. A misconfigured root path — pointing +cleanup at the wrong directory — means the API returns zero tracked files for a root +that actually contains thousands. Everything walks as an orphan. Everything gets deleted. +`LIDARR/SONARR/RADARR_MAX_DELETE_GB` requires `--i-know-what-im-doing` to proceed past it. + +--- + +## ━━━ CONFIGURATION — master.conf ━━━ + +### Permissions + +```bash +PERMISSIONS_DIR_MODE="755" +PERMISSIONS_FILE_MODE="664" +PERMISSIONS_OWNER="nobody:users" +``` + +--- + +### Media Cleaner — File Patterns + +```bash +ANIME_FILE_PATTERNS=( + "*.sfv" # checksum verification — useless after download verified + "*.md5" "*.sha1" # other checksum formats + "*.nfo" # scene info file — not library metadata + "*.url" "*.lnk" # website shortcuts + "*.rar" "*.zip" # source archives kept by some clients after extraction + "*.info" # tool output files + "*.torrent" # torrent descriptor left by some clients + "*.sample*" # scene preview clip + "*.proof*" # screenshot proving encode quality + "*sync-conflict*" # Syncthing conflict copies + "*.scr" "*.exe" # executables — should never be in a media folder + "*.srr" # scene recovery record + "*.log" # tool/client logs + "*.json" # metadata or tool output +) + +MEDIA_FILE_PATTERNS=( + "${ANIME_FILE_PATTERNS[@]}" # all anime patterns plus: + "*.iso" # disc images after ripping + "*.lrc" # lyric files in media folders +) +``` + +> **DO NOT add patterns that match media you want to keep:** +> `*.mkv *.mp4 *.avi *.m4v` — video files +> `*.flac *.mp3 *.m4a` — audio files +> `*.srt *.sub *.ass` — subtitle files (Bazarr managed) +> `*.jpg *.png` — artwork +> Always use `--dry-run` when adding new patterns. + +--- + +### Lidarr Cleanup Thresholds + +```bash +LIDARR_ORPHAN_AGE=7 # days — files newer than this are RECENT (mid-import window) +LIDARR_MIN_TRACKED_PCT=80 # abort if API returns < 80% of last known count +LIDARR_MAX_DELETE_GB=50 # require --i-know-what-im-doing above this +LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan +LIDARR_VERSION_MAJOR=3 # expected Lidarr major version (API safety check) +LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") +LIDARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.lrc") +LIDARR_TRACKED_COUNT_FILE=/boot/config/lidarr_tracked_count # persistent baseline +ARR_CLEANUP_STATS=/boot/config/arr_cleanup_stats.db # read by coffee report +``` + +--- + +### Sonarr Cleanup Thresholds + +```bash +SONARR_ORPHAN_AGE=7 +SONARR_MAX_DELETE_GB=50 +SONARR_IMPORT_SCAN_TIMEOUT=600 +SONARR_VERSION_MAJOR=4 +SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") +SONARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa") +``` + +Note: `*.ts` IS in extensions — transport stream is used for Live TV recordings tracked +by Sonarr. Orphaned `.ts` recordings should be cleaned like any other orphaned episode. + +--- + +### Radarr Cleanup Thresholds + +```bash +RADARR_ORPHAN_AGE=7 +RADARR_MAX_DELETE_GB=50 +RADARR_IMPORT_SCAN_TIMEOUT=600 +RADARR_VERSION_MAJOR=6 +RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") +RADARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa") +``` + +--- + +### Arr Sync + +```bash +ARR_SYNC_ENABLED=true +ARR_SYNC_BLOCKLIST=/boot/config/arr_sync_blocklist.tsv # tombstone file +ARR_SYNC_CONNECT_TIMEOUT=10 # SSH connect timeout in seconds +ARR_SYNC_API_TIMEOUT=60 # curl API call timeout in seconds +DOCKER_APPDATA_BASE=/mnt/user/appdata +ARR_SYNC_LIDARR_PORT=8686 +ARR_SYNC_SONARR_PORT=8989 +ARR_SYNC_RADARR_PORT=7878 +``` + +--- + +### Arr Recovery + +```bash +ARR_IMPORT_RECOVERY_AGE=6 # hours — items newer than this are skipped +SONARR_VERSION_MAJOR=4 +RADARR_VERSION_MAJOR=6 +LIDARR_VERSION_MAJOR=3 +ARR_RECOVERY_STATS=/boot/config/arr_recovery_stats.db # read by coffee report +``` + +--- + +### TMDb / TVDB Removed + +```bash +RADARR_DROPPED_ADD_EXCLUSION=true # add removed movies to Radarr import exclusion +SONARR_DROPPED_ADD_EXCLUSION=true # add removed series to Sonarr import exclusion +``` + +--- + +### Lidarr Missing Art + +```bash +FANART_API_KEY="your-fanart-tv-api-key" +LASTFM_API_KEY="your-lastfm-api-key" +LIDARR_ART_MIN_SIZE=5000 # minimum valid download size in bytes +LIDARR_ART_MAX_PARALLEL=4 # concurrent background download jobs +LIDARR_ART_RETRIES=2 # download retry attempts per image +LIDARR_ART_SLEEP_BETWEEN=1 # seconds between fanart.tv API calls (rate limit) +``` + +--- + +### Orchestrator Job Order + +```bash +MEDIA_MAINTENANCE_JOBS=( + "Media/media_shares_permissions.sh" # 1. permissions — always first + "Media/media_cleaner.sh anime" # 2. junk removal — before orphan scan + "Media/media_cleaner.sh media" # 3. + "Media/lidarr_cleanup.sh" # 4. arr cleanup — after permissions + clean + "Media/sonarr_cleanup.sh" # 5. + "Media/radarr_cleanup.sh" # 6. +) +``` + +--- + +## ━━━ CONFIGURATION — master_host*.conf ━━━ + +### master_host1.conf + +```bash +# Shares this server applies permissions to +HOST1_MEDIA_PERMISSION_SHARES=( + "/mnt/user/Movies" + "/mnt/user/Tv_Shows" + "/mnt/user/Music" + "/mnt/user/Kids_Movies" + "/mnt/user/Kids_Tv_Shows" + "/mnt/user/Sports" + "/mnt/user/stand-up_comedy" +) + +# Folders cleaned by each profile +HOST1_ANIME_CLEAN_FOLDERS=( + "/mnt/user/Anime_Movies" + "/mnt/user/Anime_Movies-Old" + "/mnt/user/Anime_Shows" + "/mnt/user/Anime_Shows-Old" +) +HOST1_MEDIA_CLEAN_FOLDERS=( + "/mnt/user/Kids_Movies" + "/mnt/user/Kids_Tv_Shows" + "/mnt/user/Movies" + "/mnt/user/Music" + "/mnt/user/Sports" + "/mnt/user/stand-up_comedy" + "/mnt/user/Tv_Shows" +) + +# Arr connection details — must match arr settings exactly +HOST1_LIDARR_URL="http://192.168.50.2:8686" +HOST1_LIDARR_API_KEY="..." +HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music" +HOST1_LIDARR_PATH_MAP="" # container→host path translation if needed + +HOST1_SONARR_URL="http://192.168.50.2:8989" +HOST1_SONARR_API_KEY="..." +HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows" +HOST1_SONARR_PATH_MAP="" + +HOST1_RADARR_URL="http://192.168.50.2:7878" +HOST1_RADARR_API_KEY="..." +HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies" +HOST1_RADARR_PATH_MAP="" + +HOST1_EMBY_URL="http://192.168.50.2:8096" +HOST1_EMBY_API_KEY="..." +``` + +> **LIDARR/SONARR/RADARR_MUSIC/TV/MOVIES_ROOT must exactly match the Root Folder path in +> the arr's own settings.** Arr UI → Settings → Media Management → Root Folders. +> A mismatch means every file on disk looks untracked — all appear as orphans. +> MAX_DELETE_GB is the only thing standing between a path mismatch and losing your library. + +--- + +## ━━━ SAFE TESTING PROCEDURE ━━━ + +> **The arr cleanup scripts permanently delete files.** There is no recycle bin, no undo. +> Follow this procedure on first use, after any root path change, after any API key change, +> and after any significant arr library change. + +### Step 1 — Dry Run With Full Logging + +```bash +lidarr_cleanup.sh --dry-run --log +sonarr_cleanup.sh --dry-run --log +radarr_cleanup.sh --dry-run --log +``` + +### Step 2 — Review the Output + +``` +Are TRACKED files the ones you expect? + → Known arr-managed files should show as TRACKED + → If they show as ORPHAN, the root path is wrong — STOP + +Is the ORPHAN count reasonable? + → Healthy cleanup removes dozens to hundreds, not tens of thousands + → Large count = stop, investigate root path before proceeding + +Are PROTECTED patterns working? + → Artwork (*.jpg) and subtitles (*.srt) must show as PROTECTED + → If they show as ORPHAN, check PROTECTED_PATTERNS config + +Are RECENT files being correctly skipped? + → Files downloaded in the last 7 days should show as RECENT, not ORPHAN +``` + +### Step 3 — Check Numbers if Something Looks Wrong + +```bash +# Root path mismatch? Compare these: +# Lidarr UI: Settings → Media Management → Root Folders +# Sonarr UI: Settings → Media Management → Root Folders +# Radarr UI: Settings → Media Management → Root Folders +# Must exactly match LIDARR_MUSIC_ROOT / SONARR_TV_ROOT / RADARR_MOVIES_ROOT + +# Is the arr running? +docker ps | grep -E "Lidarr|Sonarr|Radarr" + +# Library scan not complete? +# Trigger manual scan in arr UI and wait for completion +``` + +### Step 4 — Run Live + +```bash +# Only after dry run review passes. +lidarr_cleanup.sh +sonarr_cleanup.sh +radarr_cleanup.sh +``` + +### Step 5 — Verify in Arr UI + +``` +Library count — should not have dropped significantly + healthy cleanup removes a small number, not a large percentage +Missing files — check if any monitored content shows as missing +Emby library — should show no ghost entries (notify_emby_scan handles this automatically) +``` + +--- + +## ━━━ PROCEDURES ━━━ + +### Adding a New Arr + +```bash +# 1. Copy radarr_cleanup.sh as template +cp radarr_cleanup.sh readarr_cleanup.sh + +# 2. Replace RADARR_ prefix with READARR_ throughout +# Update API endpoint, tracked file API path, extension list, protected patterns + +# 3. Add to master_host*.conf +HOST1_READARR_URL="http://192.168.50.2:8787" +HOST1_READARR_API_KEY="your-api-key" +HOST1_READARR_BOOKS_ROOT="/mnt/user/Books" + +# 4. Add thresholds to master.conf +READARR_ORPHAN_AGE=7 +READARR_MAX_DELETE_GB=50 +READARR_EXTENSIONS=("epub" "pdf" "mobi" "azw3" "cbz" "cbr") +READARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo") + +# 5. Add to MEDIA_MAINTENANCE_JOBS in master.conf +MEDIA_MAINTENANCE_JOBS=( + ...existing jobs... + "Media/readarr_cleanup.sh" +) +``` + +media_management.sh picks it up automatically. No orchestrator changes needed. +Run `--dry-run --log` before scheduling. + +--- + +### Managing the Arr Sync Blocklist + +```bash +# Add item to blocklist (removes from all arrs + tombstones the ID) +arr_sync.sh --blocklist-add lidarr "reason" +arr_sync.sh --blocklist-add sonarr "reason" +arr_sync.sh --blocklist-add radarr "reason" + +# Remove from blocklist (un-tombstones the ID — does NOT re-add to arrs) +arr_sync.sh --blocklist-remove lidarr + +# View all blocklisted IDs +arr_sync.sh --blocklist-list +``` + +`--blocklist-add` is the only destructive operation — it simultaneously: +1. Writes the tombstone entry to the blocklist TSV file +2. Deletes the item from the local arr API (no file deletion) +3. SSHes each remote node and deletes from their arr API + +Files become orphans on all nodes — arr_cleanup removes them on the next run. + +--- + +## ━━━ TROUBLESHOOTING ━━━ + +### Arr Cleanup Deleting Files It Shouldn't + +``` +1. Check the protected patterns — artwork and subtitles must be listed + LIDARR_PROTECTED_PATTERNS / SONARR_PROTECTED_PATTERNS / RADARR_PROTECTED_PATTERNS + +2. Check the root path matches arr settings exactly + Run: lidarr_cleanup.sh --status (shows configured root path) + Compare: Lidarr UI → Settings → Media Management → Root Folders + +3. Check if files are truly orphaned + Run: lidarr_cleanup.sh --dry-run --log + Look for the specific file — verify it shows ORPHAN, not PROTECTED or TRACKED +``` + +### Arr Cleanup Aborting at Safety Layer 6 (Tracked Count Drop) + +``` +API returned far fewer tracked files than last run. +Possible causes: + - Arr database was recently rebuilt from scratch + - Large manual library removal + - Path map mismatch after arr migration + +If intentional (library intentionally reduced): + Delete LIDARR_TRACKED_COUNT_FILE to reset the baseline + Run cleanup once — it will establish a new baseline + +If unintentional: + Investigate before proceeding — the arr may have a problem +``` + +### Arr Sync Not Picking Up New Content + +``` +Is ARR_SYNC_ENABLED=true in master.conf? + +Can this host SSH to the remote without password? + → ssh -i [SSH_KEY] root@[remote-tailscale-ip] "hostname" + +Is the arr accessible on the remote? + → arr_sync.sh --status (shows each node's arr reachability) + → arr_sync.sh --log (verbose output per-node, per-arr) + +Is the item in the blocklist? + → arr_sync.sh --blocklist-list +``` + +### Emby Still Showing Ghost Entries After Cleanup + +``` +notify_emby_scan() is called automatically after every arr cleanup deletion. +If ghosts persist: + 1. Is Emby's API responding? + curl -s "http://[emby-ip]:8096/System/Info/Public" + + 2. Is EMBY_URL / EMBY_API_KEY correct in master_host*.conf? + Run: sonarr_cleanup.sh --status (shows Emby config) + + 3. Trigger manually in Emby: + Library → Manage Library → Clean Missing Files +``` + +--- + +## ━━━ FLAG REFERENCE ━━━ + +### media_shares_permissions.sh + +`media_shares_permissions.sh` +Apply correct ownership and permissions to all configured media shares. Safe to run +manually at any time — idempotent, only changes what's wrong. + +`media_shares_permissions.sh --dry-run` +Show how many files and directories would be corrected per share. If unexpectedly large, +check container PUID/PGID settings first (PUID=99 PGID=100). + +`media_shares_permissions.sh --status` +Show configured share list and ownership of the share roots. + +`media_shares_permissions.sh --log` +Show ownership correction count per share and per file (verbose). + +> On large libraries this runs 20-30 minutes. This is expected — millions of files with +> recursive walk takes time. Designed to run overnight in the maintenance window. + +--- + +### media_cleaner.sh + +`media_cleaner.sh anime` +Remove junk files from anime share folders using ANIME_FILE_PATTERNS. + +`media_cleaner.sh media` +Remove junk files from media share folders using MEDIA_FILE_PATTERNS. + +`media_cleaner.sh [profile] --dry-run` +Show what would be deleted without removing anything. Always run first when adding new +patterns or folders. + +`media_cleaner.sh [profile] --status` +Show folder list and file patterns for the profile. + +`media_cleaner.sh [profile] --log` +Show every file examined, not just those removed. + +--- + +### lidarr_cleanup.sh / sonarr_cleanup.sh / radarr_cleanup.sh + +`[script] --dry-run --log` +Preview every classification decision. **Always run this first.** See Safe Testing Procedure. + +`[script]` +Live run — deletes confirmed orphans and junk, triggers Emby clean. + +`[script] --log` +Live run with verbose per-file output. + +`[script] --status` +Show configuration, API status, tracked file count, and last run stats. + +`[script] --i-know-what-im-doing` +Bypass the MAX_DELETE_GB size threshold. Required when deletion exceeds the configured +limit. Long flag name is intentional — cannot be added accidentally. + +`[script] --skip-strike-list` +Bypass the ORPHAN_AGE age check. Deletes RECENT files too — files that are under the +age threshold. Use when you know recent downloads are actually orphans. + +`[script] --i-know-what-im-doing --skip-strike-list` +**NUCLEAR MODE** — age check and size threshold both bypassed. Deletes on first pass. +Use when you want a clean one-pass wipe of everything the arr doesn't track. +No recovery possible after deletion. + +--- + +### arrs_failed_stalled_recovery.sh + +`arrs_failed_stalled_recovery.sh` +Check all configured arrs for failed imports and stalled downloads. Blocklist + remove + +re-search for each problem item. + +`arrs_failed_stalled_recovery.sh --dry-run` +Show what would be actioned per arr without making any changes. + +`arrs_failed_stalled_recovery.sh --status` +Show configuration, arr reachability, and last recovery stats. + +`arrs_failed_stalled_recovery.sh --log` +Verbose output per item per arr. + +--- + +### arr_sync.sh + +`arr_sync.sh` +Sync all arr types across all configured nodes. + +`arr_sync.sh --dry-run` +Show what would be added/removed on each node without making changes. + +`arr_sync.sh --status` +Show node configuration, arr reachability, and blocklist count. + +`arr_sync.sh --log` +Verbose per-node, per-arr output. + +`arr_sync.sh --blocklist-add [arr] [id] "[reason]"` +Remove item from all arrs and tombstone the ID. See Procedures above. + +`arr_sync.sh --blocklist-remove [arr] [id]` +Remove tombstone — does NOT re-add item to arrs. + +`arr_sync.sh --blocklist-list` +Show all tombstoned IDs. + +--- + +### radarr_tmdb_removed.sh / sonarr_tvdb_removed.sh + +`[script]` +Remove records for entries with status="deleted" (dropped from upstream database). +Files are kept. Import exclusion is added. + +`[script] --delete-files` +Also delete associated files from disk. Most dropped entries have no files — they were +announced movies/series that were never downloaded. + +`[script] --dry-run` +Preview what would be removed without making changes. + +`[script] --status` +Show arr connection status and current count of dropped entries. + +`[script] --log` +Verbose per-entry output. + +--- + +### lidarr_missing_art.sh + +`lidarr_missing_art.sh` +Fetch all missing album and artist artwork from fanart.tv and fallback sources. +Never overwrites existing files. + +`lidarr_missing_art.sh --dry-run` +Show what would be downloaded without writing any files. + +`lidarr_missing_art.sh --status` +Show configuration and API key status. + +`lidarr_missing_art.sh --log` +Verbose per-album, per-artist output. diff --git a/Media/README-Media.md b/Media/README-Media.md index ca94ff1..9d5a657 100644 --- a/Media/README-Media.md +++ b/Media/README-Media.md @@ -1,970 +1,213 @@ -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# 🎬 MEDIA -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# ━━━━━ MEDIA ━━━━━ -**Library health, cleanliness, and consistency for a multi-server arr stack.** -Correct permissions so arrs can do their job. Junk file removal so orphan detection -is not confused by scene debris. Orphaned file cleanup against live arr APIs so -deleted shows and removed albums actually leave the disk. Emby notified automatically -so users never see ghost library entries. +Library health, consistency, and sync for a multi-server arr stack. Correct permissions +so arrs can manage files. Junk removal so orphan detection isn't confused by scene debris. +Library sync so every node tracks the same content. Orphan cleanup against live arr APIs +so deleted content actually leaves disk. Emby notified automatically after every deletion. -> **These scripts permanently delete files.** The arr cleanup scripts are protected -> by multiple safety layers that must all pass before anything is touched — but dry -> runs and log review are still the right first step on any new system or after any -> configuration change. The testing procedure at the end of this document exists -> for a reason. +> **These scripts permanently delete files.** The arr cleanup scripts are protected by +> multiple safety layers that must all pass before anything is touched — but dry runs and +> log review are still the right first step on any new system or after any configuration +> change. The testing procedure in Manual-Media.md exists for a reason. --- -## ━━━ THE PROBLEM THAT BUILT THIS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ━━━ THE PROBLEM THAT BUILT THIS ━━━ + +**Files Owned by Root That Arrs Can't Touch** +Download clients without explicit PUID/PGID write files owned by root. Arrs running as +`nobody:users` cannot rename, move, or delete them. Import fails. Upgrade attempts fail. +The failure is subtle — arr shows the file as managed but can't touch it. You only +discover this when an upgrade is requested and the old version refuses to delete. +The fix: `media_shares_permissions.sh` applies correct ownership daily. Even if a +container is misconfigured, the window is at most 24 hours. + +**Scene Junk Confusing Orphan Detection** +Scene releases include `.sfv`, `.nfo`, `.rar`, `.sample` files alongside the actual media. +After extraction and import these are worthless — but they're not tracked by any arr. +They look like orphans. Processing them as orphans means the cleanup output is full of +noise, making it hard to spot actual orphaned media. +The fix: `media_cleaner.sh` runs before any arr cleanup and removes all known junk +patterns first. By the time arr cleanup runs, every untracked file is actual media. + +**Deleted Shows and Removed Albums Still on Disk** +When you remove a series from Sonarr and the delete command fails — permission issue, +container wasn't running, path mismatch — the files stay permanently. Over years on an +active library this accumulates significantly. +The fix: arr cleanup scripts query the live API for every tracked file path, walk the +disk, and delete anything absent from the API response that's old enough to be past the +import window. + +**Emby Showing Ghost Entries After Cleanup** +After arr cleanup deletes files, Emby still shows them until its next scheduled scan — +potentially hours later. Users see broken entries that produce "file not found" errors. +The fix: `notify_emby_scan()` is called automatically after every deletion. Triggers +Emby's "Clean Missing Files" task immediately. + +**No Safety Net on Deletion Size** +A misconfigured root path — pointing cleanup at the wrong directory — means the API +returns zero tracked files for a root that actually contains thousands. Every file walks +as an orphan. Everything gets deleted. This is the catastrophic failure mode. +The fix: `LIDARR/SONARR/RADARR_MAX_DELETE_GB` — if total deletion size exceeds the +limit, the script stops and requires `--i-know-what-im-doing` to proceed. The flag name +is long and annoying by design. It cannot be added by accident. --- -### 🔴 Files Owned by Root That Arrs Can't Touch +## ━━━ WHAT THIS FOLDER DOES ━━━ -Download clients running without explicit PUID/PGID configuration write files owned -by root. Arrs running as `nobody:users` cannot rename, move, or delete those files. -Import fails. Upgrade attempts fail. Deletion requests from arr UI do nothing. -The file sits on disk permanently in a state where the arr thinks it's managed but -can't actually manage it. +Scripts divide into five functional areas: -The failure mode is subtle — arr shows the file as imported, everything looks correct -in the UI, but the file is untouchable by the processes that need to touch it. You -only discover this when an upgrade is requested and the old version refuses to delete. +**Library Foundation** +`media_shares_permissions.sh` — normalize ownership and permissions daily. Runs first in +every maintenance window because arr cleanup depends on correct ownership to delete files. -The fix: `media_shares_permissions.sh` applies `nobody:users` ownership and correct -permissions to every file across every configured share, daily. Even if a container -is misconfigured, even if a manual admin copy created root-owned files, even after an -rsync that didn't preserve ownership — the next daily run normalises everything. +**Junk Removal** +`media_cleaner.sh` — remove scene debris and tool artifacts before orphan scan. Runs +second, before any arr cleanup, so orphan detection only encounters actual media files. + +**Library Sync** +`arr_sync.sh` — full-mesh arr library sync across all nodes. Every node syncs with every +other — union model, no hierarchy. Once arrs agree on what to track, rsync spreads the +actual files. The architectural shift from file-first sync to arr-first sync. + +**Orphan Cleanup** +`lidarr_cleanup.sh`, `sonarr_cleanup.sh`, `radarr_cleanup.sh` — API-verified orphan +removal. Five classification categories (TRACKED/PROTECTED/ORPHAN/JUNK/RECENT), seven +safety layers, automatic Emby notification after deletion. + +**Database Hygiene** +`radarr_tmdb_removed.sh`, `sonarr_tvdb_removed.sh` — remove entries that upstream +databases have dropped (TMDb/TVDB status="deleted"). These generate health warnings in +arrs and can never be monitored or downloaded. Most are announced-but-never-released +entries. Files are kept by default — most have none. + +**Library Enrichment** +`arrs_failed_stalled_recovery.sh` — detect and recover failed imports and stalled +downloads across all arrs. Blocklists the bad release and triggers a re-search — hands- +free overnight recovery. +`lidarr_missing_art.sh` — fetch missing album and artist artwork from fanart.tv and +fallback sources. Never overwrites existing files. + +**Discovery (WIP)** +`playback_aware_lidarr_discovery.sh` — scoring prototype for behavior-driven music +discovery. Consumes `Kernel/decision_engine.sh`. Not yet connected to a data source. --- -### 🔴 Scene Junk Files Confusing Orphan Detection +## ━━━ EXECUTION ORDER ━━━ -Scene releases — and many download groups for anime in particular — include a -constellation of junk files alongside the actual media: `.sfv` checksum files, `.nfo` -scene info, `.rar` source archives, `.sample` preview files, `.url` group websites. -After extraction and import these are worthless. They sit in the media folder forever. +Scripts run in multiple contexts — not all are part of the daily maintenance window: -The problem for arr cleanup: these files are not tracked by any arr. They are not -music files, TV files, or movie files. They look like orphans — but classifying them -as orphans means the orphan detection logic is constantly processing noise. Worse, -some cleanup implementations would try to delete them and log every single one as -a "cleaned orphan," making the output unreadable for finding actual orphaned media. +**Daily via `media_management.sh` (MEDIA_MAINTENANCE_JOBS):** -The fix: `media_cleaner.sh` runs before any arr cleanup script and removes all -known junk patterns first. By the time arr cleanup runs, every file it finds that -isn't tracked by the arr is actually a media file that shouldn't be there — not -a `.sfv` that was never tracked by anyone. - ---- - -### 🔴 Deleted Shows and Removed Albums Still on Disk - -When you remove a series from Sonarr, Sonarr deletes its database entries and — if -configured — sends delete commands for the files. When that fails (permission issue, -container wasn't running, file was on a different path than Sonarr expected), the -files stay on disk permanently. Sonarr has forgotten about them. Radarr has forgotten -about the movie. Lidarr has forgotten about the album. The files just sit there, -consuming space, invisible to the arr that should manage them. - -Over years on an active library this accumulates. A drive that should have 2TB free -has 400GB of content that no arr knows about. - -The fix: arr cleanup scripts that query the live API for every tracked file path, walk -the disk, and identify files that exist on disk but are absent from the API response. -These are genuine orphans — content the arr has definitively moved on from. With a -configurable age threshold to avoid deleting files that are mid-import, and a maximum -deletion size gate to catch configuration errors before they become disasters. - ---- - -### 🔴 Emby Showing Ghost Entries After Cleanup - -After arr cleanup deletes files, Emby's library still shows them — ghost entries that -produce "file not found" errors when clicked. Emby's library scan runs on its own -schedule, which might not happen for hours. Users see broken entries in the library -until the next scheduled scan. - -The fix: `notify_emby_scan()` — called automatically after every arr cleanup deletion. -Triggers Emby's "Clean Missing Files" task immediately. By the time the user refreshes -the library, the ghost entries are already gone. - ---- - -### 🔴 No Safety Net on Deletion Size - -A misconfigured root path — pointing arr cleanup at the wrong directory, or a root -path mismatch between master.conf and the arr's own settings — means the API returns -zero tracked files for a root that actually contains thousands. Every file walks as -an orphan. Everything gets deleted. A library is gone. - -This is the catastrophic failure mode. It happened during development — once, with a -test library. It cannot happen again. - -The fix: `LIDARR_MAX_DELETE_GB`, `SONARR_MAX_DELETE_GB`, `RADARR_MAX_DELETE_GB`. -If the total size of files queued for deletion exceeds the configured limit, the script -stops and requires `--i-know-what-im-doing` to proceed. The flag name is long and -annoying by design — it cannot be added by accident and it cannot be forgotten -what it means. - ---- - -## ━━━ WHY ORDER MATTERS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -These scripts are called in sequence by `Orchestrators/media_management.sh`. -The order is not arbitrary — each script depends on the previous ones having run. - -```bash -# master.conf — MEDIA_MAINTENANCE_JOBS defines the order -# ───────────────────────────────────────────────────────────────────────────── -MEDIA_MAINTENANCE_JOBS=( - "Media/media_shares_permissions.sh" # 1. Permissions — always first - "Media/media_cleaner.sh anime" # 2. Junk removal — before orphan scan - "Media/media_cleaner.sh media" # 3. Same for media shares - "Media/lidarr_cleanup.sh" # 4. Arr cleanup — after permissions + clean - "Media/sonarr_cleanup.sh" # 5. - "Media/radarr_cleanup.sh" # 6. -) +``` +1. media_shares_permissions.sh — permissions first — arr cleanup depends on this +2. media_cleaner.sh anime — junk before orphan scan +3. media_cleaner.sh media +4. lidarr_cleanup.sh — after permissions + clean +5. sonarr_cleanup.sh +6. radarr_cleanup.sh ``` -**Permissions before everything else:** -Arr cleanup needs `nobody:users` ownership to delete files. If a file is owned by -`root:root` and the script runs as `nobody`, the deletion fails silently — the file -looks like it was processed but is still on disk. Permissions first ensures this -can't happen. +**Weekly arr sync (before rsync in weekly_sync_maintenance.sh):** -**Junk removal before arr cleanup:** -The arr cleanup scripts walk the disk and compare against the arr API. Junk files -(`.sfv`, `.rar`, `.nfo`) are not tracked by any arr — they look like orphans. Removing -them first means the orphan detection only encounters actual media files. Cleaner -output, more accurate detection, less noise. +``` +arr_sync.sh — arrs agree on library → rsync then spreads the files +``` -**Arr cleanup last:** -Depends on clean folders and correct ownership. Both are guaranteed by the time -arr cleanup runs. +**Daily recovery (separate schedule — 5am or every 6hr):** + +``` +arrs_failed_stalled_recovery.sh +``` + +**Ad-hoc or separate schedule:** + +``` +lidarr_missing_art.sh — fetch missing artwork +radarr_tmdb_removed.sh — weekly cleanup of TMDb-dropped entries +sonarr_tvdb_removed.sh — weekly cleanup of TVDB-dropped entries +``` + +**Why permissions before everything else:** arr cleanup needs `nobody:users` ownership to +delete files. If a file is `root:root`, deletion fails silently — the file looks processed +but stays on disk. + +**Why junk before arr cleanup:** junk files are not tracked by any arr — they look like +orphans. Removing them first means orphan detection only finds actual media. Cleaner +output, more accurate detection. + +**Why arr_sync before rsync:** once arrs agree on what to track, rsync spreads the actual +files. An upgrade on one node — new tracked path, old path no longer in API — gets +cleaned by arr_cleanup on all nodes after the next sync cycle. --- -## ━━━ HOST AWARENESS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ━━━ HOST AWARENESS ━━━ -Each script runs on both servers but only operates on the shares and arrs that belong -to that server. `detect_hosts()` aliases all `HOST*_` prefixed vars to their -unprefixed names. +Scripts run on both servers but only operate on the shares and arrs that belong to that +server. `detect_hosts()` aliases all `HOST*_` prefixed vars to their unprefixed names. ``` HOST1 — source of truth for: HOST2 — source of truth for: - Movies (Radarr) Anime_Movies (his Radarr) - Tv_Shows (Sonarr) Anime_Shows (his Sonarr) - Music (Lidarr) - -Permissions and cleaner scripts run on both servers against their own shares. -Arr cleanup scripts check MY_ID and only run against the arr they own. -Lidarr only runs on HOST1 — exits cleanly on HOST2 with no action. + Movies (Radarr) Anime_Movies (his Radarr) + Tv_Shows (Sonarr) Anime_Shows (his Sonarr) + Music (Lidarr) ``` -No manual `HOST1`/`HOST2` comparisons exist in any of these scripts. Everything -routes through `MY_ID`. +Permissions and cleaner scripts run on both servers against their own shares. Arr cleanup +scripts check the aliased URL — if empty (arr not configured on this host), they exit +cleanly with no action. No manual `HOST1`/`HOST2` comparisons exist in any script. --- -## ━━━ SCRIPTS AT A GLANCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ━━━ SCRIPTS IN THIS FOLDER ━━━ -| Script | Purpose | When | -|--------|---------|------| -| `media_shares_permissions.sh` | Apply correct ownership + permissions to all media shares | Daily via `media_management.sh` | -| `media_cleaner.sh anime` | Remove junk files from anime share folders | Daily via `media_management.sh` | -| `media_cleaner.sh media` | Remove junk files from media share folders | Daily via `media_management.sh` | +| Script | Role | When It Runs | +|--------|------|--------------| +| `media_shares_permissions.sh` | Apply `nobody:users` ownership + correct permissions to all media shares | Daily via `media_management.sh` | +| `media_cleaner.sh` | Remove junk files (two profiles: `anime` + `media`) | Daily via `media_management.sh` | +| `arr_sync.sh` | Full-mesh arr library sync — all nodes track the same content | Weekly before rsync | | `lidarr_cleanup.sh` | Delete orphaned music files not tracked by Lidarr | Daily via `media_management.sh` | | `sonarr_cleanup.sh` | Delete orphaned TV files not tracked by Sonarr | Daily via `media_management.sh` | | `radarr_cleanup.sh` | Delete orphaned movie files not tracked by Radarr | Daily via `media_management.sh` | +| `arrs_failed_stalled_recovery.sh` | Auto-recover failed imports and stalled downloads | Daily (5am or every 6hr) | +| `lidarr_missing_art.sh` | Fetch missing album and artist artwork | Ad-hoc or separate schedule | +| `radarr_tmdb_removed.sh` | Remove movies dropped from TMDb | Ad-hoc or weekly | +| `sonarr_tvdb_removed.sh` | Remove series dropped from TVDB | Ad-hoc or weekly | +| `playback_aware_lidarr_discovery.sh` | Behavior-driven music discovery scoring (WIP prototype) | Not yet scheduled | --- -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🔐 media_shares_permissions.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Applies correct ownership and permissions recursively to all configured media shares. -The first job in the daily maintenance sequence — arr cleanup depends on this running -first. A failsafe that runs daily whether or not anything appears to need fixing. - ---- - -### ── The Permissions Model ─────────────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# unRAID's standard for Docker-accessible media shares. -# All arr containers, Emby, Tdarr, and download clients use nobody:users. -# These values match the PUID=99 PGID=100 container environment variables. -# -# Directories: 755 nobody:users -# Owner (nobody) — rwx enter, list, create files -# Group (users) — r-x enter and list -# Others — r-x Samba guests can browse -# No world-write — prevents accidental deletion by unauthenticated access -# -# Files: 664 nobody:users -# Owner (nobody) — rw read + write -# Group (users) — rw arrs can import, rename, delete -# Others — r Samba guests can read -# No execute bit — media files are never executable -# - PERMISSIONS_DIR_MODE="755" - PERMISSIONS_FILE_MODE="664" - PERMISSIONS_OWNER="nobody:users" -``` - -**Two separate passes — not a single recursive chmod:** -Directories need the execute bit (to enter them). Files must never have the execute -bit. A single `chmod -R 664` would remove execute from directories, making them -inaccessible. The script runs `find -type d` and `find -type f` separately with the -correct mode for each. - ---- - -### ── Why It Runs Daily Even When Things Look Fine ──────────────────────────── - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# Ownership drift happens from multiple sources, not just misconfigured containers: -# -# rsync without --chown → brings source server's nobody:users correctly -# but a misconfigured rsync brings root:root -# Manual admin copies → scp and cp default to root:root -# New containers → default to root before PUID/PGID is set -# unRAID updates → can reset container environment variables -# Download client restarts → if PUID/PGID lost on restart, next download is root -# -# Running daily means the window between "wrong ownership created" and -# "arr fails to manage the file" is at most 24 hours. Usually it is overnight. -# -# When this script corrects zero files per run, everything is configured correctly. -# When it corrects many files per run, a container has wrong PUID/PGID. -# Check these first: SABnzbd, qBittorrent, slskd — most common culprits. -# Correct values: PUID=99 PGID=100 in Docker template environment variables. -# -HOST1_MEDIA_PERMISSION_SHARES=( - "/mnt/user/Movies" - "/mnt/user/Tv_Shows" - "/mnt/user/Music" - "/mnt/user/Kids_Movies" - "/mnt/user/Kids_Tv_Shows" - "/mnt/user/Sports" - "/mnt/user/stand-up_comedy" -) -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Normal — called by media_management.sh, rarely run directly. -# Safe to run manually at any time — idempotent, only changes what's wrong. -# ───────────────────────────────────────────────────────────────────────────── -media_shares_permissions.sh - -# ───────────────────────────────────────────────────────────────────────────── -# Dry run — show how many files would be corrected per share. -# If the number is unexpectedly large, check container PUID/PGID settings first. -# ───────────────────────────────────────────────────────────────────────────── -media_shares_permissions.sh --dry-run - -# ───────────────────────────────────────────────────────────────────────────── -# Status — show configured share list and current ownership state. -# ───────────────────────────────────────────────────────────────────────────── -media_shares_permissions.sh --status - -# Verbose — show ownership correction count per share -media_shares_permissions.sh --log -``` - -> **On large libraries this runs for 20-30 minutes.** Millions of files with a -> recursive walk and chown/chmod on each takes time. This is expected and normal — -> it is why it runs overnight in the maintenance window, not during peak hours. - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🧹 media_cleaner.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Removes junk files from media share folders using configurable pattern lists. -Two profiles — `anime` and `media` — each with their own folder list and file patterns. -Runs after `media_shares_permissions.sh` and before any arr cleanup script. - ---- - -### ── What Gets Removed ──────────────────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# These patterns appear in anime releases and general media downloads. -# None of these are content you want in your library after extraction and import. -# -ANIME_FILE_PATTERNS=( - "*.sfv" # checksum verification — useless after download verified - "*.md5" "*.sha1" # other checksum formats — same reason - "*.nfo" # scene info file — group name, release notes, not metadata - "*.url" "*.lnk" # website shortcuts — never needed - "*.rar" "*.zip" # source archives — kept by some clients after extraction - "*.info" # tool output files - "*.torrent" # torrent descriptor left by some clients - "*.sample*" # scene preview clip — never needed after import - "*.proof*" # screenshot proving encode quality — never needed - "*sync-conflict*" # Syncthing conflict copies — should not be in media folders - "*.scr" "*.exe" # executables — should NEVER be in a media folder - "*.srr" # scene recovery record — useless post-download - "*.log" # tool/client logs - "*.json" # metadata or tool output -) - -# Media profile has additional patterns not relevant for anime: -MEDIA_FILE_PATTERNS=( - "${ANIME_FILE_PATTERNS[@]}" # all anime patterns plus: - "*.iso" # disc images after ripping — keep the ripped version - "*.lrc" # lyric files in media folders — not needed here -) -``` - ---- - -### ── Two Profiles — Different Folders ─────────────────────────────────────── - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# Profiles map to different share sets on each server. -# HOST1 owns Movies/Shows/Music. HOST2 owns Anime_Movies/Anime_Shows. -# detect_hosts() ensures each server only cleans the shares it owns. -# -HOST1_ANIME_CLEAN_FOLDERS=( - # HOST1 mirrors anime FROM HOST2 — clean the mirrored copies - "/mnt/user/Anime_Movies" - "/mnt/user/Anime_Movies-Old" - "/mnt/user/Anime_Shows" - "/mnt/user/Anime_Shows-Old" -) - -HOST1_MEDIA_CLEAN_FOLDERS=( - # HOST1's own shares — source of truth for these - "/mnt/user/Kids_Movies" - "/mnt/user/Kids_Tv_Shows" - "/mnt/user/Movies" - "/mnt/user/Music" - "/mnt/user/Sports" - "/mnt/user/stand-up_comedy" - "/mnt/user/Tv_Shows" -) -``` - ---- - -### ── Safety Note ───────────────────────────────────────────────────────────── +## ━━━ HOW THE SCRIPTS RELATE ━━━ ``` -The cleaner deletes by pattern — it does not check what any arr thinks about -the file. It runs before arr cleanup specifically so arr cleanup sees clean folders. +Weekly arr sync (before rsync): + arr_sync.sh ──────────────── syncs tracked IDs across all nodes + │ union model: any node adds → all nodes get it + │ + └── then rsync spreads the actual files to all nodes + └── then arr_cleanup removes orphans on all nodes (old paths, removed content) -DO NOT add patterns that match media you want to keep: - *.mkv *.mp4 *.avi *.m4v — video files - *.flac *.mp3 *.m4a — audio files - *.srt *.sub *.ass — subtitle files (managed by Bazarr) - *.jpg *.png — artwork (generated by arrs and Emby) +Daily maintenance window (media_management.sh): + media_shares_permissions.sh + │ (permissions correct — arr can now delete files) + ▼ + media_cleaner.sh (anime + media) + │ (junk removed — orphan scan finds only actual media) + ▼ + lidarr_cleanup.sh ──────────── queries Lidarr API → walks /Music → deletes orphans + sonarr_cleanup.sh ──────────── queries Sonarr API → walks /Tv_Shows → deletes orphans + radarr_cleanup.sh ──────────── queries Radarr API → walks /Movies → deletes orphans + │ + └── each cleanup → notify_emby_scan() → Emby removes ghost entries -Always use --dry-run when adding new patterns — review before committing. +Daily recovery: + arrs_failed_stalled_recovery.sh ── importFailed/stalled → blocklist → re-search + +Ad-hoc enrichment: + lidarr_missing_art.sh ─────── discovers missing artwork → fetches from fanart.tv + radarr_tmdb_removed.sh ────── status="deleted" → remove from Radarr + add exclusion + sonarr_tvdb_removed.sh ────── status="deleted" → remove from Sonarr + add exclusion ``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Profile is required — no default. -# Always run --dry-run first when adding new patterns or folders. -# ───────────────────────────────────────────────────────────────────────────── -media_cleaner.sh anime # clean anime share folders -media_cleaner.sh media # clean media share folders - -media_cleaner.sh anime --dry-run # preview — show what would be deleted -media_cleaner.sh media --dry-run # same for media profile - -media_cleaner.sh anime --log # verbose — show every file examined -media_cleaner.sh anime --status # show folder list and patterns -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🎵 lidarr_cleanup.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Removes orphaned music files from the library that Lidarr no longer tracks. -Queries the live Lidarr API for every tracked file path, walks the music root on disk, -and deletes anything that is not tracked, not protected, and old enough to be -definitively past the import window. - -> **Lidarr runs on HOST1 only.** This script exits cleanly on HOST2 with no action. -> HOST1 is source of truth for Music. - ---- - -### ── File Classification ────────────────────────────────────────────────────── - -``` -Every file found on disk falls into exactly one category: - -TRACKED → Lidarr API returned this exact path → leave it alone -PROTECTED → matches LIDARR_PROTECTED_PATTERNS → never delete -ORPHAN → music extension, not tracked, old enough → delete -JUNK → not a music extension, not protected → delete (any age) -RECENT → not tracked, under LIDARR_ORPHAN_AGE days → skip for now -``` - ---- - -### ── Why Protected Patterns Are Critical ──────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# Lidarr generates these files alongside your music — they are NOT in the -# tracked file API response. Without this list they would be deleted as orphans. -# -# Deleting *.jpg removes cover art from every album in the library. -# Emby and Lidarr both lose artwork display. Requires a full rescan to recover. -# -# Deleting *.nfo removes metadata Lidarr generated for media center compatibility. -# -# Deleting *.lrc removes synced lyrics used by music players that support them. -# - LIDARR_PROTECTED_PATTERNS=( - "*.jpg" "*.jpeg" "*.png" # cover art — generated by Lidarr per album - "*.nfo" # metadata — generated by Lidarr for media centers - "*.lrc" # synced lyrics — generated/downloaded by Lidarr - ) -# -# Never remove from this list without understanding what Lidarr generates in -# your specific setup. The consequences are visible and annoying to fix. -``` - ---- - -### ── Safety Layers — All Must Pass Before Any Deletion ────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Every check is a gate. If any gate fails, the script exits without touching -# a single file. There is no way to "push through" a failed safety check -# without the explicit override flag. -# -# 1. Container running + healthy -# Lidarr must be running and not in starting/unhealthy state. -# A stopped container has an empty (or inaccessible) API. -# -# 2. API reachable -# curl to the Lidarr endpoint must succeed. No API = no tracked file list. -# Without the tracked list, everything on disk looks like an orphan. -# -# 3. API version matches -# Major version must match the tested version in master.conf. -# API breaking changes between major versions would corrupt classification. -# -# 4. Artist count > 0 -# If Lidarr has no artists, something is wrong with the database or config. -# Proceeding would delete everything. -# -# 5. Tracked file count > 0 -# If Lidarr reports zero tracked files, the API returned an empty response. -# Proceeding would delete everything. -# -# 6. Tracked count >= LIDARR_MIN_TRACKED_PCT % of last known count -# If the API returns far fewer tracked files than the last run, something -# changed dramatically. Could be a Lidarr database corruption or a root -# path change. Stop and alert rather than delete thousands of files. -# -# 7. Deletion size < LIDARR_MAX_DELETE_GB -# If the total size of queued deletions exceeds this limit, stop. -# Require --i-know-what-im-doing to proceed. -# This is the last line of defense against a misconfigured root path. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Age Threshold ──────────────────────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# When Lidarr downloads a file it exists on disk before it is fully processed. -# The import queue can take hours for large batches. A newly downloaded file -# that isn't tracked yet is not an orphan — it is mid-import. -# -# LIDARR_ORPHAN_AGE defines the safe window. Files under this age are -# classified as RECENT and skipped entirely, regardless of tracked status. -# -# 7 days is conservative — adjust if your import workflow is slower than this. -# Be careful reducing this below 3 days — Lidarr's import retry logic can -# hold files in queue for 24-48 hours after initial download failure. -# - LIDARR_ORPHAN_AGE=7 # days — files newer than this are never classified as orphans -``` - ---- - -### ── Override Flags ──────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# --i-know-what-im-doing -# Required when deletion would exceed LIDARR_MAX_DELETE_GB. -# The flag name is long and specific by design — you must type it deliberately. -# It cannot be added by accident. It cannot be forgotten what it means. -# Use when you have intentionally removed a large portion of your library -# and want cleanup to follow through. -# -# --skip-strike-list -# Bypasses the LIDARR_ORPHAN_AGE age check. -# Deletes RECENT files too — files that are under the age threshold. -# Use when you know a batch of recently downloaded files are actually orphans -# and you want them cleaned without waiting for the age window. -# -# ⚠️ NUCLEAR MODE — both flags active simultaneously: -# Age check bypassed. Size threshold bypassed. Deletes everything on first pass. -# Use when Soularr has filled the gaps across your whole library and you want -# a clean one-pass wipe of everything Lidarr doesn't track. -# The script author takes no responsibility for data loss with both flags active. -# The user accepts full responsibility. This is intentional by design. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Configuration ──────────────────────────────────────────────────────────── - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# HOST1 only — Lidarr runs on HOST1, Music is HOST1's share. -# - HOST1_LIDARR_URL="http://192.168.50.2:8686" - HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc" - HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music" # must match Lidarr root path EXACTLY -# -# ───────────────────────────────────────────────────────────────────────────── -# LIDARR_MUSIC_ROOT must match the path configured in Lidarr: -# Lidarr UI → Settings → Media Management → Root Folders -# -# A mismatch means every file on disk looks untracked. -# All of them appear as orphans. LIDARR_MAX_DELETE_GB is the only thing -# between a path mismatch and losing your entire music library. -# ───────────────────────────────────────────────────────────────────────────── - -# master.conf - LIDARR_ORPHAN_AGE=7 - LIDARR_MIN_TRACKED_PCT=80 # alert if API returns < 80% of last known count - LIDARR_MAX_DELETE_GB=50 # stop if total deletion size exceeds 50GB - LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") - LIDARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.lrc") -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# ALWAYS run --dry-run --log first. Review every classification decision. -# See the safe testing procedure at the end of this document. -# ───────────────────────────────────────────────────────────────────────────── -lidarr_cleanup.sh --dry-run --log # full review before committing - -lidarr_cleanup.sh # live run after dry run passes review -lidarr_cleanup.sh --log # live run with verbose output -lidarr_cleanup.sh --status # show configuration and API status - -# Override flags — use with deliberate intention -lidarr_cleanup.sh --i-know-what-im-doing -lidarr_cleanup.sh --skip-strike-list -lidarr_cleanup.sh --i-know-what-im-doing --skip-strike-list # nuclear -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 📺 sonarr_cleanup.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Removes orphaned TV episode files from the library that Sonarr no longer tracks. -Same classification logic as `lidarr_cleanup.sh` applied to TV files. - -> **Sonarr is host-aware.** HOST1 Sonarr manages `Tv_Shows`. HOST2 Sonarr manages -> `Anime_Shows`. `detect_hosts()` aliases the correct URL, API key, and root path. -> The same script works correctly on both servers with no manual routing. - ---- - -### ── Protected Patterns — Sonarr ──────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# Sonarr generates artwork and metadata that do NOT appear in the tracked file -# API response. Additionally, Bazarr manages subtitles through Sonarr — -# subtitle files are not tracked by Sonarr directly. -# - SONARR_PROTECTED_PATTERNS=( - "*.jpg" "*.jpeg" "*.png" # show + episode artwork (per series and per episode) - "*.nfo" # metadata — NFO files for media center compatibility - "*.srt" "*.sub" # subtitles — managed by Bazarr via Sonarr - "*.ass" "*.ssa" # advanced subtitle formats — same - ) -# -# *.ts (transport stream) IS in SONARR_EXTENSIONS — not protected. -# Live TV recordings are .ts files tracked by Sonarr for recorded episodes. -# Orphaned .ts recordings should be cleaned up like any other orphaned episode. -``` - ---- - -### ── Post-Deletion Emby Notification ───────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# After deleting any files, notify_emby_scan() is called automatically. -# Triggers Emby's "Clean Missing Files" task immediately. -# Users see clean library entries within seconds of deletion — not hours later -# when Emby's own scheduled scan would eventually catch up. -# -# Configured via: - HOST1_EMBY_URL="http://192.168.50.2:8096" - HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829" -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Configuration ──────────────────────────────────────────────────────────── - -```bash -# master_host1.conf (HOST2 has matching HOST2_ vars for Anime_Shows) - HOST1_SONARR_URL="http://192.168.50.2:8989" - HOST1_SONARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" - HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows" # must match Sonarr root path EXACTLY - -# master.conf - SONARR_ORPHAN_AGE=7 - SONARR_MAX_DELETE_GB=50 - SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") - SONARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa") -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -sonarr_cleanup.sh --dry-run --log # always first -sonarr_cleanup.sh # live run -sonarr_cleanup.sh --log # verbose -sonarr_cleanup.sh --status # configuration and API status -sonarr_cleanup.sh --i-know-what-im-doing # size override -sonarr_cleanup.sh --skip-strike-list # age override -sonarr_cleanup.sh --i-know-what-im-doing --skip-strike-list # nuclear -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🎞️ radarr_cleanup.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Removes orphaned movie files from the library that Radarr no longer tracks. -Same classification logic applied to movie files. - -> **Radarr is host-aware.** HOST1 Radarr manages `Movies`. HOST2 Radarr manages -> `Anime_Movies`. `detect_hosts()` aliases the correct URL, API key, and root path. - ---- - -### ── Protected Patterns — Radarr ──────────────────────────────────────────── - -```bash -# master.conf - RADARR_PROTECTED_PATTERNS=( - "*.jpg" "*.jpeg" "*.png" # movie artwork — fanart, posters, thumbnails - "*.nfo" # metadata — NFO for media center compatibility - "*.srt" "*.sub" # subtitles — managed by Bazarr via Radarr - "*.ass" "*.ssa" # advanced subtitle formats — same - ) -``` - ---- - -### ── Configuration ──────────────────────────────────────────────────────────── - -```bash -# master_host1.conf (HOST2 has matching HOST2_ vars for Anime_Movies) - HOST1_RADARR_URL="http://192.168.50.2:7878" - HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" - HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies" # must match Radarr root path EXACTLY - -# master.conf - RADARR_ORPHAN_AGE=7 - RADARR_MAX_DELETE_GB=50 - RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") - RADARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa") -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -radarr_cleanup.sh --dry-run --log # always first -radarr_cleanup.sh # live run -radarr_cleanup.sh --log # verbose -radarr_cleanup.sh --status # configuration and API status -radarr_cleanup.sh --i-know-what-im-doing # size override -radarr_cleanup.sh --skip-strike-list # age override -radarr_cleanup.sh --i-know-what-im-doing --skip-strike-list # nuclear -``` - ---- - -## ━━━ SAFE TESTING PROCEDURE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -> **The arr cleanup scripts permanently delete files.** There is no recycle bin. -> There is no undo. Follow this procedure on first use, after any root path -> change, after any API key change, and after any significant arr library change. - ---- - -### Step 1 — Dry Run With Full Logging - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# --dry-run: no files are touched -# --log: every classification decision is printed -# Review the complete output before proceeding. -# ───────────────────────────────────────────────────────────────────────────── -lidarr_cleanup.sh --dry-run --log -sonarr_cleanup.sh --dry-run --log -radarr_cleanup.sh --dry-run --log -``` - ---- - -### Step 2 — Review the Output Carefully - -``` -Ask these questions about the dry run output: - -Are TRACKED files the ones you expect? - → Files you know are in the arr should show as TRACKED. - → If known files show as ORPHAN, the root path may be wrong. - -Is the ORPHAN count reasonable? - → A healthy library cleanup removes dozens to hundreds of files, not tens of thousands. - → If the number is unexpectedly large, stop. Do not proceed. - -Are PROTECTED patterns working? - → You should see artwork (*.jpg) and subtitles (*.srt) classified as PROTECTED. - → If they show as ORPHAN, the protected patterns are wrong. - -Are RECENT files being correctly skipped? - → Files downloaded in the last 7 days should show as RECENT, not ORPHAN. - → If recent downloads show as ORPHAN, the age threshold may be too aggressive. - -Is the root path correct? - → The scan should start at the right directory. - → Check the first few lines of output — the scan root is logged. -``` - ---- - -### Step 3 — Check the Numbers - -```bash -# If dry run reports an unexpectedly large number of orphans, check: - -# 1. Root path mismatch — what does arr think the root is? -# Lidarr: Settings → Media Management → Root Folders -# Sonarr: Settings → Media Management → Root Folders -# Radarr: Settings → Media Management → Root Folders -# Must exactly match LIDARR_MUSIC_ROOT / SONARR_TV_ROOT / RADARR_MOVIES_ROOT - -# 2. API returning empty — is the arr running? -docker ps | grep -E "Lidarr|Sonarr|Radarr" - -# 3. Library scan not complete — recently added content not yet indexed -# Trigger a manual library scan in the arr UI and wait for completion -``` - ---- - -### Step 4 — Run Live - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Only after dry run review passes. Start with the arr you're most confident about. -# ───────────────────────────────────────────────────────────────────────────── -lidarr_cleanup.sh -sonarr_cleanup.sh -radarr_cleanup.sh -``` - ---- - -### Step 5 — Verify in Arr UI - -``` -After running, check in the arr's UI: - -Library count: should not have dropped significantly - a healthy cleanup removes a small number of files, - not a large percentage of the library - -Missing files: check if any monitored content shows as missing - this would indicate a tracked file was incorrectly deleted - -Emby library: should show no ghost entries (notify_emby_scan handles this automatically) - if ghost entries appear, trigger a manual "Clean Missing Files" task in Emby -``` - ---- - -## ━━━ CONFIGURATION REFERENCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - ---- - -### 📋 master_host*.conf — Per-Host Configuration - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── - -# Shares this server applies permissions to -HOST1_MEDIA_PERMISSION_SHARES=( - "/mnt/user/Movies" - "/mnt/user/Tv_Shows" - "/mnt/user/Music" -) - -# Folders cleaned by each profile -HOST1_ANIME_CLEAN_FOLDERS=( - "/mnt/user/Anime_Movies" - "/mnt/user/Anime_Shows" -) -HOST1_MEDIA_CLEAN_FOLDERS=( - "/mnt/user/Movies" - "/mnt/user/Tv_Shows" - "/mnt/user/Music" -) - -# Arr connection details — must match arr settings exactly -HOST1_LIDARR_URL="http://192.168.50.2:8686" -HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc" -HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music" - -HOST1_SONARR_URL="http://192.168.50.2:8989" -HOST1_SONARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" -HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows" - -HOST1_RADARR_URL="http://192.168.50.2:7878" -HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" -HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies" - -HOST1_EMBY_URL="http://192.168.50.2:8096" -HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829" -``` - ---- - -### 📋 master.conf — Shared Configuration - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── - -# ── Permissions ──────────────────────────────────────────────────────────── - PERMISSIONS_DIR_MODE="755" - PERMISSIONS_FILE_MODE="664" - PERMISSIONS_OWNER="nobody:users" - -# ── File Patterns ────────────────────────────────────────────────────────── - ANIME_FILE_PATTERNS=("*.sfv" "*.md5" "*.sha1" ...) - MEDIA_FILE_PATTERNS=("${ANIME_FILE_PATTERNS[@]}" "*.iso" "*.lrc") - -# ── Arr Cleanup Thresholds ───────────────────────────────────────────────── - LIDARR_ORPHAN_AGE=7 # days — files newer than this are RECENT - LIDARR_MIN_TRACKED_PCT=80 # alert if API returns < 80% of last known count - LIDARR_MAX_DELETE_GB=50 # stop if total deletion exceeds this - LIDARR_EXTENSIONS=(...) - LIDARR_PROTECTED_PATTERNS=(...) - - SONARR_ORPHAN_AGE=7 - SONARR_MAX_DELETE_GB=50 - SONARR_EXTENSIONS=(...) - SONARR_PROTECTED_PATTERNS=(...) - - RADARR_ORPHAN_AGE=7 - RADARR_MAX_DELETE_GB=50 - RADARR_EXTENSIONS=(...) - RADARR_PROTECTED_PATTERNS=(...) - -# ── Orchestrator Job Order ───────────────────────────────────────────────── - MEDIA_MAINTENANCE_JOBS=( - "Media/media_shares_permissions.sh" - "Media/media_cleaner.sh anime" - "Media/media_cleaner.sh media" - "Media/lidarr_cleanup.sh" - "Media/sonarr_cleanup.sh" - "Media/radarr_cleanup.sh" - ) -``` - ---- - -## ━━━ ADDING A NEW ARR ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -To add Readarr, Whisparr, or any other arr cleanup to the ecosystem: - -```bash -# 1. Copy radarr_cleanup.sh as the template — same classification logic applies -cp radarr_cleanup.sh readarr_cleanup.sh - -# 2. Update the API endpoint variable names and root path variable names -# Replace RADARR_ with READARR_ throughout - -# 3. Add configuration to master_host*.conf -HOST1_READARR_URL="http://192.168.50.2:8787" -HOST1_READARR_API_KEY="your-api-key" -HOST1_READARR_BOOKS_ROOT="/mnt/user/Books" - -# 4. Add thresholds to master.conf -READARR_ORPHAN_AGE=7 -READARR_MAX_DELETE_GB=50 -READARR_EXTENSIONS=("epub" "pdf" "mobi" "azw3" "cbz" "cbr") -READARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo") - -# 5. Add to MEDIA_MAINTENANCE_JOBS in master.conf -MEDIA_MAINTENANCE_JOBS=( - ...existing jobs... - "Media/readarr_cleanup.sh" # add at the end — after permissions and cleaner -) - -# ───────────────────────────────────────────────────────────────────────────── -# media_management.sh picks it up automatically. -# No changes to the orchestrator needed. -# Run --dry-run --log before scheduling. -# ───────────────────────────────────────────────────────────────────────────── -``` \ No newline at end of file diff --git a/Media/arr_sync.sh b/Media/arr_sync.sh index 61a5f90..d60962e 100755 --- a/Media/arr_sync.sh +++ b/Media/arr_sync.sh @@ -1,78 +1,116 @@ #!/bin/bash # ============================================================================================== -# ================================= ARR SYNC =================================================== +# ================================= ARR Sync =================================================== # ============================================================================================== -# Bidirectional arr library sync across all nodes in the ecosystem. -# Syncs Lidarr, Sonarr, and Radarr libraries so every node tracks the same content. -# Run before rsync — once arrs agree on library, rsync spreads the files. # -# ── DESIGN ──────────────────────────────────────────────────────────────────────────────────── -# Full mesh: every node syncs with every other node — no primary, no hierarchy. +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Full-mesh arr library sync across all nodes — Lidarr, Sonarr, and Radarr. +# Every node syncs with every other, union model, no hierarchy. Run before +# rsync in the weekly sync window: once arrs agree on what to track, rsync +# spreads the actual files. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Full mesh: every node syncs with every other — no primary, no hierarchy. # Union model: if any node tracks an item, all nodes get it (unless blocklisted). # Convergence: any node can add content; after one full cycle all nodes agree. -# Upgrade-aware: server1 upgrades a file → arr tracks new path → rsync spreads it → -# arr_cleanup removes old file on all nodes because arr no longer tracks it. +# Upgrade-aware: a file upgrade on one node → arr tracks new path → rsync spreads +# it → arr_cleanup removes old path on all nodes (arr no longer tracks it). # -# ── NODE DISCOVERY ──────────────────────────────────────────────────────────────────────────── -# Reads HOST* vars from master.conf. Add HOST3= and it joins the sync automatically. -# No scripts change when adding a new node. +# Node discovery: reads HOST* vars from master.conf. Add HOST3= and it joins the +# sync automatically — no script changes needed for a new node. # -# ── REMOTE API KEY ACCESS ───────────────────────────────────────────────────────────────────── -# Remote API keys are not stored anywhere. Script SSHes to each remote node and reads -# the key directly from that arr's config.xml in its appdata directory. -# Only the API response (JSON) is returned — key never leaves the remote node. -# Self-maintaining: remote key regeneration is picked up automatically. +# Graceful skip: arr not configured locally → skip cleanly. Arr not reachable on +# a remote → skip that node for that arr type, continue with others. # -# ── BLOCKLIST ───────────────────────────────────────────────────────────────────────────────── -# ARR_SYNC_BLOCKLIST in DATA_DIR tombstones IDs that must never be re-added anywhere. -# Read from ALL nodes via SSH at start of each run — immediate effect across all nodes. -# (Each node reads every other node's blocklist file via SSH — no rsync delay.) -# Manage via --blocklist-add / --blocklist-remove / --blocklist-list. -# -# --blocklist-add does three things atomically: -# 1. Writes the TSV tombstone entry (prevents future re-adds by arr_sync) -# 2. Deletes the item from the local arr API (deleteFiles=false) -# 3. SSHes each remote node and deletes from their arr API (deleteFiles=false) -# Files become orphans on all nodes — arr_cleanup.sh removes them on next run. -# -# ── GRACEFUL SKIP ───────────────────────────────────────────────────────────────────────────── -# Arr not configured locally → skip cleanly, no error. -# Arr not reachable on a remote → skip that node for that arr type, continue with others. -# Partial mesh works — nodes that share an arr type sync with each other. -# -# ── WHAT GETS SYNCED ────────────────────────────────────────────────────────────────────────── -# Library items (tracked artists/series/movies) keyed on stable IDs: +# What gets synced — library items keyed on stable external IDs: # Lidarr — MusicBrainz artist ID (foreignArtistId) # Sonarr — TVDB series ID (tvdbId) # Radarr — TMDB movie ID (tmdbId) -# When adding to a remote node, that node's own quality profile, metadata profile, -# and root folder path are used — settings are never copied from the source node. +# When adding to a remote, that node's own quality profile, metadata profile, +# and root folder path are used — settings are never copied from source. +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Remote API Keys Never Stored +# SSHes to each remote node and reads the key directly from that arr's +# config.xml in its appdata directory. Only the API response (JSON) is +# returned — the key never leaves the remote node. Self-maintaining: key +# regeneration on the remote is picked up automatically next run. +# +# Blocklist TSV +# ARR_SYNC_BLOCKLIST in DATA_DIR tombstones IDs that must never be re-added +# anywhere. Read from ALL nodes via SSH at run start — immediate effect with +# no rsync delay. +# +# --blocklist-add does three things atomically: +# 1. Writes the TSV tombstone entry (prevents future re-adds by arr_sync) +# 2. Deletes the item from the local arr API (deleteFiles=false) +# 3. SSHes each remote node and deletes from their arr API (deleteFiles=false) +# Files become orphans on all nodes — arr_cleanup removes them on next run. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# acquire_lock — prevents two sync instances running simultaneously +# ARR_SYNC_ENABLED — global gate, exits cleanly when false +# SSH connect timeout — ARR_SYNC_CONNECT_TIMEOUT — does not hang on unreachable node +# API call timeout — ARR_SYNC_API_TIMEOUT — does not hang on slow arr +# Graceful skip — unreachable node/arr → skip and continue, never abort +# Blocklist gate — item in blocklist → never added to any node +# Silent by default — only additions produce output, clean runs stay silent +# +# ============================================================================================== +# STATE FILES +# ============================================================================================== +# +# ARR_SYNC_BLOCKLIST — TSV file in DATA_DIR (default: DATA_DIR/arr_sync_blocklist.tsv) +# Columns: arr_type, id, reason, date_added +# Read from all nodes via SSH at the start of each run. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# +# HOST*_LIDARR_URL / HOST*_LIDARR_API_KEY — local Lidarr (aliased by detect_hosts) +# HOST*_SONARR_URL / HOST*_SONARR_API_KEY — local Sonarr +# HOST*_RADARR_URL / HOST*_RADARR_API_KEY — local Radarr +# +# master.conf +# +# ARR_SYNC_ENABLED — global on/off toggle (default: true) +# ARR_SYNC_BLOCKLIST — path to TSV blocklist file +# ARR_SYNC_CONNECT_TIMEOUT — SSH connect timeout in seconds (default: 10) +# ARR_SYNC_API_TIMEOUT — curl API call timeout in seconds (default: 60) +# DOCKER_APPDATA_BASE — base path for arr appdata dirs (default: /mnt/user/appdata) +# ARR_SYNC_LIDARR_PORT — Lidarr port on all nodes (default: 8686) +# ARR_SYNC_SONARR_PORT — Sonarr port on all nodes (default: 8989) +# ARR_SYNC_RADARR_PORT — Radarr port on all nodes (default: 7878) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # arr_sync.sh — sync all arr types, all nodes # arr_sync.sh --dry-run — preview only, no changes # arr_sync.sh --log — verbose output # arr_sync.sh --status — show config and exit -# arr_sync.sh --blocklist-add lidarr "reason" — remove from all arrs + tombstone +# +# Blocklist management: +# arr_sync.sh --blocklist-add lidarr "reason" — remove from all arrs + tombstone # arr_sync.sh --blocklist-add sonarr "reason" — remove from all arrs + tombstone # arr_sync.sh --blocklist-add radarr "reason" — remove from all arrs + tombstone # arr_sync.sh --blocklist-remove lidarr — un-tombstone (does NOT re-add) -# arr_sync.sh --blocklist-list — show all blocklisted IDs +# arr_sync.sh --blocklist-list — show all blocklisted IDs # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# ARR_SYNC_ENABLED — global on/off toggle (default true) -# ARR_SYNC_BLOCKLIST — path to TSV blocklist (default: DATA_DIR/arr_sync_blocklist.tsv) -# ARR_SYNC_CONNECT_TIMEOUT — SSH connect timeout in seconds (default 10) -# ARR_SYNC_API_TIMEOUT — curl API call timeout in seconds (default 60) -# DOCKER_APPDATA_BASE — base path for arr appdata dirs (default /mnt/user/appdata) -# ARR_SYNC_LIDARR_PORT — Lidarr port on all nodes (default 8686) -# ARR_SYNC_SONARR_PORT — Sonarr port on all nodes (default 8989) -# ARR_SYNC_RADARR_PORT — Radarr port on all nodes (default 7878) -# -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── -# HOST*_LIDARR_URL / HOST*_LIDARR_API_KEY — local Lidarr (aliased by detect_hosts) -# HOST*_SONARR_URL / HOST*_SONARR_API_KEY — local Sonarr -# HOST*_RADARR_URL / HOST*_RADARR_API_KEY — local Radarr # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Media/arrs_failed_stalled_recovery.sh b/Media/arrs_failed_stalled_recovery.sh index 6d45e81..88001f6 100644 --- a/Media/arrs_failed_stalled_recovery.sh +++ b/Media/arrs_failed_stalled_recovery.sh @@ -2,13 +2,19 @@ # ============================================================================================== # ========================= Arrs Failed / Stalled Recovery ===================================== # ============================================================================================== -# Automatically detects and recovers from failed imports and stalled downloads -# across Sonarr, Radarr, and Lidarr. Blocklists the bad release and triggers -# a new search — hands-free recovery while you sleep. # -# ── WHAT IT CHECKS ──────────────────────────────────────────────────────────────────────────── -# Four problem types from the arr queue API: -# importFailed — downloaded successfully but arr couldn't import the file +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Detect and recover failed imports and stalled downloads across Sonarr, Radarr, +# and Lidarr. Blocklists the bad release and triggers a re-search — hands-free +# overnight recovery. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Four problem types detected from the arr queue API: +# importFailed — downloaded but arr couldn't import the file # importPending — downloaded, stuck waiting to import (will not self-resolve) # error status — serious failure not covered by the above two states # stalled — download stuck with no connections or no progress @@ -16,55 +22,65 @@ # Never touches items with state "downloading" or "imported" — safe to run anytime. # Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first. # -# ── WHAT IT DOES PER PROBLEM ITEM ───────────────────────────────────────────────────────────── +# Per problem item (3-step response): # 1. Blocklist the release — prevents re-grabbing the same bad release # 2. Remove from queue — cleans up the failed item # 3. Trigger new search — finds a different release automatically # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID and aliases all arr vars: -# SONARR_URL / SONARR_API_KEY / SONARR_RECOVERY -# RADARR_URL / RADARR_API_KEY / RADARR_RECOVERY -# LIDARR_URL / LIDARR_API_KEY / LIDARR_RECOVERY (HOST1 only — exits cleanly on HOST2) -# No manual HOST1/HOST2 comparisons needed — MY_ID routes automatically. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── API VERSION SAFETY ──────────────────────────────────────────────────────────────────────── -# check_arr_version() verifies the running arr matches the tested major version in master.conf. -# If the API structure changed after an upgrade — exits rather than silently misoperating. +# acquire_lock — prevents concurrent runs overlapping +# jq validation — exits if jq not installed (required for JSON parsing) +# API pre-flight — checks each arr is reachable before querying queue +# Version check — check_arr_version() verifies running arr matches master.conf major +# version; exits rather than silently misoperating after upgrade +# Age threshold — skips items newer than ARR_IMPORT_RECOVERY_AGE (default 6hr) +# Silent by default — only problems produce output, clean arrs stay silent +# +# API version mapping (endpoint paths differ from major version labels): # Sonarr v4 → /api/v3/ (v3 endpoint retained in v4) # Radarr v6 → /api/v3/ (v3 endpoint retained in v6) # Lidarr v3 → /api/v1/ (different from Sonarr/Radarr) # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock — prevents concurrent runs overlapping -# jq validation — exits if jq not installed (required for JSON parsing) -# API pre-flight — checks each arr is reachable before querying queue -# Version check — verifies arr major version matches tested version in master.conf -# Age threshold — skips items newer than ARR_IMPORT_RECOVERY_AGE (default 6hr) -# Silent by default — only problems produce output, clean arrs stay silent +# ============================================================================================== +# STATE FILES +# ============================================================================================== +# +# ARR_RECOVERY_STATS — stats file written after each run (read by coffee report) +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── # HOST*_SONARR_URL / HOST*_SONARR_API_KEY / HOST*_SONARR_RECOVERY # HOST*_RADARR_URL / HOST*_RADARR_API_KEY / HOST*_RADARR_RECOVERY # HOST1_LIDARR_URL / HOST1_LIDARR_API_KEY / HOST1_LIDARR_RECOVERY # All aliased by detect_hosts() — script uses unprefixed names # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# ARR_IMPORT_RECOVERY_AGE — hours before item is eligible (default 6) -# SONARR_VERSION_MAJOR — expected Sonarr major version (e.g. 4) -# RADARR_VERSION_MAJOR — expected Radarr major version (e.g. 6) -# LIDARR_VERSION_MAJOR — expected Lidarr major version (e.g. 3) -# ARR_RECOVERY_STATS — stats file path (read by coffee report) +# master.conf +# +# ARR_IMPORT_RECOVERY_AGE — hours before item is eligible for recovery (default: 6) +# SONARR_VERSION_MAJOR — expected Sonarr major version (e.g. 4) +# RADARR_VERSION_MAJOR — expected Radarr major version (e.g. 6) +# LIDARR_VERSION_MAJOR — expected Lidarr major version (e.g. 3) +# ARR_RECOVERY_STATS — stats file path (read by coffee report) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # arrs_failed_stalled_recovery.sh — normal run # arrs_failed_stalled_recovery.sh --dry-run — show what would be actioned, no changes # arrs_failed_stalled_recovery.sh --log — verbose output # arrs_failed_stalled_recovery.sh --status — show config and exit # -# ── SCHEDULE ────────────────────────────────────────────────────────────────────────────────── -# Recommended: 0 5 * * * (5am daily) -# Or every 6hr: 0 */6 * * * (matches ARR_IMPORT_RECOVERY_AGE default) +# Recommended schedule: 0 5 * * * (5am daily) +# Or every 6hr: 0 */6 * * * (matches ARR_IMPORT_RECOVERY_AGE default) +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Media/lidarr_cleanup.sh b/Media/lidarr_cleanup.sh index a2e4516..bec8f9c 100644 --- a/Media/lidarr_cleanup.sh +++ b/Media/lidarr_cleanup.sh @@ -2,83 +2,96 @@ # ============================================================================================== # ================================= Lidarr Cleanup ============================================= # ============================================================================================== -# Removes orphaned music files from the library that Lidarr no longer tracks. -# Uses the Lidarr API to build a complete list of tracked file paths then compares -# against what exists on disk — anything untracked and older than LIDARR_ORPHAN_AGE -# days is considered an orphan and deleted. # -# ── FILE CLASSIFICATION ─────────────────────────────────────────────────────────────────────── -# TRACKED — Lidarr API knows about this exact file path → leave it alone -# PROTECTED — matches LIDARR_PROTECTED_PATTERNS → never delete (cover art, .nfo, .lrc etc.) -# ORPHAN — music file, not tracked, older than LIDARR_ORPHAN_AGE days → delete +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Delete orphaned music files not tracked by Lidarr. Queries the API for all +# tracked file paths, walks the library on disk, and removes anything untracked +# that is old enough to be past the import window. Triggers an Emby library +# clean after each deletion run so ghost entries disappear immediately. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Every file encountered on disk is classified into one of five categories: +# +# TRACKED — Lidarr API knows this exact path → leave it alone +# PROTECTED — matches LIDARR_PROTECTED_PATTERNS → never delete +# ORPHAN — music file, not tracked, older than LIDARR_ORPHAN_AGE → delete # JUNK — not a music extension, not protected → delete regardless of age -# RECENT — not tracked, under LIDARR_ORPHAN_AGE days old → skip (may be mid-import) +# RECENT — not tracked, under LIDARR_ORPHAN_AGE → skip (may be mid-import) # -# ── WHY PROTECTED PATTERNS MATTER ───────────────────────────────────────────────────────────── -# Lidarr generates cover art (*.jpg), metadata (*.nfo) and lyrics (*.lrc) but does NOT -# include these in its tracked file API response. Without protection these would be -# classified as orphans and deleted — breaking Lidarr and Emby metadata display. +# Lidarr generates cover art (*.jpg), metadata (*.nfo), and lyrics (*.lrc) but +# does NOT include these in its tracked file API response. Without PROTECTED +# classification these would be deleted — breaking Lidarr and Emby display. # -# ── SAFETY LAYERS — ALL MUST PASS BEFORE ANY FILE IS TOUCHED ───────────────────────────────── -# 1. Container must be running and not starting/unhealthy -# 2. API must be reachable -# 3. API version must match tested major version in master.conf -# 4. Artist count must be > 0 -# 5. Tracked file count must be > 0 -# 6. Tracked count must be >= LIDARR_MIN_TRACKED_PCT % of last known count -# 7. Deletion size must be < LIDARR_MAX_DELETE_GB — or --i-know-what-im-doing required +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── OVERRIDE FLAGS ──────────────────────────────────────────────────────────────────────────── -# --i-know-what-im-doing -# Required when deletion would exceed LIDARR_MAX_DELETE_GB. -# Long and annoying by design — cannot be added accidentally. +# Seven gates — ALL must pass before any file is touched: +# 1. Container running and not starting/unhealthy +# 2. API reachable +# 3. API version matches LIDARR_VERSION_MAJOR in master.conf +# 4. Artist count > 0 +# 5. Tracked file count > 0 +# 6. Tracked count >= LIDARR_MIN_TRACKED_PCT % of last known count +# 7. Deletion size < LIDARR_MAX_DELETE_GB — or --i-know-what-im-doing required # -# --skip-strike-list -# Bypasses the LIDARR_ORPHAN_AGE age check — deletes recent files too. -# -# NUCLEAR MODE — both flags active together: -# Age check bypassed, size threshold bypassed, deletes on first pass. -# Use when Soularr has filled the gaps and you want a clean one-pass wipe. -# ⚠️ Script author takes NO responsibility for data loss with both flags active. -# The user accepts full responsibility — this is 100% intentional by design. -# -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# Runs on any node where Lidarr is configured — skips cleanly if LIDARR_URL/API_KEY not set. -# detect_hosts() aliases LIDARR_URL, LIDARR_API_KEY, LIDARR_MUSIC_ROOT from HOST*_LIDARR_*. -# -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── # acquire_lock "wait" — large scans take time, wait for previous run to finish # jq + curl validation — exits if either tool missing # DOCKER_TIMEOUT — container checks protected against daemon hangs -# 7 safety layers — all must pass before any file is touched # Duplicate detection — temp file of tracked paths, grep before delete # validate_unraid_cmd — notify script validated before use # Silent by default — orphans/junk warn(), clean library logs silently # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# ============================================================================================== +# STATE FILES +# ============================================================================================== +# +# LIDARR_TRACKED_COUNT_FILE — persistent baseline for the tracked % safety check (gate 6) +# Updated after each successful run. Protects against misconfigured root path +# returning an empty API response and deleting the entire library. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# # HOST1_LIDARR_URL / HOST1_LIDARR_API_KEY / HOST1_LIDARR_MUSIC_ROOT # HOST1_LIDARR_PATH_MAP — container path → host path translation # All aliased by detect_hosts() — script uses unprefixed names # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# LIDARR_IMPORT_SCAN_TIMEOUT — seconds to wait for pre-flight import scan (default 600) -# LIDARR_LOCK_WARN_AGE — override default lock warning age (large libraries) -# LIDARR_ORPHAN_AGE — days before untracked file is eligible for deletion -# LIDARR_MAX_DELETE_GB — require --i-know-what-im-doing above this -# LIDARR_MIN_TRACKED_PCT — abort if tracked count drops below this % of last run -# LIDARR_TRACKED_COUNT_FILE — persistent baseline file path -# LIDARR_EXTENSIONS — music file extensions to consider for orphan classification -# LIDARR_PROTECTED_PATTERNS — file patterns that are never deleted -# LIDARR_VERSION_MAJOR — expected Lidarr major version for API safety check -# ARR_CLEANUP_STATS — stats file path (read by coffee report) +# master.conf # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# lidarr_cleanup.sh — normal run -# lidarr_cleanup.sh --dry-run — preview, no deletions -# lidarr_cleanup.sh --log — verbose output -# lidarr_cleanup.sh --status — show config and exit -# lidarr_cleanup.sh --i-know-what-im-doing — bypass size threshold +# LIDARR_ORPHAN_AGE — days before untracked file is eligible for deletion +# LIDARR_MAX_DELETE_GB — require --i-know-what-im-doing above this +# LIDARR_MIN_TRACKED_PCT — abort if tracked count drops below this % of last run +# LIDARR_TRACKED_COUNT_FILE — persistent baseline file path +# LIDARR_EXTENSIONS — music file extensions for orphan classification +# LIDARR_PROTECTED_PATTERNS — file patterns that are never deleted +# LIDARR_VERSION_MAJOR — expected Lidarr major version for API safety check +# LIDARR_IMPORT_SCAN_TIMEOUT — seconds to wait for pre-flight import scan (default 600) +# LIDARR_LOCK_WARN_AGE — override default lock warning age (large libraries) +# ARR_CLEANUP_STATS — stats file path (read by coffee report) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# lidarr_cleanup.sh — normal run +# lidarr_cleanup.sh --dry-run — preview, no deletions +# lidarr_cleanup.sh --log — verbose output +# lidarr_cleanup.sh --status — show config and exit +# lidarr_cleanup.sh --i-know-what-im-doing — bypass size threshold # lidarr_cleanup.sh --i-know-what-im-doing --skip-strike-list — NUCLEAR MODE +# +# NUCLEAR MODE: both flags bypass age check AND size threshold. Use when Soularr +# has filled the gaps and you want a clean one-pass wipe. User accepts full +# responsibility — the flag name is long and annoying by design. +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Media/lidarr_missing_art.sh b/Media/lidarr_missing_art.sh index 3b9cb6f..5b1c57e 100644 --- a/Media/lidarr_missing_art.sh +++ b/Media/lidarr_missing_art.sh @@ -2,60 +2,69 @@ # ============================================================================================== # ================================= Lidarr Missing Art ========================================= # ============================================================================================== -# Fetches missing album and artist artwork for the Lidarr music library. -# Reads from Lidarr API to discover album/artist paths, then downloads only -# what is missing — never overwrites existing files. # -# ── SAFE DESIGN ─────────────────────────────────────────────────────────────────────────────── -# READS from Lidarr only — no writes back to Lidarr -# NEVER modifies audio tags or renames media files -# NEVER overwrites existing artwork -# ONLY writes missing artwork files to existing album/artist directories +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Fetch missing album and artist artwork for the Lidarr music library. Downloads +# only what is absent — never overwrites existing files. Idempotent re-runs are +# safe. # -# ── ARTWORK TARGETS ─────────────────────────────────────────────────────────────────────────── -# Album folder: cover.jpg cdart.png back.jpg -# Artist folder: folder.jpg fanart.jpg logo.png banner.jpg +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── SOURCES ─────────────────────────────────────────────────────────────────────────────────── +# Artwork targets per album folder: cover.jpg cdart.png back.jpg +# Artwork targets per artist folder: folder.jpg fanart.jpg logo.png banner.jpg +# +# Sources (tried in order, first success wins): # Album covers: fanart.tv → iTunes fallback # Artist art: fanart.tv → Deezer fallback → Last.fm fallback # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# Lidarr runs on HOST1 only. detect_hosts() sets LIDARR_URL — if empty (HOST2) the -# script exits cleanly with no action rather than failing. +# Reads from Lidarr API only — no writes back to Lidarr. Never modifies audio +# tags or renames media files. Only writes missing artwork files to existing +# album/artist directories. # -# ── OUTPUT ──────────────────────────────────────────────────────────────────────────────────── -# Minimal by default — section headers + per-section summary always visible. -# --log shows per-item detail (each album, each artist, each file fetched). +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock — prevents concurrent runs during large library scans -# curl + jq check — fail fast if tools missing -# API reachability — verified before processing begins -# Skip existing — never overwrites, idempotent re-runs are safe -# Min file size — rejects corrupt/placeholder downloads (LIDARR_ART_MIN_SIZE) -# Parallel jobs — capped at LIDARR_ART_MAX_PARALLEL to avoid hammering APIs -# Download retries — LIDARR_ART_RETRIES attempts per image before giving up -# Dry-run mode — logs what would be downloaded without writing anything +# acquire_lock — prevents concurrent runs during large library scans +# curl + jq check — fail fast if tools missing +# API reachability — verified before processing begins +# detect_hosts() — exits cleanly if LIDARR_URL empty (HOST2, no Lidarr) +# Skip existing — never overwrites, idempotent re-runs are safe +# Min file size check — rejects corrupt/placeholder downloads (LIDARR_ART_MIN_SIZE) +# Parallel job cap — LIDARR_ART_MAX_PARALLEL — avoids hammering external APIs +# Download retries — LIDARR_ART_RETRIES attempts per image before giving up +# Rate limiting — LIDARR_ART_SLEEP_BETWEEN between fanart.tv API calls # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# FANART_API_KEY — fanart.tv API key -# LASTFM_API_KEY — last.fm API key -# LIDARR_ART_MIN_SIZE — minimum valid download size in bytes -# LIDARR_ART_MAX_PARALLEL — concurrent background download jobs -# LIDARR_ART_RETRIES — download retry attempts per image +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# +# HOST1_LIDARR_URL / HOST1_LIDARR_API_KEY +# Aliased by detect_hosts() — script uses LIDARR_URL / LIDARR_API_KEY +# +# master.conf +# +# FANART_API_KEY — fanart.tv API key +# LASTFM_API_KEY — last.fm API key +# LIDARR_ART_MIN_SIZE — minimum valid download size in bytes +# LIDARR_ART_MAX_PARALLEL — concurrent background download jobs +# LIDARR_ART_RETRIES — download retry attempts per image # LIDARR_ART_SLEEP_BETWEEN — seconds between fanart.tv API calls # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── -# HOST1_LIDARR_URL — Lidarr base URL -# HOST1_LIDARR_API_KEY — Lidarr API key -# All aliased by detect_hosts() — script uses unprefixed LIDARR_URL / LIDARR_API_KEY +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # lidarr_missing_art.sh — fetch all missing artwork # lidarr_missing_art.sh --dry-run — preview without downloading # lidarr_missing_art.sh --log — verbose per-item output # lidarr_missing_art.sh --status — show config and exit +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Media/media_cleaner.sh b/Media/media_cleaner.sh index 9d4cc69..724032b 100644 --- a/Media/media_cleaner.sh +++ b/Media/media_cleaner.sh @@ -2,20 +2,24 @@ # ============================================================================================== # ================================= Media Cleaner ============================================== # ============================================================================================== -# Removes unwanted junk files from media share folders using configurable file patterns. -# Two profiles — anime and media — each with their own folder list and file patterns. -# Runs daily via DAILY_MAINTENANCE_SCRIPTS after media_shares_permissions.sh. # -# ── PROFILES ────────────────────────────────────────────────────────────────────────────────── +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Remove junk files from media shares using configurable file patterns. Two +# profiles — anime and media — each with their own folder list and patterns. +# Runs second in the daily maintenance window, after permissions and before +# arr cleanup, so orphan detection only encounters actual media files. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Profiles: # anime — cleans ANIME_CLEAN_FOLDERS using ANIME_FILE_PATTERNS -# typical targets: *.sfv *.nfo *.url *.rar *.zip *.sample* etc. +# media — cleans MEDIA_CLEAN_FOLDERS using MEDIA_FILE_PATTERNS (adds *.iso *.lrc) # -# media — cleans MEDIA_CLEAN_FOLDERS using MEDIA_FILE_PATTERNS -# same patterns plus *.iso *.lrc (media-specific extras) -# -# ── WHAT IT REMOVES ─────────────────────────────────────────────────────────────────────────── -# Junk files left behind by download clients, scene releases, and various tools: -# *.sfv *.md5 *.sha1 — checksum verification files — useless post-download +# Removes junk left by download clients, scene releases, and tools: +# *.sfv *.md5 *.sha1 — checksum files — useless post-download # *.nfo *.url *.lnk — scene info files — not needed in media library # *.rar *.zip — archives — source files not needed after extraction # *.sample* *.proof* — scene samples — never needed @@ -24,12 +28,10 @@ # *.torrent — torrent files left by download clients # *.log *.json — tool output files # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID and aliases HOST*_ANIME_CLEAN_FOLDERS → ANIME_CLEAN_FOLDERS -# and HOST*_MEDIA_CLEAN_FOLDERS → MEDIA_CLEAN_FOLDERS. -# Each server only cleans the shares it owns — correct folders per host automatically. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── # acquire_lock "wait" — wait if previous run still active # detect_hosts() — correct folder lists per host via MY_ID aliases # Empty array guards — warns and exits cleanly if no folders or patterns configured @@ -37,22 +39,32 @@ # validate_unraid_cmd — notify script validated before use # Silent by default — only problems and removals produce output # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# # HOST*_ANIME_CLEAN_FOLDERS — folders cleaned by the anime profile on this host # HOST*_MEDIA_CLEAN_FOLDERS — folders cleaned by the media profile on this host # Aliased by detect_hosts() — script uses ANIME_CLEAN_FOLDERS / MEDIA_CLEAN_FOLDERS # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# master.conf +# # ANIME_FILE_PATTERNS — file patterns removed by the anime profile # MEDIA_FILE_PATTERNS — file patterns removed by the media profile # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# # media_cleaner.sh anime — clean anime shares # media_cleaner.sh media — clean media shares # media_cleaner.sh anime --dry-run — preview anime clean, no deletions # media_cleaner.sh media --dry-run — preview media clean, no deletions # media_cleaner.sh anime --log — verbose output # media_cleaner.sh anime --status — show config and exit +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Media/media_shares_permissions.sh b/Media/media_shares_permissions.sh index ab826d2..0fa8fd9 100644 --- a/Media/media_shares_permissions.sh +++ b/Media/media_shares_permissions.sh @@ -2,45 +2,22 @@ # ============================================================================================== # ============================= Media Shares Permissions ======================================= # ============================================================================================== -# Applies correct ownership and permissions to all configured media shares. -# Runs daily via DAILY_MAINTENANCE_SCRIPTS — first job before arr cleanup scripts. -# Arr cleanup depends on correct ownership to rename and delete files safely. # -# ── WHY THIS EXISTS ─────────────────────────────────────────────────────────────────────────── -# Originally a band-aid for 777 permissions caused by containers running as root. -# Now a proper daily failsafe — even with correct container config, files can arrive -# with wrong ownership from: -# - rsync without --chown (brings source server's ownership) -# - Manual admin copies (creates root:root files) -# - New containers not yet configured with correct PUID/PGID -# - unRAID updates that reset container environments +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Apply nobody:users ownership and correct permissions to all media shares. +# Runs daily as the first job in the maintenance window — arr cleanup depends +# on correct ownership to rename and delete files. # -# ── PERMISSIONS MODEL ───────────────────────────────────────────────────────────────────────── -# Directories: 755 nobody:users -# Owner (nobody) — rwx enter, list, create files ✅ -# Group (users) — r-x enter and list ✅ -# Others — r-x Samba guests can browse ✅ -# No world-write — prevents accidental deletion by unauthenticated access +# Now a proper daily failsafe: files arrive with wrong ownership from rsync +# without --chown, manual admin copies, containers with unconfigured PUID/PGID, +# or unRAID environment resets after updates. # -# Files: 664 nobody:users -# Owner (nobody) — rw read + write ✅ -# Group (users) — rw arrs can import/rename ✅ -# Others — r Samba guests can read ✅ -# No execute bit — media files are never executable ✅ +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── DIAGNOSTIC — HIGH CORRECTED COUNT ───────────────────────────────────────────────────────── -# If this script corrects many files every run, a container has wrong PUID/PGID: -# Correct values on unRAID: PUID=99 (nobody) PGID=100 (users) -# Add to each container's environment in its Docker template -# Common culprits: SABnzbd, qBittorrent, slskd — check these first -# Once fixed, this script should correct 0 files per run (pure failsafe) -# -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID and aliases HOST*_MEDIA_PERMISSION_SHARES → MEDIA_PERMISSION_SHARES -# Each server only applies permissions to the shares it owns. -# -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock "wait" — wait if previous run still active (large share scans take time) +# acquire_lock "wait" — wait if previous run still active (large share scans) # detect_hosts() — correct share list per host via MY_ID aliases # Empty array guard — warns and exits cleanly if no shares configured # Folder existence — skips missing shares with warning, continues others @@ -48,20 +25,35 @@ # validate_unraid_cmd — notify script validated before use # Silent by default — only failures produce output, success is silent # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# Diagnostic — high corrected count on every run means a container has wrong PUID/PGID: +# Correct values on unRAID: PUID=99 (nobody) PGID=100 (users) +# Common culprits: SABnzbd, qBittorrent, slskd — check these first +# Once fixed, this script should correct 0 files per run (pure failsafe) +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# # HOST*_MEDIA_PERMISSION_SHARES — shares this host applies permissions to # Aliased by detect_hosts() — script uses MEDIA_PERMISSION_SHARES # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# master.conf +# # PERMISSIONS_DIR_MODE — directory permissions (default 755) # PERMISSIONS_FILE_MODE — file permissions (default 664) # PERMISSIONS_OWNER — ownership applied to all files (default nobody:users) # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# # media_shares_permissions.sh — normal run # media_shares_permissions.sh --dry-run — preview without making changes # media_shares_permissions.sh --log — verbose output # media_shares_permissions.sh --status — show config and exit +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Media/playback_aware_lidarr_discovery.sh b/Media/playback_aware_lidarr_discovery.sh index 655951e..86ad335 100644 --- a/Media/playback_aware_lidarr_discovery.sh +++ b/Media/playback_aware_lidarr_discovery.sh @@ -1,24 +1,17 @@ #!/bin/bash # ============================================================================================== -# =============================== LIDARR DISCOVERY ============================================= +# =========================== Playback-Aware Lidarr Discovery ================================== # ============================================================================================== -# Lidarr discovery orchestrator. # -# Consumes: -# Kernel/decision_engine.sh +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Behavior-driven music discovery scoring prototype. Consumes +# Kernel/decision_engine.sh to score artist/album candidates for acquisition +# using family-aware and playback-weighted logic. # -# Purpose: -# Discover high-quality artists/albums for acquisition using -# family-aware and behavior-driven scoring logic. -# -# This script is intentionally selective. -# -# Music discovery is treated differently than TV/movies: -# -# Higher strictness -# Stronger quality bias -# Lower tolerance for trend chasing -# Longer behavioral memory +# WIP — not yet connected to a live data source or scheduled. Discovery logic +# is intentionally selective: higher strictness, stronger quality bias, lower +# tolerance for trend-chasing, longer behavioral memory than TV/movies. # # ============================================================================================== diff --git a/Media/radarr_cleanup.sh b/Media/radarr_cleanup.sh index 2ab1391..8e252e7 100644 --- a/Media/radarr_cleanup.sh +++ b/Media/radarr_cleanup.sh @@ -2,77 +2,87 @@ # ============================================================================================== # ================================= Radarr Cleanup ============================================= # ============================================================================================== -# Removes orphaned movie files from the library that Radarr no longer tracks. -# Uses the Radarr API to build a complete list of tracked movie file paths then compares -# against what exists on disk — anything untracked and older than RADARR_ORPHAN_AGE -# days is considered an orphan and deleted. # -# ── FILE CLASSIFICATION ─────────────────────────────────────────────────────────────────────── -# TRACKED — Radarr API knows about this exact file path → leave it alone -# PROTECTED — matches RADARR_PROTECTED_PATTERNS → never delete (artwork, subtitles, .nfo) -# ORPHAN — video file, not tracked, older than RADARR_ORPHAN_AGE days → delete +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Delete orphaned movie files not tracked by Radarr. Queries the API for all +# tracked movie file paths, walks the library on disk, and removes anything +# untracked that is old enough to be past the import window. Triggers an Emby +# library clean after each deletion run so ghost entries disappear immediately. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Every file encountered on disk is classified into one of five categories: +# +# TRACKED — Radarr API knows this exact path → leave it alone +# PROTECTED — matches RADARR_PROTECTED_PATTERNS → never delete +# ORPHAN — video file, not tracked, older than RADARR_ORPHAN_AGE → delete # JUNK — not a video extension, not protected → delete regardless of age -# RECENT — not tracked, under RADARR_ORPHAN_AGE days old → skip (may be mid-import) +# RECENT — not tracked, under RADARR_ORPHAN_AGE → skip (may be mid-import) # -# ── WHY PROTECTED PATTERNS MATTER ───────────────────────────────────────────────────────────── -# Radarr generates movie artwork (*.jpg), metadata (*.nfo) and manages subtitles (*.srt, -# *.sub, *.ass) but does NOT include these in its tracked file API response. -# Without protection these would be classified as orphans and deleted — breaking -# Radarr and Emby metadata display. +# Radarr generates movie artwork (*.jpg), metadata (*.nfo), and manages subtitles +# (*.srt, *.sub, *.ass) but does NOT include these in its tracked file API response. +# Without PROTECTED classification these would be deleted — breaking Radarr and +# Emby metadata display. # -# ── SAFETY LAYERS — ALL MUST PASS BEFORE ANY FILE IS TOUCHED ───────────────────────────────── -# 1. Container must be running and not starting/unhealthy -# 2. API must be reachable -# 3. API version must match tested major version in master.conf -# 4. Movie count must be > 0 -# 5. Tracked file count must be > 0 -# 6. Deletion size must be < RADARR_MAX_DELETE_GB — or --i-know-what-im-doing required +# After deletions: notify_emby_scan() triggers Emby "Clean Missing Files" task. +# Emby removes ghost entries immediately — no user-facing file-not-found errors. # -# ── POST-DELETION ───────────────────────────────────────────────────────────────────────────── -# After files are deleted notify_emby_scan() triggers Emby "Clean Missing Files" task. -# Emby immediately removes ghost entries — no user-facing file-not-found errors. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── OVERRIDE FLAGS ──────────────────────────────────────────────────────────────────────────── -# --i-know-what-im-doing required when deletion exceeds RADARR_MAX_DELETE_GB -# --skip-strike-list bypasses RADARR_ORPHAN_AGE age check -# NUCLEAR MODE — both active: age + size bypass, deletes on first pass -# ⚠️ User accepts full responsibility — no recovery possible after deletion +# Six gates — ALL must pass before any file is touched: +# 1. Container running and not starting/unhealthy +# 2. API reachable +# 3. API version matches RADARR_VERSION_MAJOR in master.conf +# 4. Movie count > 0 +# 5. Tracked file count > 0 +# 6. Deletion size < RADARR_MAX_DELETE_GB — or --i-know-what-im-doing required # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# HOST1 Radarr manages Movies. HOST2 Radarr manages Anime_Movies. -# detect_hosts() sets MY_ID and aliases RADARR_URL, RADARR_API_KEY, RADARR_MOVIES_ROOT. -# No manual HOST1/HOST2 comparisons — MY_ID routes correctly on any server. -# -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── # acquire_lock "wait" — large scans take time, wait for previous run to finish # jq + curl validation — exits if either tool missing # DOCKER_TIMEOUT — container checks protected against daemon hangs -# 6 safety layers — all must pass before any file is touched # notify_emby_scan() — triggers Emby clean after deletion # validate_unraid_cmd — notify script validated before use # Silent by default — orphans/junk warn(), clean library logs silently # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# # HOST*_RADARR_URL / HOST*_RADARR_API_KEY / HOST*_RADARR_MOVIES_ROOT # HOST*_RADARR_PATH_MAP — container path → host path translation # All aliased by detect_hosts() — script uses unprefixed names # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# RADARR_ORPHAN_AGE — days before untracked file eligible for deletion -# RADARR_MAX_DELETE_GB — require --i-know-what-im-doing above this -# RADARR_EXTENSIONS — video file extensions considered for orphan classification -# RADARR_PROTECTED_PATTERNS — file patterns never deleted -# RADARR_VERSION_MAJOR — expected Radarr major version for API safety check -# RADARR_IMPORT_SCAN_TIMEOUT — seconds to wait for pre-flight import scan (default 600) -# ARR_CLEANUP_STATS — stats file path (read by coffee report) +# master.conf # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# radarr_cleanup.sh — normal run -# radarr_cleanup.sh --dry-run — preview, no deletions -# radarr_cleanup.sh --log — verbose output -# radarr_cleanup.sh --status — show config and exit -# radarr_cleanup.sh --i-know-what-im-doing — bypass size threshold +# RADARR_ORPHAN_AGE — days before untracked file eligible for deletion +# RADARR_MAX_DELETE_GB — require --i-know-what-im-doing above this +# RADARR_EXTENSIONS — video file extensions for orphan classification +# RADARR_PROTECTED_PATTERNS — file patterns never deleted +# RADARR_VERSION_MAJOR — expected Radarr major version for API safety check +# RADARR_IMPORT_SCAN_TIMEOUT — seconds to wait for pre-flight import scan (default 600) +# ARR_CLEANUP_STATS — stats file path (read by coffee report) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# radarr_cleanup.sh — normal run +# radarr_cleanup.sh --dry-run — preview, no deletions +# radarr_cleanup.sh --log — verbose output +# radarr_cleanup.sh --status — show config and exit +# radarr_cleanup.sh --i-know-what-im-doing — bypass size threshold # radarr_cleanup.sh --i-know-what-im-doing --skip-strike-list — NUCLEAR MODE +# +# NUCLEAR MODE: both flags bypass age check AND size threshold. User accepts full +# responsibility — the flag name is long and annoying by design. +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Media/radarr_tmdb_removed.sh b/Media/radarr_tmdb_removed.sh index c3312b1..0112d2c 100644 --- a/Media/radarr_tmdb_removed.sh +++ b/Media/radarr_tmdb_removed.sh @@ -2,36 +2,58 @@ # ============================================================================================== # ============================= Radarr — TMDb Removed ========================================== # ============================================================================================== -# Removes movies from Radarr that have been dropped from TMDb. -# Radarr marks these with status="deleted" — they generate system health errors and -# can never be monitored or downloaded. 99% are future/announced movies that were -# delisted before release. # -# ── WHAT IT DOES ────────────────────────────────────────────────────────────────────────────── +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Remove movies from Radarr that TMDb has dropped. Radarr marks these with +# status="deleted" — they generate health errors and can never be monitored +# or downloaded. Most are announced-but-never-released films delisted before +# release. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# # Queries Radarr API for movies with status="deleted" (TMDb removal marker) -# Reports each found entry with file status and size +# Reports each entry with file status and size # Removes the movie record from Radarr # Optionally deletes associated files (disabled by default — most have none) -# Optionally adds to Radarr's import exclusion list (default: true) +# Optionally adds to Radarr's import exclusion list (default: true — prevents re-add) # -# ── SAFE DEFAULTS ───────────────────────────────────────────────────────────────────────────── -# Files are NOT deleted by default — use --delete-files to also remove from disk -# Import exclusion added by default — prevents Radarr re-adding dropped movies -# Per-deletion output always visible — deletions are never silently swallowed +# Files are NOT deleted by default — use --delete-files to also remove from disk. +# Per-deletion output is always visible — deletions are never silently swallowed. # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# Radarr runs on HOST1 only. detect_hosts() sets RADARR_URL — if empty (HOST2) -# the script exits cleanly. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# RADARR_DROPPED_ADD_EXCLUSION — add removed movies to import exclusion (default: true) +# detect_hosts() — exits cleanly if RADARR_URL empty (Radarr not on this host) +# API pre-flight — verifies Radarr reachable before querying +# Dry-run mode — full preview without removing anything +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# RADARR_DROPPED_ADD_EXCLUSION — add removed movies to import exclusion (default: true) +# +# master_host*.conf +# +# HOST1_RADARR_URL / HOST1_RADARR_API_KEY +# Aliased by detect_hosts() — script uses RADARR_URL / RADARR_API_KEY +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # radarr_tmdb_removed.sh — remove records, keep files, add exclusion # radarr_tmdb_removed.sh --delete-files — also delete files from disk # radarr_tmdb_removed.sh --dry-run — preview without removing anything # radarr_tmdb_removed.sh --log — verbose output # radarr_tmdb_removed.sh --status — show config and exit +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Media/sonarr_cleanup.sh b/Media/sonarr_cleanup.sh index 70f516c..bec4296 100644 --- a/Media/sonarr_cleanup.sh +++ b/Media/sonarr_cleanup.sh @@ -2,77 +2,87 @@ # ============================================================================================== # ================================= Sonarr Cleanup ============================================= # ============================================================================================== -# Removes orphaned TV episode files from the library that Sonarr no longer tracks. -# Uses the Sonarr API to build a complete list of tracked episode file paths then compares -# against what exists on disk — anything untracked and older than SONARR_ORPHAN_AGE -# days is considered an orphan and deleted. # -# ── FILE CLASSIFICATION ─────────────────────────────────────────────────────────────────────── -# TRACKED — Sonarr API knows about this exact file path → leave it alone -# PROTECTED — matches SONARR_PROTECTED_PATTERNS → never delete (artwork, subtitles, .nfo) -# ORPHAN — video file, not tracked, older than SONARR_ORPHAN_AGE days → delete +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Delete orphaned TV episode files not tracked by Sonarr. Queries the API for +# all tracked episode file paths, walks the library on disk, and removes anything +# untracked that is old enough to be past the import window. Triggers an Emby +# library clean after each deletion run so ghost entries disappear immediately. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Every file encountered on disk is classified into one of five categories: +# +# TRACKED — Sonarr API knows this exact path → leave it alone +# PROTECTED — matches SONARR_PROTECTED_PATTERNS → never delete +# ORPHAN — video file, not tracked, older than SONARR_ORPHAN_AGE → delete # JUNK — not a video extension, not protected → delete regardless of age -# RECENT — not tracked, under SONARR_ORPHAN_AGE days old → skip (may be mid-import) +# RECENT — not tracked, under SONARR_ORPHAN_AGE → skip (may be mid-import) # -# ── WHY PROTECTED PATTERNS MATTER ───────────────────────────────────────────────────────────── -# Sonarr generates show artwork (*.jpg), metadata (*.nfo) and manages subtitles (*.srt, -# *.sub, *.ass) but does NOT include these in its tracked file API response. -# Without protection these would be classified as orphans and deleted — breaking -# Sonarr and Emby metadata display. +# Sonarr generates show artwork (*.jpg), metadata (*.nfo), and manages subtitles +# (*.srt, *.sub, *.ass) but does NOT include these in its tracked file API response. +# Without PROTECTED classification these would be deleted — breaking Sonarr and +# Emby metadata display. # -# ── SAFETY LAYERS — ALL MUST PASS BEFORE ANY FILE IS TOUCHED ───────────────────────────────── -# 1. Container must be running and not starting/unhealthy -# 2. API must be reachable -# 3. API version must match tested major version in master.conf -# 4. Series count must be > 0 -# 5. Tracked file count must be > 0 -# 6. Deletion size must be < SONARR_MAX_DELETE_GB — or --i-know-what-im-doing required +# After deletions: notify_emby_scan() triggers Emby "Clean Missing Files" task. +# Emby removes ghost entries immediately — no user-facing file-not-found errors. # -# ── POST-DELETION ───────────────────────────────────────────────────────────────────────────── -# After files are deleted notify_emby_scan() triggers Emby "Clean Missing Files" task. -# Emby immediately removes ghost entries — no user-facing file-not-found errors. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── OVERRIDE FLAGS ──────────────────────────────────────────────────────────────────────────── -# --i-know-what-im-doing required when deletion exceeds SONARR_MAX_DELETE_GB -# --skip-strike-list bypasses SONARR_ORPHAN_AGE age check -# NUCLEAR MODE — both active: age + size bypass, deletes on first pass -# ⚠️ User accepts full responsibility — no recovery possible after deletion +# Six gates — ALL must pass before any file is touched: +# 1. Container running and not starting/unhealthy +# 2. API reachable +# 3. API version matches SONARR_VERSION_MAJOR in master.conf +# 4. Series count > 0 +# 5. Tracked file count > 0 +# 6. Deletion size < SONARR_MAX_DELETE_GB — or --i-know-what-im-doing required # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# HOST1 Sonarr manages Tv_Shows. HOST2 Sonarr manages Anime_Shows. -# detect_hosts() sets MY_ID and aliases SONARR_URL, SONARR_API_KEY, SONARR_TV_ROOT. -# No manual HOST1/HOST2 comparisons — MY_ID routes correctly on any server. -# -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── # acquire_lock "wait" — large scans take time, wait for previous run to finish # jq + curl validation — exits if either tool missing # DOCKER_TIMEOUT — container checks protected against daemon hangs -# 6 safety layers — all must pass before any file is touched # notify_emby_scan() — triggers Emby clean after deletion # validate_unraid_cmd — notify script validated before use # Silent by default — orphans/junk warn(), clean library logs silently # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# # HOST*_SONARR_URL / HOST*_SONARR_API_KEY / HOST*_SONARR_TV_ROOT # HOST*_SONARR_PATH_MAP — container path → host path translation # All aliased by detect_hosts() — script uses unprefixed names # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# SONARR_ORPHAN_AGE — days before untracked file eligible for deletion -# SONARR_MAX_DELETE_GB — require --i-know-what-im-doing above this -# SONARR_EXTENSIONS — video file extensions considered for orphan classification -# SONARR_PROTECTED_PATTERNS — file patterns never deleted -# SONARR_VERSION_MAJOR — expected Sonarr major version for API safety check -# SONARR_IMPORT_SCAN_TIMEOUT — seconds to wait for pre-flight import scan (default 600) -# ARR_CLEANUP_STATS — stats file path (read by coffee report) +# master.conf # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# sonarr_cleanup.sh — normal run -# sonarr_cleanup.sh --dry-run — preview, no deletions -# sonarr_cleanup.sh --log — verbose output -# sonarr_cleanup.sh --status — show config and exit -# sonarr_cleanup.sh --i-know-what-im-doing — bypass size threshold +# SONARR_ORPHAN_AGE — days before untracked file eligible for deletion +# SONARR_MAX_DELETE_GB — require --i-know-what-im-doing above this +# SONARR_EXTENSIONS — video file extensions for orphan classification +# SONARR_PROTECTED_PATTERNS — file patterns never deleted +# SONARR_VERSION_MAJOR — expected Sonarr major version for API safety check +# SONARR_IMPORT_SCAN_TIMEOUT — seconds to wait for pre-flight import scan (default 600) +# ARR_CLEANUP_STATS — stats file path (read by coffee report) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# sonarr_cleanup.sh — normal run +# sonarr_cleanup.sh --dry-run — preview, no deletions +# sonarr_cleanup.sh --log — verbose output +# sonarr_cleanup.sh --status — show config and exit +# sonarr_cleanup.sh --i-know-what-im-doing — bypass size threshold # sonarr_cleanup.sh --i-know-what-im-doing --skip-strike-list — NUCLEAR MODE +# +# NUCLEAR MODE: both flags bypass age check AND size threshold. User accepts full +# responsibility — the flag name is long and annoying by design. +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Media/sonarr_tvdb_removed.sh b/Media/sonarr_tvdb_removed.sh index b369fbd..ea22c7d 100644 --- a/Media/sonarr_tvdb_removed.sh +++ b/Media/sonarr_tvdb_removed.sh @@ -2,35 +2,57 @@ # ============================================================================================== # ============================= Sonarr — TVDB Removed ========================================== # ============================================================================================== -# Removes series from Sonarr that have been dropped from TVDB. -# Sonarr marks these with status="deleted" — they generate system health errors and -# can never be monitored or downloaded. # -# ── WHAT IT DOES ────────────────────────────────────────────────────────────────────────────── +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Remove series from Sonarr that TVDB has dropped. Sonarr marks these with +# status="deleted" — they generate health errors and can never be monitored +# or downloaded. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# # Queries Sonarr API for series with status="deleted" (TVDB removal marker) -# Reports each found entry with file count and total size +# Reports each entry with file count and total size # Removes the series record from Sonarr # Optionally deletes associated files (disabled by default) -# Optionally adds to Sonarr's import exclusion list (default: true) +# Optionally adds to Sonarr's import exclusion list (default: true — prevents re-add) # -# ── SAFE DEFAULTS ───────────────────────────────────────────────────────────────────────────── -# Files are NOT deleted by default — use --delete-files to also remove from disk -# Import exclusion added by default — prevents Sonarr re-adding dropped series -# Per-deletion output always visible — deletions are never silently swallowed +# Files are NOT deleted by default — use --delete-files to also remove from disk. +# Per-deletion output is always visible — deletions are never silently swallowed. # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# Sonarr runs on HOST1 only. detect_hosts() sets SONARR_URL — if empty (HOST2) -# the script exits cleanly. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# SONARR_DROPPED_ADD_EXCLUSION — add removed series to import exclusion (default: true) +# detect_hosts() — exits cleanly if SONARR_URL empty (Sonarr not on this host) +# API pre-flight — verifies Sonarr reachable before querying +# Dry-run mode — full preview without removing anything +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# SONARR_DROPPED_ADD_EXCLUSION — add removed series to import exclusion (default: true) +# +# master_host*.conf +# +# HOST1_SONARR_URL / HOST1_SONARR_API_KEY +# Aliased by detect_hosts() — script uses SONARR_URL / SONARR_API_KEY +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # sonarr_tvdb_removed.sh — remove records, keep files, add exclusion # sonarr_tvdb_removed.sh --delete-files — also delete files from disk # sonarr_tvdb_removed.sh --dry-run — preview without removing anything # sonarr_tvdb_removed.sh --log — verbose output # sonarr_tvdb_removed.sh --status — show config and exit +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Monitors/Manual-Monitors.md b/Monitors/Manual-Monitors.md new file mode 100644 index 0000000..803ff9f --- /dev/null +++ b/Monitors/Manual-Monitors.md @@ -0,0 +1,532 @@ +# ━━━━━ MONITORS — Manual ━━━━━ + +Config reference, procedures, operational workflows. +For overview see README-Monitors.md. For per-script detail see script headers. + +--- + +## ━━━ CERT MONITOR ━━━ + +### Domain Configuration + +Each domain and subdomain has an independent TLS certificate and must be listed +separately. `cert_monitor.sh` makes one openssl connection per entry. + +```bash +# master_host1.conf +HOST1_CERT_MONITOR_DOMAINS=( + "Gmer4Lfe.com" + "Gmer4Lfe.us" + # "auth.Gmer4Lfe.com" # add subdomains separately + # "cloud.Gmer4Lfe.com" +) + +# master_host2.conf +HOST2_CERT_MONITOR_DOMAINS=( + "Jayred365.com" + # "auth.Jayred365.com" +) +``` + +Each server monitors its own domains. `detect_hosts()` aliases the correct list +to `CERT_MONITOR_DOMAINS` based on `MY_ID`. + +### Threshold Configuration + +```bash +# master.conf +CERT_WARN_DAYS=30 # warn this many days before expiry — time to investigate +CERT_CRIT_DAYS=7 # critical alert — action needed now +CERT_TIMEOUT=10 # seconds per domain before declaring FAILED +``` + +**Sizing guidance:** +- `CERT_WARN_DAYS=30` gives a month to investigate and renew. If Let's Encrypt + auto-renewal is working, you'll only see warnings when renewal breaks. +- `CERT_CRIT_DAYS=7` — at 7 days remaining, manual action is needed today. +- `CERT_TIMEOUT=10` — sufficient for typical public domains; increase for slow + DNS resolution or high-latency connections. + +### Notification Behavior + +| State | Condition | Action | +|-------|-----------|--------| +| HEALTHY | > CERT_WARN_DAYS remaining | Silent — no notification | +| WARNING | ≤ CERT_WARN_DAYS remaining | One notification listing all WARNING domains | +| CRITICAL | ≤ CERT_CRIT_DAYS remaining | Separate notification listing all CRITICAL domains | +| FAILED | Could not connect or parse cert | Notification — treat as critical | + +--- + +## ━━━ SMART HEALTH ━━━ + +### Drive Ignore List + +```bash +# master_host1.conf +HOST1_SMART_IGNORE_DRIVES=( + "sda" # boot USB flash drive — no meaningful SMART data +) + +# master_host2.conf +HOST2_SMART_IGNORE_DRIVES=( + "sda" # boot USB flash drive +) +``` + +unRAID typically boots from a USB flash drive that appears as `sda`. Flash drives +either don't support SMART or report values that are meaningless for health assessment. +Add any drive that produces spurious SMART data here. + +### Temperature Configuration + +```bash +# master.conf — fallback values if dynamix.cfg is not found +SMART_TEMP_WARN=45 # °C +SMART_TEMP_CRIT=55 # °C +``` + +`smart_health.sh` reads temperature thresholds directly from +`/boot/config/plugins/dynamix/dynamix.cfg` at runtime, using the same hot/max/hotssd/maxssd +values as unRAID's dashboard. The `master.conf` values are only used as a fallback +if `dynamix.cfg` is not found (e.g., running outside of unRAID). + +### Attribute Reference + +| Attribute | Threshold | Meaning | +|-----------|-----------|---------| +| `Reallocated_Sector_Ct` | Any > 0 is warning | Drive found bad sectors and swapped in spares. 0 = healthy. Count growing = degrading. | +| `Current_Pending_Sector` | Any > 0 is warning | Sectors suspected bad, not yet confirmed. May recover on next read, may escalate. Watch it. | +| `Offline_Uncorrectable` | Any > 0 is critical | Could not correct during offline tests. No spares. Data loss risk. | +| Overall SMART status | FAILED = immediate alert | Drive's own self-assessment. FAILED = get data off now. | +| `Temperature_Celsius` | vs dynamix.cfg thresholds | Sustained high temp shortens drive life significantly. | +| `Power_On_Hours` | Informational | Drive age in days — useful context for other attributes. | + +--- + +## ━━━ BACKUP VERIFY ━━━ + +### Share Configuration + +```bash +# master_host1.conf +HOST1_BACKUP_VERIFY_SHARES=( + # empty — uses HOST1_DAILY_SYNC_SHARES automatically + # "/mnt/user/Movies" # override to check specific shares only +) +``` + +Leave `HOST*_BACKUP_VERIFY_SHARES` empty and `backup_verify.sh` will automatically +verify the same shares configured for daily rsync. Only populate it if you want to +verify a different set. + +### Sample Configuration + +```bash +# master.conf +BACKUP_VERIFY_SAMPLE=10 # random files sampled per share per run +BACKUP_VERIFY_MIN_SIZE="1M" # skip files smaller than this +``` + +**Sizing guidance:** +- `BACKUP_VERIFY_SAMPLE=10` is a spot check — catches systematic hardware problems + while running in minutes. Increase to 25-50 for deeper confidence on a large library; + decrease to 5 if the run is too slow against many shares. +- `BACKUP_VERIFY_MIN_SIZE="1M"` — tiny files (NFO, thumbnails, subtitles) have negligible + corruption risk and each one adds an SSH round-trip. Skip them. + +### Result States + +| State | Meaning | Action | +|-------|---------|--------| +| MATCH | Checksums identical on both servers | Correctly mirrored — no action | +| MISMATCH | File exists on both, checksums differ | Sync failure or corruption — investigate | +| MISSING | File exists locally, not on remote | Not yet synced or deleted on remote — may be normal | + +--- + +## ━━━ BANDWIDTH MONITOR ━━━ + +### Log Configuration + +```bash +# master.conf +BANDWIDTH_LOG="/boot/config/bandwidth_history.db" # survives reboots +BANDWIDTH_LOG_RETENTION=90 # days — file stays bounded, never grows unbounded +BANDWIDTH_WARN_GB=50 # flag transfers or daily totals exceeding this +``` + +**Why `/boot/config/`**: The log needs to survive reboots to build a useful history. +`/boot/config/` is on the USB flash drive, which survives reboots and is backed up +by unRAID's flash backup. The log is bounded by `BANDWIDTH_LOG_RETENTION` so it never +grows unbounded. + +**`BANDWIDTH_WARN_GB`**: Set to a value that represents "unexpectedly large" for your +setup. If your typical daily sync transfers 5-10GB, 50GB would flag a full-library resync. +Adjust based on your share sizes. + +### Log Format + +One line per transfer — do not edit manually: +``` +YYYY-MM-DD|HH:MM|profile|duration_seconds|status|bytes_transferred + +# Example entries: +2026-04-14|01:23|critical-data|143|success|2847362048 +2026-04-14|01:31|arrs_stack|287|success|891234567 +2026-04-14|02:15|movies|1847|failed|0 +``` + +### How rsync.sh Calls bandwidth_monitor.sh + +`rsync.sh` automatically calls `bandwidth_monitor.sh --log-transfer` after each sync +profile completes. You never need to call `--log-transfer` manually. The call: + +```bash +bandwidth_monitor.sh --log-transfer "movies" 1847 "failed" 0 +# profile secs status bytes +``` + +--- + +## ━━━ WEEKLY HEALTH DIGEST ━━━ + +### Profile Configuration + +```bash +# master.conf +DIGEST_PROFILE="weekly" # always | smart | weekly +DIGEST_DAY="Sunday" # for weekly profile — must match `date +%A` output +``` + +The cron schedule is always `0 8 * * *` (8am daily). Change only `DIGEST_PROFILE` +to switch behavior — no cron edit needed. + +**Profile guide:** + +| Profile | When to use | Result | +|---------|-------------|--------| +| `always` | You want a daily check-in regardless of system state | Notification every morning | +| `smart` | Quiet operation, only alert on real issues | Silent when healthy, notification when something needs attention | +| `weekly` | One weekly summary is enough | Single notification on `DIGEST_DAY`, silent all other days | + +### Smart Profile Triggers + +```bash +# master.conf — each independently toggleable +DIGEST_SMART_ON_WATCHDOG=true # send if any active watchdog strikes +DIGEST_SMART_ON_FALLBACK=true # send if fallback state is not NORMAL +DIGEST_SMART_ON_CERT_WARN=true # send if any cert within CERT_WARN_DAYS +DIGEST_SMART_ON_BANDWIDTH=true # send if any transfer exceeded BANDWIDTH_WARN_GB +``` + +Turn any trigger off by setting it to `false`. The digest still runs and collects +data — it just doesn't trigger a notification for that condition. + +### Data Sources + +`weekly_health_digest.sh` reads (but never writes): + +| State File | Source | What It Shows | +|-----------|--------|---------------| +| `FALLBACK_STATE_FILE` | `Fallback/fallback.sh` | Current fallback state (NORMAL/FALLBACK/etc.) | +| `SYS_WATCHDOG_FAILED_FILE` | `Docker_Essentials/docker_watchdog.sh` | Container skip list — needs human attention | +| `WATCHDOG_STATE_FILE` | `Docker_Essentials/docker_watchdog.sh` | Active container strike counts | +| `SYS_WATCHDOG_STATE_FILE` | `unRAID_Essentials/system_watchdog.sh` | Active system watchdog strikes | +| `BANDWIDTH_LOG` | `bandwidth_monitor.sh` | Yesterday's transfer history | +| `TRANSCODE_DAILY_LOG` | `Transcodes/` | Weekly transcode statistics | +| `CERT_MONITOR_DOMAINS` | live openssl check | Current cert status per domain | + +--- + +## ━━━ EMBY SESSION REPORT ━━━ + +### Emby Credentials + +```bash +# master_host1.conf +HOST1_EMBY_URL="http://192.168.50.2:8096" +HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829" + +# master_host2.conf +HOST2_EMBY_URL="http://192.168.50.3:8096" +HOST2_EMBY_API_KEY="" +``` + +To generate an API key: Emby UI → Settings → API Keys → New API Key. +Give it a descriptive name (e.g., `unraid_scripts`). The key is only shown once. + +### Report Configuration + +```bash +# master.conf +EMBY_REPORT_DAYS=7 # report period in days +EMBY_REPORT_TOP_N=10 # top N content items to show +``` + +--- + +## ━━━ SYSTEM TUNING MONITOR ━━━ + +### Threshold Configuration + +```bash +# master.conf +INOTIFY_WARN_PCT=80 # warn if inotify instances above 80% of kernel limit +PHP_FPM_WARN_PCT=80 # warn if php-fpm workers above 80% of max +TUNING_MONITOR_LOG="$DATA_DIR/tuning_monitor.db" +TUNING_LOG_RETENTION=30 # days — log stays bounded +``` + +**`INOTIFY_WARN_PCT`**: 80% leaves headroom before kernel-level exhaustion. If you +regularly see warnings at 80% and your system is stable, increase to 90%. Don't set +higher than 90% — at 100% utilisation new inotify watches silently fail. + +**`PHP_FPM_WARN_PCT`**: 80% means the WebGUI is using most of its workers. At 100% +new requests queue (WebGUI feels sluggish) or time out. + +**`PHP_MAX_CHILDREN`**: Set by `php_fpm_max_children.sh` in `unRAID_Essentials/` — do +not set manually here. + +### Reading the Weekly Digest Data + +`system_tuning_monitor.sh` feeds data into `weekly_health_digest.sh`. The digest +shows for the week: +- inotify: peak utilisation, average utilisation, number of snapshots that hit the warning threshold +- php-fpm: same + +A few warning snapshots per week is normal. A rising peak or many warnings per week +means the limits should be adjusted — use `inotify_tuning.sh` or `php_fpm_max_children.sh` +in `unRAID_Essentials/`. + +--- + +## ━━━ ZFS MEMORY SNAPSHOT ━━━ + +### Pool Ignore List + +```bash +# master_host1.conf +HOST1_ZFS_REPORT_IGNORE_POOLS=( + "disk10" # JBOD member — high usage expected, exclude from report noise + "disk9" + "disk8" + "disk6" + "disk5" +) + +# master_host2.conf +HOST2_ZFS_REPORT_IGNORE_POOLS=( + # list host2's JBOD members here +) +``` + +In a JBOD ZFS configuration each disk appears as its own pool. These single-disk +pools normally show high capacity utilisation by design — including them would produce +constant warnings. The ignore list removes them from the report while unRAID's built-in +monitoring continues to watch them. + +### Threshold Configuration + +```bash +# master.conf +ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log" # tmpfs — resets on reboot +ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC using more than 90% of its max +ZFS_REPORT_FREE_WARN_GB=10 # warn if free RAM below 10GB +ZFS_REPORT_AVAIL_WARN_GB=20 # warn if available RAM below 20GB +ZFS_REPORT_DOCKER_TOP=10 # top 10 Docker containers by memory +``` + +**`ZFS_REPORT_LOG`**: Written to `/var/log/` (tmpfs) — resets on reboot. This is +intentional — the log is for the current uptime's week-over-week comparison, not +long-term history. It avoids flash drive writes entirely. + +--- + +## ━━━ FULL CONFIGURATION REFERENCE ━━━ + +### master.conf + +```bash +# cert_monitor.sh +CERT_WARN_DAYS=30 +CERT_CRIT_DAYS=7 +CERT_TIMEOUT=10 + +# smart_health.sh +SMART_TEMP_WARN=45 +SMART_TEMP_CRIT=55 + +# backup_verify.sh +BACKUP_VERIFY_SAMPLE=10 +BACKUP_VERIFY_MIN_SIZE="1M" + +# bandwidth_monitor.sh +BANDWIDTH_LOG="/boot/config/bandwidth_history.db" +BANDWIDTH_LOG_RETENTION=90 +BANDWIDTH_WARN_GB=50 + +# weekly_health_digest.sh +DIGEST_PROFILE="weekly" +DIGEST_DAY="Sunday" +DIGEST_SMART_ON_WATCHDOG=true +DIGEST_SMART_ON_FALLBACK=true +DIGEST_SMART_ON_CERT_WARN=true +DIGEST_SMART_ON_BANDWIDTH=true + +# emby_session_report.sh +EMBY_REPORT_DAYS=7 +EMBY_REPORT_TOP_N=10 + +# system_tuning_monitor.sh +INOTIFY_WARN_PCT=80 +PHP_FPM_WARN_PCT=80 +TUNING_MONITOR_LOG="$DATA_DIR/tuning_monitor.db" +TUNING_LOG_RETENTION=30 + +# zfs_memory_snapshot.sh +ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log" +ZFS_REPORT_ARC_WARN_PCT=90 +ZFS_REPORT_FREE_WARN_GB=10 +ZFS_REPORT_AVAIL_WARN_GB=20 +ZFS_REPORT_DOCKER_TOP=10 +``` + +### master_host*.conf + +```bash +# cert_monitor.sh +HOST1_CERT_MONITOR_DOMAINS=("domain1.com" "domain2.com") +HOST2_CERT_MONITOR_DOMAINS=("domain3.com") + +# smart_health.sh +HOST1_SMART_IGNORE_DRIVES=("sda") +HOST2_SMART_IGNORE_DRIVES=("sda") + +# backup_verify.sh (leave empty to use DAILY_SYNC_SHARES) +HOST1_BACKUP_VERIFY_SHARES=() +HOST2_BACKUP_VERIFY_SHARES=() + +# emby_session_report.sh +HOST1_EMBY_URL="http://192.168.50.2:8096" +HOST1_EMBY_API_KEY="" +HOST2_EMBY_URL="http://192.168.50.3:8096" +HOST2_EMBY_API_KEY="" + +# zfs_memory_snapshot.sh +HOST1_ZFS_REPORT_IGNORE_POOLS=("disk10" "disk9" "disk8" "disk6" "disk5") +HOST2_ZFS_REPORT_IGNORE_POOLS=() +``` + +--- + +## ━━━ RECOMMENDED SCHEDULE ━━━ + +```bash +# Daily +0 8 * * * weekly_health_digest.sh # DIGEST_PROFILE controls notify frequency + +# Every 6 hours — background snapshot +0 */6 * * * system_tuning_monitor.sh # inotify + php-fpm utilisation tracking + +# Sunday morning block — runs after nightly maintenance completes (~3am) +# By 6am the weekly restarts, log clears, and media maintenance have finished. +# Monitors see a freshly maintained system. +0 6 * * 0 zfs_memory_snapshot.sh # ZFS + memory — first, before everything +0 7 * * 0 smart_health.sh # drive SMART health +0 9 * * 0 cert_monitor.sh # SSL cert expiry +0 10 * * 0 backup_verify.sh # rsync mirror integrity +0 11 * * 0 emby_session_report.sh # Emby streaming usage +0 11 * * 0 bandwidth_monitor.sh --report # rsync transfer summary + +# Automatic — no scheduling needed +# bandwidth_monitor.sh --log-transfer is called by rsync.sh after each sync. +``` + +--- + +## ━━━ FLAG REFERENCE ━━━ + +All monitor scripts support the same four flags: + +### --dry-run + +Runs checks without sending any notifications. Useful for: +- First-time setup validation (verify the script finds your domains/drives/shares) +- Ad-hoc health checks without triggering notification noise +- Testing after configuration changes + +```bash +cert_monitor.sh --dry-run # check all certs, show results, no notify +smart_health.sh --dry-run # show which drives would be checked +backup_verify.sh --dry-run # show which files would be sampled +bandwidth_monitor.sh --dry-run # generate report without sending +weekly_health_digest.sh --dry-run # generate digest, no notify regardless of profile +zfs_memory_snapshot.sh --dry-run # console output only, no log write +system_tuning_monitor.sh --dry-run # measure and show, no log write +emby_session_report.sh --dry-run # test connectivity, generate report, no notify +``` + +### --status + +Shows current configuration and exits without running checks. Use to verify +configuration is loaded correctly after editing `master.conf` or `master_host*.conf`. + +```bash +cert_monitor.sh --status # domain list, CERT_WARN_DAYS, CERT_CRIT_DAYS, timeout +smart_health.sh --status # ignore list, temperature thresholds +backup_verify.sh --status # share list, sample size, min file size +bandwidth_monitor.sh --status # log path, retention, warn threshold, log stats +weekly_health_digest.sh --status # profile, DIGEST_DAY, smart trigger settings +zfs_memory_snapshot.sh --status # pool ignore list, memory thresholds +system_tuning_monitor.sh --status # warn thresholds, log path, retention +emby_session_report.sh --status # Emby URL, API key (masked), report period +``` + +### --log + +Verbose per-item output. Every domain/drive/file/section logs its result explicitly +instead of being silent on pass. Use when investigating or after a configuration change. + +### Normal run (no flags) + +Silent on healthy — only problems produce output or notifications. +`weekly_health_digest.sh` is the exception: it always generates a digest, but +`DIGEST_PROFILE` controls whether a notification is sent. + +--- + +## ━━━ ADDING A NEW MONITOR ━━━ + +All monitor scripts share the same pattern: + +```bash +#!/bin/bash +# [standard header with PURPOSE / SAFEGUARDS / CONFIGURATION / RUNTIME MODES] + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../load_config.sh" + +SILENT_MODE=false # monitor script — output is the point + +parse_args "$@" + +# root check if needed +# tool validation (validate_unraid_cmd) +acquire_lock +detect_hosts # if host-specific config needed + +# [status block] +# [dry-run block] + +# do checks +# notify() only on problems — silent on healthy +``` + +Key properties: +- `SILENT_MODE=false` — monitors produce output +- `acquire_lock` — prevent duplicate runs +- `notify()` only on problems — never notify on healthy results +- `--dry-run` skips `notify()` calls entirely +- `--status` shows config and exits before any checks run +- `--log` enables verbose per-item output diff --git a/Monitors/README-Monitors.md b/Monitors/README-Monitors.md index d6c2b32..afcd8a0 100644 --- a/Monitors/README-Monitors.md +++ b/Monitors/README-Monitors.md @@ -1,19 +1,10 @@ -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# 📊 MONITORS -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# ━━━━━ MONITORS ━━━━━ **Observe, measure, and report — never act.** Scripts in this folder watch the system and notify when something needs attention. They do not restart containers, do not reboot servers, do not delete files. The other folders handle intervention. This folder handles awareness. -``` -Monitors/ ← observes and reports (this folder) -Docker_Essentials/ ← acts on containers -unRAID_Essentials/ ← acts on the server -Media/ ← maintains the library -``` - > **All monitor scripts are safe to run at any time.** They are read-only except for > `bandwidth_monitor.sh` (one bounded append per rsync run) and > `system_tuning_monitor.sh` (one bounded append every 6 hours). Every other script @@ -21,133 +12,91 @@ Media/ ← maintains the library --- -## ━━━ THE PROBLEM THAT BUILT THIS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - ---- - -### 🔴 Certs Expiring Silently Despite Renewal +## ━━━ THE PROBLEM THAT BUILT THIS ━━━ +**Certs Expiring Silently Despite Renewal** Certbot runs on schedule. NPM shows the cert as renewed. But the web server was not reloaded after renewal — the old expired cert is still being served to external clients. -NPM's dashboard shows green. `openssl s_client` from the outside shows a cert that -expired three days ago. Users see certificate errors. You see a healthy NPM status. +NPM's dashboard shows green. Users see certificate errors. +Fix: `cert_monitor.sh` connects directly via `openssl` — not to NPM's API, but to the +actual TLS handshake the outside world sees. If users would see a certificate error, +this script catches it. -The fix: `cert_monitor.sh` connects directly via `openssl` — not to NPM's API, not -to any internal check, but to the actual TLS handshake the outside world sees. It -catches renewed-but-not-reloaded certs, wrong certs being served, chain issues that -are invisible from the inside. If users would see an error, this script catches it. +**No Early Warning on Drive Degradation** +A drive accumulates reallocated sectors silently over weeks. Nothing fails. No error +in the Docker logs. No alert from unRAID. The drive is heading toward failure and the +only way to know is to actively query SMART data. +Fix: `smart_health.sh` queries SMART attributes for every drive every week. Drive failure +has a warning curve — weekly checks give you time to act on it. + +**rsync Succeeds But the Copy Is Wrong** +rsync reports success. Exit code 0. But a storage hardware issue on either server +silently corrupted a file during the write. rsync is satisfied — sizes match, timestamps +match — but the content is wrong. +Fix: `backup_verify.sh` independently computes MD5 checksums on both sides and compares. +It does not trust rsync's exit code. It verifies the actual content matches. + +**inotify Exhaustion Producing Unexplained Symptoms** +inotify watches are silently exhausted. Downloads finish but arr containers don't see +them. Live TV stutters. Library updates stop. None of these produce clear error messages. +Fix: `system_tuning_monitor.sh` snapshots inotify and PHP-FPM utilisation every 6 hours. +The weekly digest shows peak, average, and warning count so you see the trend before it +becomes a problem. + +**No Single View of the Ecosystem's Health** +System state is spread across a dozen state files — watchdog strikes, fallback state, +bandwidth history, cert status, transcode stats. No single place to see how everything +is doing. +Fix: `weekly_health_digest.sh` reads all state files and compiles them into one digest. +Three profiles (always / smart / weekly) control how often it notifies. --- -### 🔴 No Early Warning on Drive Degradation - -A drive accumulates reallocated sectors silently over weeks. The count climbs from 0 -to 12 to 47. Nothing fails. No error in the Docker logs. No alert from unRAID. -The drive is in progressive failure — heading toward a point of no return — and the -only way to know is to actively query SMART data. Which nobody does until something -goes wrong. - -The fix: `smart_health.sh` queries SMART attributes for every drive every week. -Reallocated sectors, pending sectors, uncorrectable sectors, temperature — anything -concerning triggers a notification. Drive failure has a warning curve. Weekly SMART -checks give you time to act on it. - ---- - -### 🔴 rsync Succeeds But the Copy Is Wrong - -`rsync` reports success. Transfer completed. Exit code 0. But a storage hardware -issue on either server silently corrupted a file during the write. The file exists -on both ends. rsync is satisfied — sizes match, timestamps match. Nobody knows the -content is wrong until someone tries to play the file and it fails, or worse, until -a drive failure reveals the backup was corrupt from the start. - -The fix: `backup_verify.sh` independently computes MD5 checksums on both sides of -the rsync mirror and compares them. It does not trust rsync's exit code. It does not -trust that the file exists. It verifies the actual content matches. Silent data -corruption is the failure mode backup systems exist to catch — and the only way to -catch it is to verify independently. - ---- - -### 🔴 inotify Exhaustion Producing Unexplained Symptoms - -inotify watches are silently exhausted. The kernel limit is hit. New file events -stop being delivered — silently. Downloads finish but arr containers don't see them. -Live TV stutter appears because media files aren't being detected as they arrive. -Library updates stop happening. None of these produce clear error messages — the -containers are running, the files are there, nothing is obviously wrong. The symptom -looks like a network problem or a container bug. It is an inotify problem. - -The same issue appears with PHP-FPM worker exhaustion — WebGUI requests start -queueing, timeouts appear, the dashboard feels sluggish. The worker count is at -ceiling and you have no visibility into how often it happens or how close it gets. - -The fix: `system_tuning_monitor.sh` snapshots inotify and PHP-FPM utilisation every -6 hours. The weekly coffee report shows peak, average, and warning count. You see -the trend before it becomes a problem. `inotify_tuning.sh` and `php_fpm_max_children.sh` -fix the limits — this script tells you whether the fix is holding. - ---- - -### 🔴 No Single View of What the Continuous Scripts Are Doing - -Three scripts run continuously as background processes: `system_watchdog.sh`, -`docker_watchdog.sh`, and `failover.sh`. Are they running? How long have they been -up? Are there active strikes against any container? Is failover in a non-NORMAL state? -Is the skip list empty? You have to check each state file individually, parse the -format yourself, and mentally correlate the information. - -The fix: `continuous_scripts_status.sh` — a live dashboard that reads all state files -and presents a single coherent view. Run it any time to know exactly what the watchdogs -are doing, what strikes are active, what the failover state is, and how the system -health looks right now. One command, complete picture. - ---- - -## ━━━ WHAT THIS FOLDER DOES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ━━━ WHAT THIS FOLDER DOES ━━━ Four distinct observation roles: ---- +**Live Connectivity Checks** — Active probes against real endpoints +`cert_monitor.sh` and `backup_verify.sh` go out and test things directly. They connect, +compute, compare. The result is a measurement of the actual state of the world — not a +reading of a cached status. -### 🔍 Live Connectivity Checks — Active probes against real endpoints +**State File Aggregation** — Synthesising the ecosystem's own data +`weekly_health_digest.sh` reads the state files the watchdogs and orchestrators have +already written. It doesn't gather new data — it organises existing data into a +human-readable summary. -`cert_monitor.sh` and `backup_verify.sh` go out and test things. They connect, -compute, compare. The result is a direct measurement of the actual state of the -world — not a reading of a cached status. +**Time-Series Snapshots** — Tracking trends over time +`bandwidth_monitor.sh` and `system_tuning_monitor.sh` write one bounded log entry per +interval. Over time this builds a picture of trends — whether inotify pressure is growing, +whether transfer sizes are increasing, whether PHP-FPM is consistently near ceiling. + +**Hardware Health** — Checking what the hardware itself reports +`smart_health.sh` and `zfs_memory_snapshot.sh` query the hardware and kernel directly — +drive SMART attributes, ZFS pool state, ARC statistics, kernel memory pressure. --- -### 📋 State File Aggregation — Synthesising the ecosystem's own data +## ━━━ RELATIONSHIP TO OTHER FOLDERS ━━━ -`weekly_health_digest.sh` and `continuous_scripts_status.sh` read the state files -the watchdogs and orchestrators have already written. They don't gather new data — -they organise existing data into human-readable summaries. +``` +Monitors/ ← observes and reports (this folder) +Docker_Essentials/ ← acts on containers (docker_watchdog starts/stops) +unRAID_Essentials/ ← acts on the server (system_watchdog, inotify_tuning) +Fallback/ ← acts on the full stack (failover, handback) +Rsync/ ← calls bandwidth_monitor.sh (auto-logs each sync) +``` + +`weekly_health_digest.sh` reads state files written by scripts in Docker_Essentials, +unRAID_Essentials, Fallback, and Rsync. It is the only script in this folder with +runtime dependencies on other folders' output — everything else is fully independent. --- -### 📈 Time-Series Snapshots — Tracking trends over time +## ━━━ FLASH DRIVE WRITE POLICY ━━━ -`bandwidth_monitor.sh` and `system_tuning_monitor.sh` write one bounded log entry -per interval. Over time this builds a picture of trends — whether inotify pressure -is growing, whether transfer sizes are increasing, whether PHP-FPM is consistently -near ceiling. - ---- - -### 🖥️ Hardware Health — Checking what the hardware itself reports - -`smart_health.sh` and `zfs_memory_snapshot.sh` query the hardware and kernel -directly — drive SMART attributes, ZFS pool state, ARC statistics, kernel memory -pressure. Things that are invisible in application-level monitoring. - ---- - -## ━━━ FLASH DRIVE WRITE POLICY ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -unRAID boots from a USB flash drive. Flash drives have limited write cycles. -Every script in this folder is designed with this constraint in mind. +unRAID boots from a USB flash drive with limited write cycles. Every script in this +folder is designed with this constraint in mind. | Script | Writes to flash | Detail | |--------|----------------|--------| @@ -157,825 +106,50 @@ Every script in this folder is designed with this constraint in mind. | `backup_verify.sh` | Never | SSH + MD5 comparison only | | `emby_session_report.sh` | Never | API queries only | | `weekly_health_digest.sh` | Never | reads existing state files only | -| `continuous_scripts_status.sh` | Never | reads existing state files only | | `bandwidth_monitor.sh` | One append + one trim per rsync run | Bounded — never exceeds `BANDWIDTH_LOG_RETENTION` lines | | `system_tuning_monitor.sh` | One append + one trim every 6 hours | Bounded — trimmed to `TUNING_LOG_RETENTION` days | -The two scripts that write to flash are designed to be minimal and bounded. They -write one line and trim old entries on every write — the files never grow. - --- -## ━━━ SCRIPTS AT A GLANCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ━━━ SCRIPTS IN THIS FOLDER ━━━ | Script | What It Watches | When | |--------|----------------|------| -| `cert_monitor.sh` | SSL cert expiry via live TLS connection | Weekly — Sunday 9am | -| `smart_health.sh` | Drive SMART health attributes | Weekly — Sunday 7am | | `zfs_memory_snapshot.sh` | ZFS pool health, ARC stats, memory pressure | Weekly — Sunday 6am | +| `smart_health.sh` | Drive SMART health attributes | Weekly — Sunday 7am | +| `cert_monitor.sh` | SSL cert expiry via live TLS connection | Weekly — Sunday 9am | | `backup_verify.sh` | rsync mirror integrity via MD5 checksums | Weekly — Sunday 10am | -| `bandwidth_monitor.sh` | rsync transfer history and trends | Auto (by rsync.sh) + weekly report | -| `weekly_health_digest.sh` | Full ecosystem health aggregation | Daily (profile controls notify) | | `emby_session_report.sh` | Emby streaming usage and library stats | Weekly — Sunday 11am | +| `bandwidth_monitor.sh` | rsync transfer history and trends | Auto (called by rsync.sh) + weekly report Sunday 11am | +| `weekly_health_digest.sh` | Full ecosystem health aggregation | Daily 8am (DIGEST_PROFILE controls notify) | | `system_tuning_monitor.sh` | inotify + PHP-FPM utilisation trends | Every 6 hours | -| `continuous_scripts_status.sh` | Live dashboard for all background processes | On demand | --- -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🔐 cert_monitor.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -SSL certificate expiry monitoring via live TLS connection. Connects directly to each -domain via `openssl s_client` and reads the certificate actually being served — -not the certificate file, not NPM's API, not any cached status. - -```bash -# Scheduled: 0 9 * * 0 (Sunday 9am weekly) -``` - ---- - -### ── Why Direct openssl, Not an API ────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# API-based cert checks ask the certificate manager if the cert is valid. -# openssl checks ask the server what cert it is actually serving. -# These are not the same question and the answers can differ. -# -# Direct openssl catches: -# - Cert renewed in NPM but web server not reloaded (old cert still serving) -# - Wrong cert being served (subdomain pointing to wrong vhost) -# - Chain issues visible externally but not internally -# - NPM reporting healthy while clients see an expired cert -# -# If a user would see a certificate error in their browser, this script -# catches it. If the cert manager says it's fine but the server is serving -# an old cert, this script catches that too. -``` - ---- - -### ── Results Per Domain ─────────────────────────────────────────────────────── +## ━━━ HOW THE SCRIPTS RELATE ━━━ ``` -HEALTHY > CERT_WARN_DAYS remaining → silent ✅ -WARNING ≤ CERT_WARN_DAYS remaining → notification -CRITICAL ≤ CERT_CRIT_DAYS remaining → urgent notification -FAILED could not connect or parse cert → notification +rsync.sh ──────────────────────────────────► bandwidth_monitor.sh --log-transfer + (after each sync completes) (one log entry per sync, bounded) + +Sunday morning block (after nightly maintenance): + 6am zfs_memory_snapshot.sh ─────────────► ZFS_REPORT_LOG (tmpfs) + 7am smart_health.sh ───────────────────── (notification only) + 9am cert_monitor.sh ───────────────────── (notification only) + 10am backup_verify.sh ──────────────────── (notification only) + 11am emby_session_report.sh ────────────── (notification only) + 11am bandwidth_monitor.sh --report ───────► reads BANDWIDTH_LOG + +Every 6h: + system_tuning_monitor.sh ──────────────────► TUNING_MONITOR_LOG + +Daily 8am: + weekly_health_digest.sh ── reads ──────────► FALLBACK_STATE_FILE + ── reads ──────────► WATCHDOG_STATE_FILE + ── reads ──────────► SYS_WATCHDOG_STATE_FILE + ── reads ──────────► SYS_WATCHDOG_FAILED_FILE + ── reads ──────────► BANDWIDTH_LOG + ── reads ──────────► TRANSCODE_DAILY_LOG + ── reads ──────────► TUNING_MONITOR_LOG + ── live openssl ────► CERT_MONITOR_DOMAINS ``` - -Notifications are batched by severity — one message lists all domains at WARNING, -a separate message lists all domains at CRITICAL. Not one notification per domain. - ---- - -### ── Configuration ──────────────────────────────────────────────────────────── - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# Each server monitors its own domains — detect_hosts() aliases the correct list. -# Each domain and subdomain is a separate entry — they have independent certs. -# -HOST1_CERT_MONITOR_DOMAINS=( - "Gmer4Lfe.com" # primary domain — checks the actual TLS handshake - "Gmer4Lfe.us" # secondary domain - # "auth.Gmer4Lfe.com" # add subdomains separately — they have separate certs - # "cloud.Gmer4Lfe.com" -) - -# master.conf - CERT_WARN_DAYS=30 # warn this many days before expiry — time to investigate - CERT_CRIT_DAYS=7 # critical alert — action needed now - CERT_TIMEOUT=10 # seconds per domain before declaring FAILED -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -cert_monitor.sh # normal run — silent if all healthy -cert_monitor.sh --dry-run # check all certs and show results, no notification sent -cert_monitor.sh --log # verbose output per domain -cert_monitor.sh --status # show domain list and thresholds -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 💾 smart_health.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Drive SMART health monitoring. Queries SMART attributes for every drive on the system -weekly. Catches drive degradation before it becomes drive failure — most spinning -drives go through a warning period before dying outright. - -```bash -# Scheduled: 0 7 * * 0 (Sunday 7am weekly) -``` - ---- - -### ── What It Monitors Per Drive ────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# The attributes that actually indicate drive health problems: -# -Reallocated_Sector_Ct any > 0 = warning - The drive found a bad sector and swapped in a spare. - 0 spare = healthy. 1 spare = warning. More = escalating concern. - This is the first sign a drive is wearing out. - -Current_Pending_Sector any > 0 = warning - Sectors the drive suspects are bad but hasn't confirmed yet. - May recover on next read. May escalate to Reallocated. Watch it. - -Offline_Uncorrectable any > 0 = critical - Sectors the drive tried to correct during offline tests and couldn't. - No spares. No recovery. Data loss risk. - -Temperature_Celsius vs thresholds from dynamix.cfg - High sustained temperature shortens drive life significantly. - Script reads unRAID's own configured thresholds — no duplication needed. - -Overall SMART status PASSED / FAILED - The drive's own self-assessment. FAILED = get data off this drive now. - -Power_On_Hours informational - Drive age estimate — useful context for evaluating other attributes. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Drive Discovery + Temperature Source ──────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Drives discovered automatically — /dev/sd* and /dev/nvme* on every run. -# NVMe drives use different attribute names — detected and handled automatically. -# No drive list to maintain in configuration. -# -# Temperature thresholds read from /boot/config/plugins/dynamix/dynamix.cfg: -# hot, max, hotssd, maxssd — unRAID's own configured values. -# Falls back to SMART_TEMP_WARN / SMART_TEMP_CRIT from master.conf if not found. -# This means smart_health.sh and unRAID's dashboard use the same thresholds. -``` - ---- - -### ── Configuration ──────────────────────────────────────────────────────────── - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# Drives to skip — typically the boot USB flash drive. -# unRAID boots from USB which appears as sda. Flash drives either don't -# support SMART or report meaningless values. Skip it. -# -HOST1_SMART_IGNORE_DRIVES=( - "sda" # boot USB flash drive — no meaningful SMART data -) - -# master.conf - SMART_TEMP_WARN=45 # °C — fallback if dynamix.cfg not found - SMART_TEMP_CRIT=55 # °C — fallback -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -smart_health.sh # normal run — silent if all drives healthy -smart_health.sh --dry-run # show which drives would be checked, no smartctl -smart_health.sh --log # verbose — full attribute dump per drive -smart_health.sh --status # show config and ignore list -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🗄️ zfs_memory_snapshot.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Weekly ZFS pool health and memory diagnostic report. Combines ZFS pool status, ARC -statistics, Docker memory usage, and kernel memory pressure into a single snapshot. -Informational only — `system_watchdog.sh` handles threshold-based intervention. - -```bash -# Scheduled: 0 6 * * 0 (Sunday 6am weekly — before other Sunday scripts) -``` - ---- - -### ── What It Reports ───────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# ZFS pool health: -# Status (ONLINE/DEGRADED/FAULTED) per pool -# Size, allocated, free, capacity, health per pool -# Pools in ZFS_REPORT_IGNORE_POOLS are excluded from reporting -# (they remain fully monitored by unRAID — this only affects the report) -# -# ARC statistics: -# Current ARC size vs max — what percentage of max is being used -# Metadata ARC — how much ARC is occupied by metadata vs data -# ARC hit rate — cache effectiveness -# Warns if ARC utilisation exceeds ZFS_REPORT_ARC_WARN_PCT -# -# Memory status: -# Total, free, available RAM -# Warns if free < ZFS_REPORT_FREE_WARN_GB -# Warns if available < ZFS_REPORT_AVAIL_WARN_GB -# -# Docker memory: -# Top ZFS_REPORT_DOCKER_TOP containers by memory usage -# Useful for spotting containers approaching their watchdog limits -# -# Kernel pressure: -# vmstat snapshot — 3 samples showing CPU and memory activity -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Output Goes to Console and Log ───────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# Output is written to both console (for User Scripts output log) and ZFS_REPORT_LOG. -# The log file lets you compare pool health week over week without having to -# remember what last week's numbers were. Open it to see the trend. -# In dry-run mode — console only, nothing written to log. -# - ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log" # tmpfs — resets on reboot -``` - ---- - -### ── Configuration ──────────────────────────────────────────────────────────── - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# Single-disk ZFS array members can be excluded from the report. -# They remain fully monitored by unRAID — this only affects the weekly snapshot. -# Each disk in a JBOD ZFS configuration appears as its own pool — there can be many. -# -HOST1_ZFS_REPORT_IGNORE_POOLS=( - "disk10" # JBOD member — high usage expected, exclude from report noise - "disk9" - "disk8" - "disk6" - "disk5" -) - -# master.conf - ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC using more than 90% of its max - ZFS_REPORT_FREE_WARN_GB=10 # warn if free RAM below 10GB - ZFS_REPORT_AVAIL_WARN_GB=20 # warn if available RAM below 20GB - ZFS_REPORT_DOCKER_TOP=10 # top 10 Docker containers by memory -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -zfs_memory_snapshot.sh # normal run — writes to log + console -zfs_memory_snapshot.sh --dry-run # console only, no log write -zfs_memory_snapshot.sh --log # verbose output -zfs_memory_snapshot.sh --status # show config and pool ignore list -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## ✅ backup_verify.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -rsync mirror integrity verification via independent MD5 checksums. Randomly samples -files from each share, computes checksums on both sides, and compares. Does not trust -rsync's exit code. Verifies the actual content matches. - -```bash -# Scheduled: 0 10 * * 0 (Sunday 10am weekly) -``` - ---- - -### ── Why Verify Independently ───────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# rsync reports success when: -# - The transfer completed without network errors -# - The file sizes and modification times match -# -# rsync does NOT detect: -# - Silent corruption during the transfer (bitflip in transit) -# - Corruption written to storage at rest (faulty drive sector) -# - Files that matched size/mtime but had wrong content (rare, happens) -# -# These failure modes produce exit code 0. rsync says "done." -# backup_verify.sh checks whether "done" means "correct." -# -# Sample size is intentionally small (10 files per share, default) — -# this is a spot check, not an exhaustive verify. It catches systematic -# problems and hardware issues while running in minutes, not hours. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Results Per File ───────────────────────────────────────────────────────── - -``` -MATCH checksums identical on both sides → correctly mirrored ✅ -MISMATCH file exists on both, checksums differ → sync failure or corruption -MISSING file exists locally but not on remote → not yet synced or deleted -``` - -All MISMATCHes and any significant number of MISSINGs trigger a notification. - ---- - -### ── Pre-flight Checks Before Any SSH ─────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Two checks before computing a single checksum: -# -# check_connectivity() — is the remote Tailscale IP reachable? -# Without this: all files would show as MISSING (SSH timeout per file). -# False alarm from a network hiccup. -# -# check_remote_array() — is the remote /mnt/user mounted? -# Without this: remote files don't exist yet (array not started). -# All files would show as MISSING. False catastrophic alarm. -# -# Both must pass before any checksums are computed. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Configuration ──────────────────────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── - BACKUP_VERIFY_SAMPLE=10 # random files sampled per share per run - BACKUP_VERIFY_MIN_SIZE="1M" # skip files smaller than this (checksums waste - # time on tiny files with low corruption risk) - -# master_host*.conf -# ───────────────────────────────────────────────────────────────────────────── -# Leave empty to use HOST*_DAILY_SYNC_SHARES automatically. -# No additional configuration needed for the standard setup. -# - HOST1_BACKUP_VERIFY_SHARES=( - # empty — uses HOST1_DAILY_SYNC_SHARES automatically - # "/mnt/user/Movies" # override here to check specific shares only - ) -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -backup_verify.sh # normal run — silent if all samples match -backup_verify.sh --dry-run # show which files would be sampled, no checksums -backup_verify.sh --log # verbose — show checksum comparison per file -backup_verify.sh --status # show share list and sample configuration -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 📡 bandwidth_monitor.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -rsync transfer history logging and weekly summary reporting. Called automatically by -`rsync.sh` after each sync — no manual scheduling needed for the logging part. The -report can be run manually or on a weekly schedule. - -```bash -# Log mode: called automatically by rsync.sh — no manual scheduling needed -# Report mode — scheduled: 0 11 * * 0 (Sunday 11am weekly) -``` - ---- - -### ── Two Modes ──────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# --log-transfer profile duration_seconds status bytes_transferred -# Called by rsync.sh after each sync completes. -# Appends one line to BANDWIDTH_LOG. -# Trims entries older than BANDWIDTH_LOG_RETENTION days. -# Flags transfers exceeding BANDWIDTH_WARN_GB for report highlighting. -# You never call this manually. -# -# --report (or no arguments) -# Reads the accumulated log and generates a weekly summary. -# Per-profile breakdown: run count, total transferred, average duration, failures. -# Last 7 days activity timeline. -# Flags any days or transfers that exceeded BANDWIDTH_WARN_GB. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Log Format — Version-Proof Design ────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# One line per transfer: -# YYYY-MM-DD|HH:MM|profile|duration_seconds|status|bytes_transferred -# -# Example entries: -# 2026-04-14|01:23|critical-data|143|success|2847362048 -# 2026-04-14|01:31|arrs_stack|287|success|891234567 -# 2026-04-14|02:15|movies|1847|failed|0 -# -# Why this format: -# Earlier designs parsed rsync's human-readable output for bytes transferred. -# rsync changes its output format between versions — those parsers break silently. -# Status (success/failed) and duration are always available regardless of rsync version. -# Bytes captured from rsync --stats output via awk — version-stable field names. -# This format survives any rsync update with no changes. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Configuration ──────────────────────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── - BANDWIDTH_LOG="/boot/config/bandwidth_history.db" # survives reboots - BANDWIDTH_LOG_RETENTION=90 # days — file stays bounded, never grows unbounded - BANDWIDTH_WARN_GB=50 # flag transfers or daily totals exceeding this -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -bandwidth_monitor.sh # generate report (default mode) -bandwidth_monitor.sh --report # generate report (explicit) -bandwidth_monitor.sh --status # show config and log statistics -bandwidth_monitor.sh --log # verbose output - -# This is called automatically by rsync.sh — never call manually: -bandwidth_monitor.sh --log-transfer "profile" 287 "success" 891234567 -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 📋 weekly_health_digest.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Full ecosystem health aggregation. Reads state files from across the entire system — -watchdog strikes, failover state, skip list, bandwidth history, transcode stats, cert -status — and compiles them into a single digest report. Reads only, writes nothing. - -```bash -# Scheduled: 0 8 * * * (8am daily — DIGEST_PROFILE controls when it notifies) -``` - ---- - -### ── The Key Design — Schedule Daily, Profile Controls Behaviour ───────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# The cron schedule never changes — always daily at 8am. -# DIGEST_PROFILE controls what actually happens on each run. -# Switch profiles by changing one line in master.conf — no cron edit needed. -# - DIGEST_PROFILE="weekly" # always | smart | weekly -``` - ---- - -### ── Three Profiles ─────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# always — sends every run regardless of findings -# Use: daily digest of everything, even when healthy -# Result: daily notification with full system snapshot -# -# smart — sends only when something worth reporting is found -# Use: quiet operation, alerts on issues -# Result: silent on clean days, notification when something needs attention -# What triggers "worth reporting" is configurable via DIGEST_SMART_ON_* toggles -# -# weekly — sends once per week on DIGEST_DAY, silent all other days -# Use: weekly summary without daily noise -# Result: one notification on Sunday, silent Monday-Saturday -# ───────────────────────────────────────────────────────────────────────────── - - DIGEST_DAY="Sunday" # for weekly profile — must match `date +%A` output - -# Smart profile triggers — each independently toggleable - DIGEST_SMART_ON_WATCHDOG=true # send if any active watchdog strikes - DIGEST_SMART_ON_FAILOVER=true # send if failover state is not NORMAL - DIGEST_SMART_ON_CERT_WARN=true # send if any cert within CERT_WARN_DAYS - DIGEST_SMART_ON_BANDWIDTH=true # send if any transfer exceeded BANDWIDTH_WARN_GB -``` - ---- - -### ── Data Sources — Reads Only ──────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Every source is a read — this script writes nothing, changes nothing. -# - FAILOVER_STATE_FILE # current failover state (NORMAL/FAILOVER/etc.) - SYS_WATCHDOG_FAILED_FILE # container skip list — anything here needs attention - WATCHDOG_STATE_FILE # active container watchdog strikes - SYS_WATCHDOG_STATE_FILE # active system watchdog strikes - BANDWIDTH_LOG # yesterday's transfer history - TRANSCODE_DAILY_LOG # weekly transcode statistics - CERT_MONITOR_DOMAINS # live openssl cert check per domain - RAMDISK_PATH / TRANSCODE_LINK # current transcode location and usage -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -weekly_health_digest.sh # normal run — profile determines if notification sent -weekly_health_digest.sh --dry-run # show digest output, no notification regardless of profile -weekly_health_digest.sh --log # verbose per-section output -weekly_health_digest.sh --status # show profile, day, and trigger configuration -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🎬 emby_session_report.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Emby usage report via the Emby API. Queries activity logs and session history to -produce a weekly summary of streaming activity. No persistent state — queries fresh -on every run. - -```bash -# Scheduled: 0 11 * * 0 (Sunday 11am weekly) -``` - ---- - -### ── What It Reports ───────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── - Server info: name, version, uptime - Active sessions: current streams, direct play vs transcode breakdown - Library stats: movie count, episode count, song count - Activity history: play events from the last EMBY_REPORT_DAYS days - Top content: most played items in the period (top EMBY_REPORT_TOP_N) - Most active users: who watched the most in the period - Transcode ratio: how often transcoding was needed vs direct play - notifies if > 80% of streams are transcoding (config issue) - Ramdisk status: current transcode location and usage -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Configuration ──────────────────────────────────────────────────────────── - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# detect_hosts() aliases these from HOST*_ prefixed vars. -# Each server reports on its own Emby instance automatically. -# - HOST1_EMBY_URL="http://192.168.50.2:8096" - HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829" -# -# To get an API key: -# Emby UI → Settings → API Keys → New API Key -# Give it a name (e.g. "unraid_scripts") — copy the key - -# master.conf - EMBY_REPORT_DAYS=7 # report period in days - EMBY_REPORT_TOP_N=10 # top N content items to show -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -emby_session_report.sh # generate report -emby_session_report.sh --dry-run # test API connectivity only, no notification -emby_session_report.sh --log # verbose per-section output -emby_session_report.sh --status # show config and Emby connection status -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## ⚙️ system_tuning_monitor.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -inotify and PHP-FPM utilisation tracking. Snapshots current usage every 6 hours and -writes to a bounded log. The weekly coffee report reads this log for peak/average/ -warning counts over the week. Silent when healthy — warns when thresholds are hit. - -```bash -# Scheduled: 0 */6 * * * (every 6 hours) -``` - ---- - -### ── What It Tracks ─────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# inotify instances: -# Current in use vs kernel limit (INOTIFY_MAX_INSTANCES) -# Utilisation % — warns above INOTIFY_WARN_PCT (default 80%) -# Top 5 consumers by instance count -# -# What inotify exhaustion looks like at the application level: -# Downloads complete but arrs don't detect them -# Live TV stutter as media files aren't seen arriving -# Library updates stop happening -# No clear error anywhere — things just stop working -# -# php-fpm workers: -# Active workers vs PHP_MAX_CHILDREN limit -# Utilisation % — warns above PHP_FPM_WARN_PCT (default 80%) -# -# What PHP-FPM exhaustion looks like: -# unRAID WebGUI becomes slow or times out -# Dashboard requests queue behind each other -# Settings saves hang -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Log Format ────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# One line per snapshot — trimmed to TUNING_LOG_RETENTION days on each write: -# DATE|TIME|INOTIFY_USED|INOTIFY_LIMIT|INOTIFY_PCT|INOTIFY_WARN| -# PHPFPM_ACTIVE|PHPFPM_MAX|PHPFPM_PCT|PHPFPM_WARN -# -# INOTIFY_WARN and PHPFPM_WARN are 1/0 flags — 1 means threshold was exceeded -# during this snapshot. The weekly coffee report counts warnings over the week. -``` - ---- - -### ── Configuration ──────────────────────────────────────────────────────────── - -```bash -# master.conf - INOTIFY_WARN_PCT=80 # warn if inotify instances above 80% of limit - PHP_FPM_WARN_PCT=80 # warn if php-fpm workers above 80% of max - TUNING_MONITOR_LOG="$DATA_DIR/tuning_monitor.db" - TUNING_LOG_RETENTION=30 # days — file stays bounded -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🖥️ continuous_scripts_status.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Live dashboard for all continuously running background scripts. Run any time to get -an immediate picture of the ecosystem's operational state. No schedule — on demand only. - ---- - -### ── What It Shows ──────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# For each continuous script (system_watchdog, docker_watchdog, failover): -# Running state, PID, uptime, approximate cycle count -# -# system_watchdog: -# Rootfs usage, free RAM, ZFS ARC size, load average, zombie count, CPU temp -# Active strikes (any check with non-zero strike count) -# Recent watchdog-triggered reboots -# -# docker_watchdog: -# Running vs stopped vs unhealthy container counts -# Required containers — which are running, which are stopped -# Memory-monitored containers — current usage vs hard limits -# Active container strikes -# Recent restart history (per container) -# Skip list — anything here needs human attention -# -# failover: -# Current state (NORMAL/FAILOVER/NO_INTERNET/DARK) -# Which tier containers are active -# Outage duration if in FAILOVER -# Remote server Tailscale visibility (ping result) -# DDNS containers running on this server -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Run any time — no cron, no schedule. This is an on-demand status tool. -# ───────────────────────────────────────────────────────────────────────────── -continuous_scripts_status.sh # live dashboard -continuous_scripts_status.sh --log # verbose output with full state file dumps -``` - ---- - -## ━━━ RECOMMENDED SCHEDULE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Daily -# ───────────────────────────────────────────────────────────────────────────── -0 8 * * * weekly_health_digest.sh # DIGEST_PROFILE controls notify frequency - -# ───────────────────────────────────────────────────────────────────────────── -# Every 6 hours — background snapshot -# ───────────────────────────────────────────────────────────────────────────── -0 */6 * * * system_tuning_monitor.sh # inotify + php-fpm utilisation tracking - -# ───────────────────────────────────────────────────────────────────────────── -# Sunday morning block — runs after nightly maintenance completes (~3am) -# By 6am the weekly restarts, log clears, and media maintenance have finished. -# Monitors see a freshly maintained system. -# ───────────────────────────────────────────────────────────────────────────── -0 6 * * 0 zfs_memory_snapshot.sh # ZFS + memory — first, before everything -0 7 * * 0 smart_health.sh # drive SMART health -0 9 * * 0 cert_monitor.sh # SSL cert expiry -0 10 * * 0 backup_verify.sh # rsync mirror integrity -0 11 * * 0 emby_session_report.sh # Emby streaming usage -0 11 * * 0 bandwidth_monitor.sh --report # rsync transfer summary - -# ───────────────────────────────────────────────────────────────────────────── -# Automatic — no scheduling needed -# ───────────────────────────────────────────────────────────────────────────── -# bandwidth_monitor.sh --log-transfer is called by rsync.sh after each sync. -# continuous_scripts_status.sh is run on demand — no cron entry. -``` - ---- - -## ━━━ ADDING A NEW MONITOR ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -The monitor pattern is consistent across all scripts in this folder: - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# All monitor scripts share these properties: -# -# SILENT_MODE=false — output is the point (monitors are meant to produce output) -# No action taken — observe only, intervention is handled elsewhere -# acquire_lock — prevent concurrent runs producing duplicate output -# detect_hosts() — correct configuration per server via MY_ID -# validate_unraid_cmd — notify script validated before use -# --dry-run support — run checks without sending notifications -# --status support — show configuration and exit -# --log support — verbose per-item output -# Silent on healthy — no notification when everything passes -# -# To add a new monitor: -# 1. Source load_config.sh — gives you all common functions -# 2. Set SILENT_MODE=false — this is a monitor, output is expected -# 3. Call detect_hosts() — correct per-host configuration -# 4. Call acquire_lock — prevent duplicate runs -# 5. Do your checks — read, connect, compare -# 6. notify() only on problems — silent on healthy results -# 7. Support --dry-run (no notify) and --status (show config, exit) -# ───────────────────────────────────────────────────────────────────────────── -``` \ No newline at end of file diff --git a/Monitors/backup_verify.sh b/Monitors/backup_verify.sh index 4924e1d..cef25db 100644 --- a/Monitors/backup_verify.sh +++ b/Monitors/backup_verify.sh @@ -1,49 +1,103 @@ #!/bin/bash # ============================================================================================== -# ================================= Backup Verify ============================================== +# ============================= Backup Verify ================================================== # ============================================================================================== -# Verifies the rsync mirror is healthy by comparing random file samples between -# local and remote servers using MD5 checksums. # -# ── WHAT IT DOES ────────────────────────────────────────────────────────────────────────────── -# Randomly samples BACKUP_VERIFY_SAMPLE files per share above BACKUP_VERIFY_MIN_SIZE, -# computes MD5 checksums locally, then computes the same checksums on the remote via SSH -# and compares results. Catches silent corruption or incomplete syncs that rsync itself -# would not detect. +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# rsync mirror integrity verification via independent MD5 checksums. Scheduled +# weekly (Sunday 10am). Randomly samples BACKUP_VERIFY_SAMPLE files per share +# above BACKUP_VERIFY_MIN_SIZE, computes checksums locally, then computes the +# same checksums on the remote via SSH and compares. # -# ── RESULTS PER FILE ────────────────────────────────────────────────────────────────────────── -# MATCH — checksums identical, file is correctly mirrored ✅ -# MISMATCH — file exists on both but checksums differ — sync may have partially failed -# MISSING — file exists locally but not on remote — not yet synced or deleted on remote +# Per file: MATCH (checksums identical) | MISMATCH (file exists on both but +# checksums differ — sync failure or corruption) | MISSING (file exists locally +# but not on remote). All MISMATCHes and significant MISSINGs trigger notification. +# rsync exit code 0 is not trusted — this script verifies actual content. # -# ── SHARE SELECTION ─────────────────────────────────────────────────────────────────────────── -# Uses HOST*_BACKUP_VERIFY_SHARES if defined, falls back to HOST*_DAILY_SYNC_SHARES. -# Both aliased by detect_hosts() — no manual HOST1/HOST2 selection needed. +# Share list from HOST*_BACKUP_VERIFY_SHARES if defined, otherwise falls back +# to HOST*_DAILY_SYNC_SHARES. Both aliased by detect_hosts(). # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock — prevents concurrent runs producing duplicate/conflicting results -# check_connectivity() — verifies remote reachable before attempting SSH calls -# check_remote_array() — verifies remote array mounted before checksums -# remote array down = all files "missing" = false alarm ✅ -# version parity — verifies both servers on compatible unRAID before trusting results -# SSH_TIMEOUT — all SSH calls protected against hangs -# validate_unraid_cmd — notify script validated before use -# Silent by default — only issues produce output, all-match runs are silent +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── -# HOST*_BACKUP_VERIFY_SHARES — override share list (empty = use DAILY_SYNC_SHARES) -# HOST*_DAILY_SYNC_SHARES — fallback share list -# All aliased by detect_hosts() +# Independent Verification +# rsync reports success when the transfer completed without network errors and +# file sizes and modification times match. It does not detect silent corruption +# during transfer (bitflip in transit), corruption written to storage at rest +# (faulty drive sector), or files that matched size/mtime but had wrong content. +# All of these produce exit code 0. This script checks whether "done" means "correct." # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# BACKUP_VERIFY_SAMPLE — random files to check per share (default 10) -# BACKUP_VERIFY_MIN_SIZE — minimum file size to include in sample (default 1M) +# Intentionally Small Sample +# 10 files per share (default) — a spot check, not an exhaustive verify. +# Catches systematic problems and hardware issues while running in minutes, not +# hours. Full verification would take longer than the rsync itself. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Single Instance Lock +# acquire_lock prevents concurrent runs producing conflicting results. +# +# Remote Connectivity Check +# check_connectivity() verifies the remote Tailscale IP is reachable before +# any SSH calls. Without this, all files show as MISSING on a network hiccup. +# +# Remote Array Check +# check_remote_array() verifies /mnt/user is mounted on the remote before +# computing checksums. Array not started = all files "missing" = false alarm. +# +# Version Parity +# Refuses to run if remote unRAID version doesn't match local. A mismatch +# may mean the remote is in an unexpected state. +# +# SSH Timeout +# SSH_TIMEOUT caps all SSH calls. One hung connection does not block the run. +# +# Notification Validated +# validate_unraid_cmd confirms the notify script is present before use. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# +# HOST*_BACKUP_VERIFY_SHARES +# Shares to verify. Leave empty to use HOST*_DAILY_SYNC_SHARES automatically. +# Aliased by detect_hosts() → BACKUP_VERIFY_SHARES. +# +# HOST*_DAILY_SYNC_SHARES +# Fallback share list if BACKUP_VERIFY_SHARES is empty. Aliased by detect_hosts(). +# +# master.conf +# +# BACKUP_VERIFY_SAMPLE +# Random files checked per share per run. (default: 10) +# +# BACKUP_VERIFY_MIN_SIZE +# Minimum file size to include in sample — tiny files have low corruption +# risk and slow checksums. (default: 1M) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# backup_verify.sh +# Sample files from all shares and compare checksums. Notify on MISMATCH +# or significant MISSING count. Silent when all samples match. +# +# backup_verify.sh --dry-run +# Show which files would be sampled. No checksums computed, no notifications. +# +# backup_verify.sh --status +# Show share list, sample size, and min file size configuration. Then exit. +# +# backup_verify.sh --log +# Verbose per-file checksum comparison output during the run. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# backup_verify.sh — normal run -# backup_verify.sh --dry-run — show sample selection only, no checksums -# backup_verify.sh --log — verbose output -# backup_verify.sh --status — show config and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Monitors/bandwidth_monitor.sh b/Monitors/bandwidth_monitor.sh index 2d4bb39..dafeb88 100644 --- a/Monitors/bandwidth_monitor.sh +++ b/Monitors/bandwidth_monitor.sh @@ -1,46 +1,89 @@ #!/bin/bash # ============================================================================================== -# ================================= Bandwidth Monitor ========================================== +# ============================= Bandwidth Monitor ============================================== # ============================================================================================== -# Logs rsync transfer history and generates weekly summary reports. -# Designed for minimal flash drive impact — one bounded write per rsync run. # -# ── TWO MODES ───────────────────────────────────────────────────────────────────────────────── +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# rsync transfer history logging and weekly summary reporting. Log mode is called +# automatically by rsync.sh after each sync — no manual scheduling needed for +# logging. Report mode scheduled weekly (Sunday 11am). # -# --log-transfer "profile" duration_seconds status -# Called automatically by rsync.sh after each sync completes. -# Appends one line to the log and trims entries older than BANDWIDTH_LOG_RETENTION. -# Flags syncs exceeding BANDWIDTH_WARN_GB in the log for weekly report highlighting. +# Log mode (--log-transfer): appends one line per sync to BANDWIDTH_LOG, trims +# entries older than BANDWIDTH_LOG_RETENTION days. One bounded write per rsync run. # -# --report (or no args) -# Generates a summary from the accumulated log. -# Shows per-profile breakdown, last 7 days, and overall totals. -# This is a monitor script — SILENT_MODE=false — output is the point. +# Report mode (default): reads the accumulated log and generates a summary — +# per-profile breakdown, run count, total transferred, average duration, failures, +# last 7 days activity timeline, any transfers or days exceeding BANDWIDTH_WARN_GB. # -# ── LOG FORMAT ──────────────────────────────────────────────────────────────────────────────── -# One line per transfer — version-proof, never needs rsync output parsing: -# YYYY-MM-DD|HH:MM|profile|duration_seconds|status|bytes_transferred +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== # -# Log file stays bounded to BANDWIDTH_LOG_RETENTION days — trimmed on every write. -# Minimal flash drive impact: one append + one trim per rsync run. +# Version-Proof Log Format +# Earlier designs parsed rsync's human-readable output for bytes transferred. +# rsync changes its output format between versions — those parsers break silently. +# The log line (YYYY-MM-DD|HH:MM|profile|duration|status|bytes) captures bytes +# from rsync --stats via awk using version-stable field names. Survives any rsync +# update with no changes. # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock "wait" — prevents log corruption from concurrent rsync completions -# validate_unraid_cmd — notify script validated before use -# Atomic log write — temp file + mv prevents partial writes on trim -# Log existence check — creates log directory if needed, exits cleanly if unwritable +# Minimal Flash Drive Impact +# unRAID boots from USB flash. One append + one trim per rsync run. The log file +# is bounded to BANDWIDTH_LOG_RETENTION days and never grows unbounded. +# Atomic write: tmp file + mv prevents partial writes during trim. # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# BANDWIDTH_LOG — log file path -# BANDWIDTH_LOG_RETENTION — days before old entries are purged (default 90) -# BANDWIDTH_WARN_GB — flag syncs larger than this in report (default 50) +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Lock With Wait +# acquire_lock "wait" — multiple rsync profiles may complete close together. +# Waiting (not skipping) prevents log entries from being lost. +# +# Atomic Log Write +# Trim uses tmp file + mv — partial writes on log trim cannot corrupt the log. +# +# Log Directory Guard +# Creates the log directory if it doesn't exist. Exits cleanly if unwritable. +# +# Notification Validated +# validate_unraid_cmd confirms the notify script is present before use. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# BANDWIDTH_LOG +# Log file path. Stored on /boot/ to survive reboots. +# (default: /boot/config/bandwidth_history.db) +# +# BANDWIDTH_LOG_RETENTION +# Days before old entries are purged. File stays bounded. (default: 90) +# +# BANDWIDTH_WARN_GB +# Flag transfers or daily totals exceeding this in the report. (default: 50) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# bandwidth_monitor.sh +# Generate transfer history report from accumulated log. +# +# bandwidth_monitor.sh --report +# Generate report (explicit form). +# +# bandwidth_monitor.sh --log-transfer profile secs status bytes +# Log a completed rsync transfer. Called by rsync.sh — do not call manually. +# +# bandwidth_monitor.sh --status +# Show log path, retention, warn threshold, and log statistics. Then exit. +# +# bandwidth_monitor.sh --log +# Verbose output during report generation. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# bandwidth_monitor.sh — generate report -# bandwidth_monitor.sh --report — generate report (explicit) -# bandwidth_monitor.sh --log-transfer profile secs ok — log a transfer (called by rsync.sh) -# bandwidth_monitor.sh --status — show config and exit -# bandwidth_monitor.sh --log — verbose output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Monitors/cert_monitor.sh b/Monitors/cert_monitor.sh index 19226b9..de7b990 100644 --- a/Monitors/cert_monitor.sh +++ b/Monitors/cert_monitor.sh @@ -1,55 +1,90 @@ #!/bin/bash # ============================================================================================== -# ================================= Certificate Monitor ======================================== +# ============================= Certificate Monitor ============================================ # ============================================================================================== -# Monitors SSL certificate expiry for all configured domains by connecting directly -# via openssl — no dependency on NPM or any other service. Reads the actual certificate -# the server is presenting to the outside world. # -# ── WHY DIRECT OPENSSL ──────────────────────────────────────────────────────────────────────── -# Catches real-world cert issues that API-based checks miss: -# - Cert renewed in NPM but server not reloaded (old cert still serving) -# - Wrong cert being served to external clients -# - Cert chain issues not visible from the internal network -# - NPM reporting healthy while the world sees an expired cert +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# SSL certificate expiry monitoring for all configured domains. Scheduled weekly +# (Sunday 9am). Connects via openssl directly to each domain — not to NPM's API, +# not to any internal check, but to the actual TLS handshake the outside world sees. # -# ── BEHAVIOUR ───────────────────────────────────────────────────────────────────────────────── -# Each domain is checked independently — they have independent certs. -# Results per domain: -# HEALTHY — > CERT_WARN_DAYS remaining — silent ✅ -# WARNING — <= CERT_WARN_DAYS remaining — notifies -# CRITICAL — <= CERT_CRIT_DAYS remaining — notifies with urgency -# FAILED — could not connect or parse cert — notifies +# Per domain: HEALTHY (> CERT_WARN_DAYS remaining, silent) | WARNING (≤ CERT_WARN_DAYS) +# | CRITICAL (≤ CERT_CRIT_DAYS) | FAILED (could not connect or parse cert). +# Notifications batched by severity — one message lists all WARNING domains, a +# separate message lists all CRITICAL domains. Not one notification per domain. # -# Notifications batched per severity — one message per severity level, not per domain. -# This is a monitor script — SILENT_MODE=false — output is the point. +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID and aliases HOST*_CERT_MONITOR_DOMAINS → CERT_MONITOR_DOMAINS. -# Each server monitors its own domains — HOST1 monitors Gmer4Lfe.com etc. +# Direct openssl, Not an API +# API-based cert checks ask the certificate manager whether the cert is valid. +# openssl checks ask the server what cert it is actually serving. These are not +# the same question and the answers can differ. Catches: cert renewed in NPM but +# server not reloaded (old cert still serving), wrong cert being served to external +# clients, chain issues visible externally but not internally, NPM reporting healthy +# while the outside world sees an expired cert. # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock — prevents concurrent runs -# detect_hosts() — correct domain list per host via MY_ID aliases -# Empty array guard — warns and exits cleanly if no domains configured -# CERT_TIMEOUT — openssl connects are time-limited per domain -# validate_unraid_cmd — openssl and notify validated before use -# Silent healthy certs — only problems produce visible output +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── -# HOST*_CERT_MONITOR_DOMAINS — domains checked by this host -# Aliased by detect_hosts() — script uses CERT_MONITOR_DOMAINS +# Single Instance Lock +# acquire_lock prevents concurrent runs producing duplicate notifications. # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# CERT_WARN_DAYS — warn when cert expires within this many days (default 30) -# CERT_CRIT_DAYS — critical alert within this many days (default 7) -# CERT_TIMEOUT — seconds per domain before giving up (default 10) +# Per-Host Domain List +# detect_hosts() aliases HOST*_CERT_MONITOR_DOMAINS → CERT_MONITOR_DOMAINS. +# Each server monitors its own domains only. +# +# Empty Array Guard +# Warns and exits cleanly if CERT_MONITOR_DOMAINS is empty — no silent no-op. +# +# Connection Timeout +# CERT_TIMEOUT caps each openssl connection attempt. One unreachable domain +# does not block the remaining domains. +# +# Notification Validated +# validate_unraid_cmd confirms openssl and notify script are present before use. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# +# HOST*_CERT_MONITOR_DOMAINS +# Domains this host monitors. Each domain and subdomain is a separate entry — +# they have independent certs. Aliased by detect_hosts() → CERT_MONITOR_DOMAINS. +# +# master.conf +# +# CERT_WARN_DAYS +# Days before expiry at which to send a warning notification. (default: 30) +# +# CERT_CRIT_DAYS +# Days before expiry at which to send a critical notification. (default: 7) +# +# CERT_TIMEOUT +# Seconds to wait per domain before declaring FAILED. (default: 10) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# cert_monitor.sh +# Check all configured domains and notify on WARNING, CRITICAL, or FAILED. +# Silent when all domains are healthy. +# +# cert_monitor.sh --dry-run +# Check all domains and show results. No notifications sent regardless of result. +# +# cert_monitor.sh --status +# Show domain list, warning thresholds, and timeout. Then exit. +# +# cert_monitor.sh --log +# Verbose per-domain output during the run. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# cert_monitor.sh — normal run -# cert_monitor.sh --dry-run — check certs and show results, no notifications -# cert_monitor.sh --log — verbose output -# cert_monitor.sh --status — show config and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Monitors/emby_session_report.sh b/Monitors/emby_session_report.sh index 1728b91..8437b3b 100644 --- a/Monitors/emby_session_report.sh +++ b/Monitors/emby_session_report.sh @@ -1,53 +1,86 @@ #!/bin/bash # ============================================================================================== -# ================================= Emby Session Report ======================================== +# ============================= Emby Session Report ============================================ # ============================================================================================== -# Generates a usage report from the Emby media server via its API. -# Queries activity logs and session history to produce a summary of what was -# watched, by whom, and how over the configured report period. # -# ── REPORT INCLUDES ─────────────────────────────────────────────────────────────────────────── -# Server info — name, version, uptime -# Active sessions — current streams, direct play vs transcode -# Library stats — movie, episode, song counts -# Activity history — play events from the last EMBY_REPORT_DAYS days -# Top content — most played items in the period (top EMBY_REPORT_TOP_N) -# Most active users — who watched the most in the period -# Transcode ratio — how often transcoding was needed vs direct play -# Ramdisk status — current transcode location and usage +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Emby usage report via the Emby API. Scheduled weekly (Sunday 11am). Queries +# activity logs and session history to produce a summary of what was watched, +# by whom, and how often. No persistent state — queries fresh on every run. # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID and aliases EMBY_URL and EMBY_API_KEY. -# Each server reports on its own Emby instance automatically. +# Reports: server info and uptime, active sessions and transcode ratio, library +# counts (movies/episodes/songs), activity history for the last EMBY_REPORT_DAYS +# days, top EMBY_REPORT_TOP_N content items, most active users, and ramdisk +# transcode status. # -# ── BEHAVIOUR ───────────────────────────────────────────────────────────────────────────────── -# No persistent writes — queries API fresh each run. -# This is a monitor/report script — SILENT_MODE=false — output is the point. -# Silent when healthy (no notification on clean run). -# Notifies only if transcoding is very high (>80% of streams) — may indicate config issue. +# Notifies only if transcoding exceeds 80% of streams — may indicate a client +# configuration issue. Silent on clean runs. # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock — prevents duplicate reports running simultaneously -# check_api() — verifies Emby reachable before queries -# jq + curl validation — exits if either tool missing -# validate_unraid_cmd — notify script validated before use -# Per-section guards — API failure in one section does not abort others +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── -# HOST*_EMBY_URL / HOST*_EMBY_API_KEY -# Aliased by detect_hosts() — script uses EMBY_URL / EMBY_API_KEY +# Single Instance Lock +# acquire_lock prevents duplicate reports running simultaneously. # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# EMBY_REPORT_DAYS — days to include in the report period (default 7) -# EMBY_REPORT_TOP_N — number of top content items to show (default 10) -# RAMDISK_PATH — ramdisk mount path (for transcode status) -# TRANSCODE_LINK — symlink path (for transcode location) +# API Connectivity Check +# check_api() verifies Emby is reachable before any queries. API failure in +# one section does not abort the others — each section guards itself. +# +# Tool Validation +# Checks for curl and jq at startup — exits with a clear error if either is missing. +# +# Per-Host Credentials +# detect_hosts() aliases HOST*_EMBY_URL and HOST*_EMBY_API_KEY → EMBY_URL / EMBY_API_KEY. +# Each server reports on its own Emby instance automatically. +# +# Notification Validated +# validate_unraid_cmd confirms the notify script is present before use. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# +# HOST*_EMBY_URL +# Emby server URL for this host. Aliased by detect_hosts() → EMBY_URL. +# +# HOST*_EMBY_API_KEY +# Emby API key for this host. Aliased by detect_hosts() → EMBY_API_KEY. +# Generate via Emby UI → Settings → API Keys → New API Key. +# +# master.conf +# +# EMBY_REPORT_DAYS +# Number of days to include in the activity history section. (default: 7) +# +# EMBY_REPORT_TOP_N +# Number of top content items to show in the report. (default: 10) +# +# RAMDISK_PATH +# Ramdisk mount path — used for transcode status reporting. +# +# TRANSCODE_LINK +# Symlink path — used to determine current transcode location. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# emby_session_report.sh +# Generate and send Emby usage report. +# +# emby_session_report.sh --dry-run +# Test API connectivity and generate report output. No notification sent. +# +# emby_session_report.sh --status +# Show Emby URL, API key (masked), and report configuration. Then exit. +# +# emby_session_report.sh --log +# Verbose per-section output during report generation. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# emby_session_report.sh — generate report -# emby_session_report.sh --dry-run — test API connectivity only, no notification -# emby_session_report.sh --log — verbose output -# emby_session_report.sh --status — show config and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Monitors/smart_health.sh b/Monitors/smart_health.sh index 6332048..a8943c6 100644 --- a/Monitors/smart_health.sh +++ b/Monitors/smart_health.sh @@ -1,53 +1,79 @@ #!/bin/bash # ============================================================================================== -# ================================= SMART Health Monitor ======================================= +# ============================= SMART Health Monitor =========================================== # ============================================================================================== -# Checks SMART health attributes for all drives on the system. -# Reads data live from each drive via smartctl — no persistent writes. -# Designed to run weekly as a scheduled report. # -# ── MONITORED ATTRIBUTES ────────────────────────────────────────────────────────────────────── -# Overall SMART status — PASSED/FAILED — immediate fail = drive is dying -# Reallocated_Sector_Ct — bad sectors remapped — any > 0 is concerning -# Current_Pending_Sector — sectors waiting for reallocation — any > 0 is concerning -# Offline_Uncorrectable — sectors that could not be corrected — any > 0 is critical -# Temperature_Celsius — vs thresholds from dynamix.cfg (or master.conf fallback) -# Power_On_Hours — informational — drive age in days +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Drive SMART health monitoring for all drives on the system. Scheduled weekly +# (Sunday 7am). Queries live SMART attributes via smartctl — no persistent writes. # -# ── DRIVE DISCOVERY ─────────────────────────────────────────────────────────────────────────── -# Discovers drives automatically via /dev/sd* and /dev/nvme* — no config needed. -# NVMe drives use different attribute names — detected and handled automatically. -# HOST*_SMART_IGNORE_DRIVES skips specific drives (e.g. boot USB flash drive). +# Monitored per drive: overall SMART status (PASSED/FAILED), Reallocated_Sector_Ct +# (any > 0 is concerning), Current_Pending_Sector (any > 0 is concerning), +# Offline_Uncorrectable (any > 0 is critical), Temperature_Celsius vs thresholds, +# Power_On_Hours (informational). NVMe drives use different attribute names — +# detected and handled automatically. Silent when all drives pass. # -# ── TEMPERATURE THRESHOLDS ──────────────────────────────────────────────────────────────────── -# Reads hot/max/hotssd/maxssd from /boot/config/plugins/dynamix/dynamix.cfg at runtime. -# Uses unRAID's own configured thresholds — no need to duplicate them here. -# Falls back to SMART_TEMP_WARN / SMART_TEMP_CRIT from master.conf if dynamix.cfg not found. +# Temperature thresholds read from /boot/config/plugins/dynamix/dynamix.cfg — +# unRAID's own configured values. Falls back to SMART_TEMP_WARN / SMART_TEMP_CRIT +# from master.conf if dynamix.cfg is not found. # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID and aliases HOST*_SMART_IGNORE_DRIVES → SMART_IGNORE_DRIVES. -# Each server monitors its own drives with its own ignore list. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock — smartctl calls are slow, prevent duplicate runs -# detect_hosts() — correct ignore list per host via MY_ID aliases -# validate_unraid_cmd — smartctl and notify validated before use -# Silent healthy drives — only problems produce output -# Silent healthy run — no notify when all drives pass +# Single Instance Lock +# acquire_lock prevents concurrent runs — smartctl calls are slow. # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── -# HOST*_SMART_IGNORE_DRIVES — drives skipped in SMART monitoring -# Aliased by detect_hosts() — script uses SMART_IGNORE_DRIVES +# Per-Host Ignore List +# detect_hosts() aliases HOST*_SMART_IGNORE_DRIVES → SMART_IGNORE_DRIVES. +# Typically used to skip the boot USB flash drive (no meaningful SMART data). # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# SMART_TEMP_WARN — fallback warn threshold in °C (if dynamix.cfg not found) -# SMART_TEMP_CRIT — fallback crit threshold in °C (if dynamix.cfg not found) +# Automatic Drive Discovery +# Scans /dev/sd* and /dev/nvme* on every run — no drive list to maintain. +# +# Dynamix Temperature Thresholds +# Reads hot/max/hotssd/maxssd from dynamix.cfg so smart_health.sh and unRAID's +# dashboard use the same thresholds. Falls back to master.conf values if not found. +# +# Notifications Validated +# validate_unraid_cmd confirms smartctl and notify script are present before use. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# +# HOST*_SMART_IGNORE_DRIVES +# Drives skipped in SMART monitoring. Aliased by detect_hosts() → +# SMART_IGNORE_DRIVES. Typically includes the boot USB flash drive (sda). +# +# master.conf +# +# SMART_TEMP_WARN +# Fallback warn threshold in °C if dynamix.cfg not found. (default: 45) +# +# SMART_TEMP_CRIT +# Fallback critical threshold in °C if dynamix.cfg not found. (default: 55) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# smart_health.sh +# Query SMART attributes for all drives. Notify on any concerning results. +# Silent when all drives pass. +# +# smart_health.sh --dry-run +# Show which drives would be checked. No smartctl queries, no notifications. +# +# smart_health.sh --status +# Show configured ignore list and temperature thresholds. Then exit. +# +# smart_health.sh --log +# Verbose per-drive attribute output during the run. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# smart_health.sh — normal run -# smart_health.sh --dry-run — show which drives would be checked -# smart_health.sh --log — verbose output -# smart_health.sh --status — show config and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Monitors/system_tuning_monitor.sh b/Monitors/system_tuning_monitor.sh index d169462..07a1f0b 100644 --- a/Monitors/system_tuning_monitor.sh +++ b/Monitors/system_tuning_monitor.sh @@ -2,58 +2,101 @@ # ============================================================================================== # ============================= System Tuning Monitor ========================================== # ============================================================================================== -# Tracks inotify and php-fpm usage over time. -# Snapshots written every 6 hours — read by sunday_morning_coffee_report.sh for weekly summary. -# Schedule: 0 */6 * * * (every 6 hours via User Scripts) # -# ── WHAT IT TRACKS ──────────────────────────────────────────────────────────────────────────── -# inotify instances: -# Current in use vs kernel limit -# % utilization — warns above INOTIFY_WARN_PCT (default 80%) -# Top 5 consumers by instance count -# Symptom of exhaustion: containers miss file events, downloads not detected, -# Live TV stutter, library not updated +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# inotify and PHP-FPM utilisation tracking via time-series snapshots. Scheduled +# every 6 hours (0 */6 * * *). Writes one bounded log entry per run to +# TUNING_MONITOR_LOG. weekly_health_digest.sh reads this log to report peak, +# average, and warning counts over the week. # -# php-fpm workers: -# Active workers vs PHP_MAX_CHILDREN limit -# % utilization — warns above PHP_FPM_WARN_PCT (default 80%) -# Symptom: unRAID WebGUI slowdowns or timeouts under load +# Background snapshot script — no output when healthy. Warns (and notifies) only +# when INOTIFY_WARN_PCT or PHP_FPM_WARN_PCT thresholds are exceeded. # -# ── LOG FORMAT ──────────────────────────────────────────────────────────────────────────────── -# DATE|TIME|INOTIFY_USED|INOTIFY_LIMIT|INOTIFY_PCT|INOTIFY_WARN|PHPFPM_ACTIVE|PHPFPM_MAX|PHPFPM_PCT|PHPFPM_WARN -# Log trimmed to TUNING_LOG_RETENTION days on each write — bounded size. +# inotify exhaustion symptoms: downloads complete but arrs don't detect them, +# live TV stutter, library updates stop. PHP-FPM exhaustion symptoms: unRAID +# WebGUI slowdowns or timeouts under load. # -# ── WHAT THE WEEKLY REPORT SHOWS ────────────────────────────────────────────────────────────── -# inotify: peak, average, warning count over the week -# php-fpm: peak workers, average workers, warning count over the week +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── SILENT BY DEFAULT ───────────────────────────────────────────────────────────────────────── -# Background snapshot script — no output when healthy. -# Warns to stderr when thresholds exceeded — visible in User Scripts output log. -# Does NOT notify on every snapshot — only when threshold exceeded. +# Each run snapshots: +# inotify: instances in use vs INOTIFY_MAX_INSTANCES kernel limit. +# Top 5 consumers by instance count. Warns above INOTIFY_WARN_PCT. +# php-fpm: active workers vs PHP_MAX_CHILDREN limit. +# Warns above PHP_FPM_WARN_PCT. # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# Each server writes to its own DATA_DIR — no collision between servers. -# MY_ID included in warning output for clarity in shared notification channels. +# Log line format (one per run, trimmed to TUNING_LOG_RETENTION days): +# DATE|TIME|INOTIFY_USED|INOTIFY_LIMIT|INOTIFY_PCT|INOTIFY_WARN| +# PHPFPM_ACTIVE|PHPFPM_MAX|PHPFPM_PCT|PHPFPM_WARN +# INOTIFY_WARN and PHPFPM_WARN are 1/0 flags. weekly_health_digest.sh counts +# warnings over the week to show trend severity. # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock — prevents overlapping 6-hour snapshots -# root check — /proc/*/fd requires root access -# atomic log write — tmp file + mv prevents partial writes on trim -# validate_unraid — notify script validated before use +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# INOTIFY_WARN_PCT — warn threshold % (default 80) -# PHP_FPM_WARN_PCT — warn threshold % (default 80) -# PHP_MAX_CHILDREN — max php-fpm workers (set by php_fpm_max_children.sh) -# TUNING_MONITOR_LOG — log file path -# TUNING_LOG_RETENTION — days before old entries purged (default 30) +# Single Instance Lock +# acquire_lock prevents overlapping 6-hour snapshot runs. +# +# Root Enforcement +# /proc/*/fd enumeration for inotify consumer counting requires root access. +# +# Atomic Log Write +# Trim uses tmp file + mv — partial writes during log rotation cannot corrupt +# the accumulated history. +# +# Notification Validated +# validate_unraid_cmd confirms the notify script is present before use. +# +# ============================================================================================== +# STATE FILES +# ============================================================================================== +# +# TUNING_MONITOR_LOG — DATA_DIR/tuning_monitor.db +# One line per 6-hour snapshot. Trimmed to TUNING_LOG_RETENTION days on +# each write — bounded size. Read by weekly_health_digest.sh for trend +# reporting. Resets if DATA_DIR is cleared. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# INOTIFY_WARN_PCT +# Warn if inotify instances exceed this percentage of the kernel limit. (default: 80) +# +# PHP_FPM_WARN_PCT +# Warn if php-fpm active workers exceed this percentage of PHP_MAX_CHILDREN. (default: 80) +# +# PHP_MAX_CHILDREN +# Maximum php-fpm workers — set by php_fpm_max_children.sh in unRAID_Essentials. +# +# TUNING_MONITOR_LOG +# Log file path. (default: DATA_DIR/tuning_monitor.db) +# +# TUNING_LOG_RETENTION +# Days before old entries are purged. (default: 30) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# system_tuning_monitor.sh +# Take snapshot. Write to log. Warn (and notify) if thresholds exceeded. +# Silent when healthy. +# +# system_tuning_monitor.sh --dry-run +# Measure inotify and PHP-FPM utilisation and display results. No log write. +# +# system_tuning_monitor.sh --status +# Show thresholds, log path, and retention. Then exit. +# +# system_tuning_monitor.sh --log +# Verbose output during the snapshot including top inotify consumers. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# system_tuning_monitor.sh — normal snapshot run -# system_tuning_monitor.sh --dry-run — measure and show, no log write -# system_tuning_monitor.sh --log — verbose output -# system_tuning_monitor.sh --status — show config and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Monitors/weekly_health_digest.sh b/Monitors/weekly_health_digest.sh index b882b1c..462e475 100644 --- a/Monitors/weekly_health_digest.sh +++ b/Monitors/weekly_health_digest.sh @@ -1,61 +1,115 @@ #!/bin/bash # ============================================================================================== -# ================================= Health Digest ============================================== +# ============================= Health Digest ================================================== # ============================================================================================== -# Aggregates system health data from across the ecosystem into a single digest report. -# Reads existing state files — no new writes. # -# ── THREE PROFILES ──────────────────────────────────────────────────────────────────────────── +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Full ecosystem health aggregation from existing state files. Scheduled daily +# (8am). DIGEST_PROFILE controls when notifications actually send — the cron +# schedule never changes, only the profile in master.conf. +# +# Reads state files from across the system (watchdog strikes, fallback state, +# skip list, bandwidth history, transcode stats, cert status) and compiles them +# into a single digest. Reads only — writes nothing, changes nothing. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Three profiles — switch by changing DIGEST_PROFILE in master.conf: +# # always — sends every run regardless of findings -# schedule daily for a daily digest +# Use: daily digest of everything, even when healthy # -# smart — sends only if something worth reporting was found -# runs every run but stays silent when all healthy -# DIGEST_SMART_ON_* toggles control what triggers a send +# smart — sends only when something worth reporting was found +# Stays silent on clean days. DIGEST_SMART_ON_* toggles control +# what triggers a send — all independently configurable. # -# weekly — sends once per week on DIGEST_DAY regardless of schedule frequency -# run daily, digest only fires on DIGEST_DAY (default Sunday) +# weekly — sends once per week on DIGEST_DAY (default Sunday), silent all other days +# Use: one weekly summary without daily noise # -# The cron schedule stays the same regardless of profile — change DIGEST_PROFILE in -# master.conf to switch behaviour. No cron changes needed. -# -# ── DATA SOURCES (reads only) ───────────────────────────────────────────────────────────────── -# FALLBACK_STATE_FILE — current fallback state -# SYS_WATCHDOG_FAILED_FILE — container skip list (manual intervention needed) -# WATCHDOG_STATE_FILE — active container watchdog strikes -# SYS_WATCHDOG_STATE_FILE — active system watchdog strikes -# BANDWIDTH_LOG — yesterday's transfer totals -# TRANSCODE_DAILY_LOG — weekly transcode statistics -# CERT_MONITOR_DOMAINS — live SSL cert check via openssl +# Data sources (reads only): +# FALLBACK_STATE_FILE — current fallback state +# SYS_WATCHDOG_FAILED_FILE — container skip list (manual intervention needed) +# WATCHDOG_STATE_FILE — active container watchdog strikes +# SYS_WATCHDOG_STATE_FILE — active system watchdog strikes +# BANDWIDTH_LOG — yesterday's transfer totals +# TRANSCODE_DAILY_LOG — weekly transcode statistics +# CERT_MONITOR_DOMAINS — live SSL cert check via openssl # RAMDISK_PATH / TRANSCODE_LINK — current transcode location and usage # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID and aliases CERT_MONITOR_DOMAINS, RAMDISK_WARN_GB, -# RAMDISK_SIZE, RAMDISK_LOW_GB and all other host-specific vars used in this report. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock — report takes time, prevent duplicate runs -# detect_hosts() — correct vars per host -# validate_unraid_cmd — notify and openssl validated before use -# Per-section guards — missing state file skipped cleanly -# Silent smart profile — completely silent when nothing to report +# Single Instance Lock +# acquire_lock prevents duplicate reports — report generation takes time. +# +# Per-Host Variables +# detect_hosts() aliases CERT_MONITOR_DOMAINS, RAMDISK_WARN_GB, RAMDISK_SIZE, +# RAMDISK_LOW_GB, and all other host-specific vars used in the report. +# +# Per-Section Guards +# Each data source section checks whether its state file exists before reading. +# A missing state file is skipped cleanly — it does not abort the report. +# +# Silent Smart Profile +# smart profile produces no output and no notification when nothing worth +# reporting is found. +# +# Notifications Validated +# validate_unraid_cmd confirms notify and openssl are present before use. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# DIGEST_PROFILE +# Notification frequency: always | smart | weekly. (default: weekly) +# +# DIGEST_DAY +# Day name for weekly profile — must match `date +%A` output. (default: Sunday) +# +# DIGEST_SMART_ON_WATCHDOG +# Send smart profile notification if any active watchdog strikes. (default: true) +# +# DIGEST_SMART_ON_FALLBACK +# Send smart profile notification if fallback state is not NORMAL. (default: true) +# +# DIGEST_SMART_ON_CERT_WARN +# Send smart profile notification if any cert is within CERT_WARN_DAYS. (default: true) +# +# DIGEST_SMART_ON_BANDWIDTH +# Send smart profile notification if any transfer exceeded BANDWIDTH_WARN_GB. (default: true) # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# DIGEST_PROFILE — always | smart | weekly -# DIGEST_DAY — day name for weekly profile (e.g. Sunday) -# DIGEST_SMART_ON_WATCHDOG — send on active watchdog strikes -# DIGEST_SMART_ON_FALLBACK — send on non-NORMAL fallback state -# DIGEST_SMART_ON_CERT_WARN — send on cert warning -# DIGEST_SMART_ON_BANDWIDTH — send on high bandwidth day # CERT_WARN_DAYS / CERT_CRIT_DAYS / CERT_TIMEOUT +# Cert check thresholds — shared with cert_monitor.sh. +# # BANDWIDTH_WARN_GB +# High-transfer threshold — shared with bandwidth_monitor.sh. +# # TRANSCODE_DAILY_LOG +# Path to the transcode statistics log. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# weekly_health_digest.sh +# Run digest. DIGEST_PROFILE determines whether a notification is sent. +# +# weekly_health_digest.sh --dry-run +# Generate and display digest output. No notification sent regardless of profile. +# +# weekly_health_digest.sh --status +# Show profile, day, and smart trigger configuration. Then exit. +# +# weekly_health_digest.sh --log +# Verbose per-section output during digest generation. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# weekly_health_digest.sh — normal run -# weekly_health_digest.sh --dry-run — generate report, no notification -# weekly_health_digest.sh --log — verbose output -# weekly_health_digest.sh --status — show config and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Monitors/zfs_memory_snapshot.sh b/Monitors/zfs_memory_snapshot.sh index 32e6828..b8a7083 100644 --- a/Monitors/zfs_memory_snapshot.sh +++ b/Monitors/zfs_memory_snapshot.sh @@ -1,53 +1,112 @@ #!/bin/bash # ============================================================================================== -# ================================= ZFS Memory Snapshot ======================================== +# ============================= ZFS Memory Snapshot ============================================ # ============================================================================================== -# Weekly ZFS pool health and memory diagnostic report. -# Combines ZFS pool status, ARC statistics, memory summary, Docker memory usage -# and kernel pressure into a single report. Informational only — no action taken. -# system_watchdog.sh handles threshold-based intervention. # -# ── WHAT IT REPORTS ─────────────────────────────────────────────────────────────────────────── -# ZFS pool health — status, state, errors per pool (excluding ignored pools) -# ARC statistics — current size, max, utilization %, metadata pressure -# Memory status — total/free/available RAM vs thresholds -# Docker memory — top ZFS_REPORT_DOCKER_TOP containers by memory usage -# Kernel pressure — vmstat snapshot (3 samples) +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Weekly ZFS pool health and memory diagnostic report. Scheduled Sunday 6am — +# first in the Sunday monitoring block, before other scripts run. Informational +# only — system_watchdog.sh handles threshold-based intervention. # -# ── OUTPUT ──────────────────────────────────────────────────────────────────────────────────── -# Output goes to both console and ZFS_REPORT_LOG for later review. -# In dry-run mode — console only, nothing written to log. -# Notifies if any warning thresholds are exceeded. -# Silent when all healthy — only problems produce output. +# Combines ZFS pool status, ARC statistics, Docker memory usage, and kernel +# memory pressure into a single snapshot. Output goes to both console (for User +# Scripts output log) and ZFS_REPORT_LOG for week-over-week comparison. # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS → ZFS_REPORT_IGNORE_POOLS. -# Each server ignores its own single-disk ZFS array pools — not the peer's. +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock — zpool + docker stats are slow, prevent duplicates -# detect_hosts() — correct pool ignore list per host -# validate_unraid_cmd — notify validated before use -# DOCKER_TIMEOUT — docker stats protected against hung daemon -# ZFS not available — skips pool and ARC sections gracefully -# Docker not available — skips container section gracefully +# Five report sections (each skips gracefully if its data source is unavailable): # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── -# HOST*_ZFS_REPORT_IGNORE_POOLS — pools excluded from health reporting -# Aliased by detect_hosts() — script uses ZFS_REPORT_IGNORE_POOLS +# ZFS pool health — status, state, errors per pool. Pools in +# ZFS_REPORT_IGNORE_POOLS excluded from the report +# (still fully monitored by unRAID — report-only exclusion). +# ARC statistics — current ARC vs max, metadata pressure, hit rate. +# Warns if ARC utilisation exceeds ZFS_REPORT_ARC_WARN_PCT. +# Memory status — total, free, available RAM. +# Warns if free < ZFS_REPORT_FREE_WARN_GB or +# available < ZFS_REPORT_AVAIL_WARN_GB. +# Docker memory — top ZFS_REPORT_DOCKER_TOP containers by memory usage. +# Useful for spotting containers approaching watchdog limits. +# Kernel pressure — vmstat snapshot (3 samples). # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# ZFS_REPORT_LOG — log file path for weekly reports -# ZFS_REPORT_ARC_WARN_PCT — warn if ARC using more than this % of max -# ZFS_REPORT_FREE_WARN_GB — warn if less than this GB free RAM -# ZFS_REPORT_AVAIL_WARN_GB — warn if less than this GB available RAM -# ZFS_REPORT_DOCKER_TOP — how many top Docker containers to show +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Single Instance Lock +# acquire_lock prevents duplicate runs — zpool and docker stats are slow. +# +# Per-Host Pool Ignore List +# detect_hosts() aliases HOST*_ZFS_REPORT_IGNORE_POOLS → ZFS_REPORT_IGNORE_POOLS. +# Single-disk JBOD members excluded from report noise per server. +# +# ZFS Availability Guard +# Skips pool and ARC sections gracefully if ZFS is not available on this server. +# +# Docker Availability Guard +# Skips container memory section gracefully if Docker is not responding. +# +# Docker Stats Timeout +# DOCKER_TIMEOUT caps docker stats calls. A hung daemon does not block the report. +# +# Notification Validated +# validate_unraid_cmd confirms the notify script is present before use. +# +# ============================================================================================== +# STATE FILES +# ============================================================================================== +# +# ZFS_REPORT_LOG — /var/log/zfs-weekly-health.log (tmpfs, resets on reboot) +# Weekly report written here for comparison across runs. Open the log to +# see pool health trend week over week without remembering last week's values. +# In dry-run mode, console only — nothing written. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# +# HOST*_ZFS_REPORT_IGNORE_POOLS +# Pools excluded from health reporting. Single-disk JBOD members generate +# expected high-usage warnings — exclude them to reduce report noise. +# Aliased by detect_hosts() → ZFS_REPORT_IGNORE_POOLS. +# +# master.conf +# +# ZFS_REPORT_LOG +# Log file path for weekly reports. (default: /var/log/zfs-weekly-health.log) +# +# ZFS_REPORT_ARC_WARN_PCT +# Warn if ARC is using more than this percentage of its configured max. (default: 90) +# +# ZFS_REPORT_FREE_WARN_GB +# Warn if free RAM is below this threshold in GB. (default: 10) +# +# ZFS_REPORT_AVAIL_WARN_GB +# Warn if available RAM is below this threshold in GB. (default: 20) +# +# ZFS_REPORT_DOCKER_TOP +# Number of top Docker containers by memory usage to include. (default: 10) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# zfs_memory_snapshot.sh +# Generate report, write to ZFS_REPORT_LOG and console. Notify on warnings. +# +# zfs_memory_snapshot.sh --dry-run +# Generate report to console only. No log write, no notifications. +# +# zfs_memory_snapshot.sh --status +# Show pool ignore list and threshold configuration. Then exit. +# +# zfs_memory_snapshot.sh --log +# Verbose output during report generation. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# zfs_memory_snapshot.sh — normal report (writes to log) -# zfs_memory_snapshot.sh --dry-run — console only, no log write -# zfs_memory_snapshot.sh --log — verbose output -# zfs_memory_snapshot.sh --status — show config and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Partnership/Manual-Partnership.md b/Partnership/Manual-Partnership.md new file mode 100644 index 0000000..30102c8 --- /dev/null +++ b/Partnership/Manual-Partnership.md @@ -0,0 +1,623 @@ +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# 🤝 PARTNERSHIP — Manual +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Config reference, setup procedures, and operational how-tos. +For architecture see README-Partnership.md. For per-script detail see script headers. + +--- + +## ━━━ THE RELATIONSHIP MODEL ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +``` +HOST1 (owner) HOST2 (mirror) +──────────────── ──────────────────────────── +Source of truth Warm copy — always current +Auth stack config Auth containers running +NPM proxy rules NPM serving mirror's domain +LLDAP users LLDAP — same users +Authelia policies Authelia — same policies +Certs Certs — mirrored, valid +Emby Emby — dirty-synced every 15min + +Changes made here ──→ Propagated every 15 minutes +WebUI management ──→ Redirected to HOST1 via Tailscale +Config → git push ──→ Received via git pull on next cycle +``` + +**Mirror's daily experience:** open unRAID Docker UI, click NginxProxyManager, +browser opens HOST1's NPM WebUI automatically via Tailscale. Make a proxy rule +change. 15 minutes later it's live on HOST2. The mirror operator never manages +auth directly — every WebUI redirect is transparent and automatic. + +**Owner's daily experience:** manage auth as normal. Opens NPM → adds a proxy +rule → 15min later live on both servers. Never needs to SSH to HOST2 or think +about HOST2 during normal operation. + +--- + +## ━━━ SETUP PREREQUISITES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +Before `partnership_onboard.sh` can run, five things must be in place: + +### 1. Tailscale Connected on Both Servers + +```bash +# Verify HOST2 is visible from HOST1: +tailscale ip -4 unRAID-Jayred365 # returns HOST2's Tailscale IP + +# Verify reachability: +tailscale ping unRAID-Jayred365 +``` + +Hostnames must match their exact Tailscale device names — enforced already via +`REMOTE_SERVER_NAME` in master.conf. + +### 2. Auth Containers Exist on HOST2 + +```bash +# Containers must exist — they can be stopped: +ssh root@[HOST2-ip] "docker inspect NginxProxyManager --format '{{.State.Status}}'" +# Expected: created, exited, or running — NOT "no such container" +``` + +`partnership_onboard.sh` will stop these before deploying the owner's auth stack. +List them in `HOST2_PARTNERSHIP_REPLACE_CONTAINERS`. + +### 3. XML Templates Exist on HOST1 + +```bash +ls /boot/config/plugins/dockerMan/templates-user/my-NginxProxyManager.xml +ls /boot/config/plugins/dockerMan/templates-user/my-Authelia.xml +# etc — one XML per container listed in HOST1_PARTNERSHIP_AUTH_STACK +``` + +These are copied to HOST2 during onboard. The same XMLs that Unraid's Docker +Manager uses — no extra configuration needed on HOST2 after deploy. + +### 4. Critical-Data rsync Profile Configured + +```bash +# master.conf — must include the auth stack appdata path: +CRITICAL_SYNC_SHARES=( + "/mnt/user/appdata-Fallback/Critical-Data" # auth config, certs, NPM rules +) +``` + +This is what keeps HOST2 current after onboard. Without it, onboard succeeds but +the mirror's auth stack drifts from the owner's within hours. + +### 5. Tailscale API Key (if PARTNERSHIP_REMOVE_TAILSCALE=true) + +```bash +# master.conf +PARTNERSHIP_REMOVE_TAILSCALE=true +TAILSCALE_API_KEY="tskey-api-..." # from tailscale.com/admin/settings/keys +TAILSCALE_TAILNET="example.github" # your tailnet name + +# Required scope: Devices write +``` + +--- + +## ━━━ FULL CONFIGURATION REFERENCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +### master.conf + +```bash +# ── Partnership Gate ─────────────────────────────────────────────────────── +PARTNERSHIP_ENABLED=false # set true once both servers are configured +PARTNERSHIP_OWNER_HOST="HOST1" # "HOST1" or "HOST2" — flips on --transfer + +# ── Tailscale Removal ────────────────────────────────────────────────────── +PARTNERSHIP_REMOVE_TAILSCALE=true +TAILSCALE_API_KEY="" # tskey-api-... from tailscale.com/admin/settings/keys +TAILSCALE_TAILNET="" # your tailnet name (e.g. example.github) + +# ── Timing ───────────────────────────────────────────────────────────────── +PARTNERSHIP_GRACE_HOURS=6 # hours before Tailscale removal after offboard + # backup access also expires at this time +PARTNERSHIP_OFFLINE_THRESHOLD=30 # days unreachable before auto-offboard +PARTNERSHIP_SYNC_INTERVAL=15 # informational — actual schedule in cron + +# ── Transfer Safety ──────────────────────────────────────────────────────── +PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership" +PARTNERSHIP_TRANSFER_STRIKES=3 # consecutive health checks required +PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20 + +# ── Onboard Behaviour ────────────────────────────────────────────────────── +PARTNERSHIP_ONBOARD_VERIFY=true # curl-verify each WebUI after onboard +PARTNERSHIP_ONBOARD_NOTIFY=true # notify both servers on successful onboard +``` + +### master_host1.conf (owner side) + +```bash +# Containers whose WebUI URLs are redirected to owner's Tailscale IP on onboard. +# Restored to localhost on offboard. +# Format: "ContainerName|WebUIPort" +HOST1_PARTNERSHIP_AUTH_WEBUIS=( + "NginxProxyManager|81" + "Lldap-Gmer4Lfe|17170" + "Authelia|9091" + "Authelia-Secondary|9092" +) + +# XML templates pushed to mirror during onboard (auth stack). +# ORDER MATTERS: database dependencies must come before Authelia. +# Mariadb/Redis are health-checked after deploy before continuing. +HOST1_PARTNERSHIP_AUTH_STACK=( + "my-Mariadb-Authelia.xml" + "my-Mariadb-Authelia-Secondary.xml" + "my-Redis-Authelia.xml" + "my-Redis-Authelia-Secondary.xml" + "my-Authelia.xml" + "my-Authelia-Secondary.xml" + "my-NginxProxyManager.xml" + "my-Lldap-Gmer4Lfe.xml" +) + +# XML templates pushed to mirror during onboard (arr stack). +# Leave empty to skip arr stack deploy entirely. +HOST1_PARTNERSHIP_ARR_STACK=( + # "my-Sonarr.xml" + # "my-Radarr.xml" + # "my-Lidarr.xml" + # "my-Prowlarr.xml" +) + +# Paths mirror can collect during grace window after offboard. +HOST1_PARTNERSHIP_MIRROR_BACKUPS=( + # "/mnt/user/appdata-Fallback/Jayred365-Emby" +) + +# Containers parked on this server when partnership is active. +# Stopped on onboard (owner deploys auth stack instead), restarted on offboard. +HOST1_PARTNERSHIP_OWN_CONTAINERS=( + # "Emby" + # "NginxProxyManager" +) +``` + +### master_host2.conf (mirror side) + +```bash +# Containers whose WebUI URLs are redirected on onboard. +# Usually left empty on mirror — owner's AUTH_WEBUIS drives the redirect. +HOST2_PARTNERSHIP_AUTH_WEBUIS=( + # "NginxProxyManager|81" +) + +# Auth containers to stop on this server before owner deploys auth stack. +HOST2_PARTNERSHIP_REPLACE_CONTAINERS=( + "NginxProxyManager" + "Authelia" + "Authelia-Secondary" + "Mariadb-Authelia" + "Mariadb-Authelia-Secondary" + "Redis-Authelia" + "Redis-Authelia-Secondary" + "Lldap" +) + +# Arr containers to stop on this server before owner deploys arr stack. +HOST2_PARTNERSHIP_ARR_REPLACE_CONTAINERS=( + # "Sonarr" + # "Radarr" + # "Lidarr" + # "Prowlarr" +) + +# Containers parked on this server when partnership is active. +# Stopped on onboard, restarted on offboard. +HOST2_PARTNERSHIP_OWN_CONTAINERS=( + # "Emby" + # "NginxProxyManager" +) + +# Emby admin provisioning — owner reads these during onboard. +HOST2_PARTNERSHIP_EMBY_ADMIN_USER="" +HOST2_PARTNERSHIP_EMBY_ADMIN_PASS="" +``` + +--- + +## ━━━ PROCEDURES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +### Establishing a Partnership (First-Time Setup) + +```bash +# Step 1: Run on HOST2 (mirror) first — generates SSH key only +Partnership/partnership_onboard.sh + +# Step 2: Run on HOST1 (owner) — completes setup remotely +Partnership/partnership_onboard.sh --dry-run # review first +Partnership/partnership_onboard.sh + +# Step 3: Verify from either server +Partnership/partnership_manager.sh --status +``` + +HOST2 must run first to generate its SSH key so HOST1 can reach it during Step 2. +If SSH is already configured, use `--skip-ssh` on both sides. + +**Skip flags for partial re-runs** (if something failed midway): +```bash +--skip-ssh # SSH already set up +--skip-auth-stack # auth stack already deployed (skips Steps 3-4) +--skip-arr-stack # arr stack already deployed (skips Steps 5-6) +--skip-arr-sync # arr sync not needed yet (arrs not live on mirror) +``` + +--- + +### Offboarding + +Either server can initiate. Always dry-run first. + +```bash +# From either server — review the sequence +Partnership/partnership_manager.sh --offboard --dry-run + +# Live offboard — 10-second countdown before executing +Partnership/partnership_manager.sh --offboard +``` + +**Owner-initiated** (HOST1 runs `--offboard`): +1. Stop any running rsync +2. Final sync — mirror leaves with current auth config and certs +3. Reconfigure mirror WebUIs → localhost +4. Disable critical rsync (`CRITICAL_RSYNC_ENABLED=false`) +5. Write INACTIVE state on both servers, blocklist mirror +6. Local fallback cleanup — remove partner containers + appdata from HOST1 +7. Restart own stack (HOST1's own parked containers) +8. Remote cleanup — remove auth/arr stack containers + appdata from mirror; remove fallback containers +9. Restart mirror's own stack +10. Revoke Emby admin, SSH key revocation (both directions), Tailscale removal + +**Mirror-initiated** (HOST2 runs `--offboard`): +1. Stop any running rsync +2. Reconfigure own WebUIs → localhost (immediately independent) +3. Remove owner-deployed containers locally (reads owner's auth/arr stack arrays via SSH) +4. Remove local fallback coverage containers +5. Disable critical rsync (`CRITICAL_RSYNC_ENABLED=false`) +6. Revoke own Emby admin account from local Emby instance +7. Restart own stack (HOST2's own parked containers) +8. SSH key revocation, write INACTIVE state locally, push state to HOST1 if reachable + +HOST1 finalises its own side on the next `--check` cycle after seeing HOST2's INACTIVE state. + +**What the mirror leaves with after offboard:** +- Its own parked containers restarted (from `PARTNERSHIP_OWN_CONTAINERS`) +- Own arr media library (arr appdata is in `appdata-Fallback/Arrs_Stack/` and is retained) +- Full git mirror of the ecosystem +- Auth WebUIs pointing to localhost — ready to set up own independent auth + +--- + +### Transferring Ownership + +Transfers auth stack ownership from current owner to mirror. Only the current owner +can initiate. Both servers must be healthy. + +```bash +# Always dry-run first — shows health check results and master.conf changes +Partnership/partnership_manager.sh --transfer --dry-run + +# Live transfer — requires exact confirmation string +Partnership/partnership_manager.sh --transfer --confirm=i-understand-this-transfers-ownership +``` + +Transfer sequence: +1. Display current and future ownership clearly +2. Require exact confirmation string +3. Both servers pass `PARTNERSHIP_TRANSFER_STRIKES` consecutive health checks +4. Final sync in current direction (current owner → current mirror) +5. Reconfigure WebUI templates on both servers +6. Flip `PARTNERSHIP_OWNER_HOST` in master.conf on both servers +7. Write updated state files and notify + +After transfer, HOST2 is the owner. Run `--transfer` from HOST2 to transfer back. + +--- + +### Re-onboarding After Offboard + +Former partners are blocklisted on offboard. Remove from blocklist first: + +```bash +# Check what's on the blocklist +Partnership/partnership_manager.sh --status + +# Remove the block (use exact hostname shown in status) +Partnership/partnership_manager.sh --unblock unRAID-Jayred365 + +# Then re-onboard normally +Partnership/partnership_onboard.sh --dry-run +Partnership/partnership_onboard.sh +``` + +--- + +### Adding a New Container to the Auth Stack + +1. Create the XML template on HOST1 (`/boot/config/plugins/dockerMan/templates-user/my-NewContainer.xml`) +2. Add the XML filename to `HOST1_PARTNERSHIP_AUTH_STACK` in `master_host1.conf` + - If it has a database dependency, put the dep earlier in the array +3. Add the container name to `HOST2_PARTNERSHIP_REPLACE_CONTAINERS` in `master_host2.conf` +4. Re-run the auth stack portion: + ```bash + Partnership/partnership_onboard.sh --skip-ssh --skip-arr-stack --skip-arr-sync + ``` + +--- + +### SSH Key Validation and Re-keying + +```bash +# Check current key state and remote connectivity +Partnership/ssh_setup.sh --status + +# Validate SSH auth (tracks strikes) +Partnership/ssh_setup.sh --validate + +# Regenerate key and re-copy to remote (if key is compromised or expired) +Partnership/ssh_setup.sh --force +``` + +Strike tracking: `SSH_MAX_STRIKES` consecutive auth failures → notify. +Network unreachability (Tailscale down) does not count as a strike. +Counter resets after `SSH_STRIKE_RESET_HRS` of clean connectivity. + +--- + +## ━━━ FLAG REFERENCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +### partnership_onboard.sh + +| Flag | Effect | +|------|--------| +| `--dry-run` | Preview all steps without making changes | +| `--log` | Verbose per-step output | +| `--skip-ssh` | Skip SSH key setup (Step 1) — key already in place | +| `--skip-auth-stack` | Skip auth stop + deploy (Steps 3-4) — already done | +| `--skip-arr-stack` | Skip arr stop + deploy (Steps 5-6) — not needed or already done | +| `--skip-arr-sync` | Skip arr library bootstrap (Step 8) — arrs not live yet | + +### partnership_offboard.sh + +Called automatically by `partnership_manager.sh --offboard`. Can also be run directly. + +| Flag | Effect | +|------|--------| +| `--dry-run` | Preview all steps without making changes | +| `--log` | Verbose per-step output | +| `--reason=` | Tag the offboard reason in state file and blocklist (default: `manual`) | + +### partnership_manager.sh + +| Flag | Effect | +|------|--------| +| `--dry-run` | Show the complete sequence without executing | +| `--log` | Verbose per-operation output | +| `--onboard` | Establish mirror relationship (owner only) | +| `--offboard` | Clean separation (either server) — 10s countdown | +| `--transfer --confirm=...` | Flip ownership (owner only) | +| `--check --remote-seen\|--remote-unseen` | 15-min health check (called by orchestrator) | +| `--status` | Show state files, blocklist, SSH key status | +| `--unblock ` | Remove hostname from blocklist | + +### ssh_setup.sh + +| Flag | Effect | +|------|--------| +| `--force` | Regenerate key even if it exists, re-copy to remote | +| `--validate` | Test SSH auth, track strikes, notify at limit | +| `--status` | Show key path, fingerprint, remote connectivity | +| `--dry-run` | Preview without creating or copying | + +--- + +## ━━━ TROUBLESHOOTING ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +### Auth Stack Deploy Fails on Onboard + +```bash +# Check that XML files exist on HOST1: +ls /boot/config/plugins/dockerMan/templates-user/my-Authelia.xml + +# Check that mirror is reachable via Tailscale: +tailscale ping unRAID-Jayred365 + +# Check that SSH key is working: +Partnership/ssh_setup.sh --validate + +# Re-run auth stack only: +Partnership/partnership_onboard.sh --skip-ssh --skip-arr-stack --skip-arr-sync +``` + +If Mariadb/Redis deployed but Authelia still fails: the health-wait timed out +(60s default). Authelia needs Mariadb to be fully initialized, which can take longer +on first boot. Re-run with `--skip-auth-stack` removed — `docker start` on an +already-created container will retry cleanly. + +--- + +### WebUI Still Pointing to Old IP After Onboard + +```bash +# Check template was updated on HOST2: +ssh root@[HOST2-ip] "grep -i 'tailscale\|[HOST1-IP]' \ + /boot/config/plugins/dockerMan/templates-user/my-NginxProxyManager.xml" + +# Check status output: +Partnership/partnership_manager.sh --status + +# Some containers need a restart to pick up new WebUI URL: +ssh root@[HOST2-ip] "docker restart NginxProxyManager" + +# Verify Tailscale routing: +# From HOST2, should reach HOST1's NPM: +curl http://[HOST1-tailscale-ip]:81 +``` + +--- + +### Transfer Health Checks Failing + +```bash +# Both arrays must be fully started: +ls /mnt/user # should show share directories on both servers + +# Both Docker daemons responding: +docker ps # should return a list, not hang + +# Tailscale connected on both servers: +tailscale status # remote peer should show online + +# Health check attempts before giving up: PARTNERSHIP_TRANSFER_MAX_ATTEMPTS (default 20) +``` + +--- + +### Mirror's Domain Broken After Offboard + +The owner-deployed auth stack has been removed from HOST2 as part of offboard cleanup. +HOST2's own parked containers (from `PARTNERSHIP_OWN_CONTAINERS`) are restarted +automatically, but if HOST2 had no pre-existing auth stack of its own, it needs one set up. + +```bash +# Check what's running: +docker ps + +# Verify own parked containers came back up: +# (listed in HOST2_PARTNERSHIP_OWN_CONTAINERS in master_host2.conf) + +# If you need a fresh auth stack, deploy from HOST2's own XML templates: +docker create ... && docker start NginxProxyManager # etc. + +# Check cert expiry — auto-renewal no longer happens via HOST1: +cert_monitor.sh --dry-run +``` + +If the offboard was unexpected or incomplete, check state first: +```bash +Partnership/partnership_manager.sh --status +``` + +--- + +### State Mismatch Between Servers + +```bash +# Check both state files: +Partnership/partnership_manager.sh --status # shows both sides via SSH + +# If one shows ACTIVE and other INACTIVE: +# The INACTIVE side has already offboarded. +# Run --offboard on the ACTIVE side to sync the state. + +# If HOST2 unreachable: +# HOST2 self-resolves on next --check when reachable. +# HOST1 reads INACTIVE state → finalises from owner side automatically. +``` + +--- + +### Auto-Offboard Triggered Unexpectedly + +```bash +# Check the offline counter: +cat /boot/config/partnership_offline_days.db + +# Extended Tailscale outage may have incremented the counter. +# Check Tailscale peer visibility: +tailscale status + +# If partnership should continue — re-onboard (first unblock the partner): +Partnership/partnership_manager.sh --unblock unRAID-Jayred365 +Partnership/partnership_manager.sh --onboard --dry-run +Partnership/partnership_manager.sh --onboard +``` + +--- + +## ━━━ STATE FILES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +```bash +/boot/config/partnership_HOST1.db # HOST1 writes only +/boot/config/partnership_HOST2.db # HOST2 writes only +/boot/config/partnership_blocklist.db # hostname|timestamp|reason + +# Example state file: +state=ACTIVE +last_updated=2026-05-14 03:00:00 +last_seen_remote=2026-05-14 03:00:00 +owner_host=HOST1 +offline_days=0 +``` + +Each server writes only its own state file. The other server reads via SSH. +State propagates through SSH reads — no rsync, no shared filesystem. +`/boot/config` — survives reboots, available before array starts, minimal flash wear. +flock on all writes — prevents concurrent corruption from overlapping --check cycles. + +--- + +## ━━━ ROLE-BASED ACCESS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +``` + HOST1 (owner) HOST2 (mirror) +--onboard ✅ ❌ ownership is granted, not taken +--offboard ✅ ✅ clean exit available to both parties +--transfer ✅ ❌ owner only — mirror cannot self-promote +--check ✅ ✅ both servers monitor state +--status ✅ ✅ status is always available +--unblock ✅ ✅ either server can clear its own blocklist +``` + +AM_OWNER / AM_MIRROR flags set by `detect_hosts()` from `PARTNERSHIP_OWNER_HOST`. +All routing decisions use these flags — no hostname string comparisons. +Mirror attempting `--onboard` or `--transfer`: blocked with a clear error message. + +--- + +## ━━━ DESIGN NOTES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +**Why HOST1/HOST2 instead of PARTNERSHIP_OWNER/MIRROR?** +HOST1 and HOST2 are already defined in master.conf — SSH keys, Tailscale names, +all connection details. A single `PARTNERSHIP_OWNER_HOST` var flips ownership. +Duplicating as `PARTNERSHIP_OWNER` and `PARTNERSHIP_MIRROR` would require maintaining +identical values in two places. One source of truth for the host definition. + +**Why does transfer require a confirmation string?** +Transfer touches master.conf on both servers, reconfigures WebUIs, and flips sync +direction. A misstep mid-sequence leaves both servers with different auth configs and +no clear source of truth. The confirmation string makes accidental execution impossible, +not just unlikely. + +**Why do Tailscale removal and backup access expire at the same time?** +If the mirror can't reach the owner's server via Tailscale, the backup is also +unreachable. `PARTNERSHIP_GRACE_HOURS` controls both — one var, consistent behaviour. +No misleading "data available for 30 days" when access is gone in 6 hours. + +**Why does either server auto-offboard after 30 days offline?** +30 consecutive days of missed sync cycles means the relationship has effectively ended +regardless of intent. Auto-offboard makes the state official. Each server acts +independently — no coordination required to finalise. + +**Why are auth/arr stack containers removed on offboard?** +The auth stack running on the mirror is the owner's stack — deployed from the owner's +XML templates, managed by the owner. On offboard, the mirror restores its own parked +containers (`PARTNERSHIP_OWN_CONTAINERS`) and sets up independent auth from scratch. +Nothing from the owner's deployment lingers. Appdata is deleted alongside containers +so there's no stale config left behind. Both fallback coverage containers (named +`*-Owner`) and the owner-deployed stack (from `PARTNERSHIP_AUTH_STACK` / `PARTNERSHIP_ARR_STACK`) +are removed — the mirror gets a clean slate. + +**Why does XML ordering matter for the auth stack?** +Authelia requires MariaDB and Redis to be running and accepting connections before it +starts. `deploy_container_from_xml()` starts each container immediately after creating +it. If Authelia is deployed before MariaDB, it will fail to connect and may not recover +automatically. The array order in `HOST1_PARTNERSHIP_AUTH_STACK` is enforced by +convention — database containers first, then applications. diff --git a/Partnership/README-Partnership.md b/Partnership/README-Partnership.md index 6b9d4fd..4541237 100644 --- a/Partnership/README-Partnership.md +++ b/Partnership/README-Partnership.md @@ -2,16 +2,15 @@ # 🤝 PARTNERSHIP # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -**Manages the full lifecycle of a two-server partnership — setup, daily operation, -clean separation, and ownership transfer.** One server owns the shared services. -The other mirrors them and benefits from them. The relationship has a beginning, -an ongoing operation, and an end — and every phase deserves the same engineering -discipline as the rest of the ecosystem. +**Managed lifecycle for a two-server partnership — setup, ongoing operation, +and clean separation.** One server owns the shared services. The other mirrors +them and benefits from them. Every phase of the relationship has the same +engineering discipline as the rest of the ecosystem. -> **This script exists because a clean exit should be as easy as a clean setup.** -> The partnership is not a permanent commitment. `--offboard` works from either server -> at any time. Everything HOST2 needs to run independently is already on HOST2. -> The only thing that stops working immediately is the sync — and that's intentional. +> **This folder exists because a clean exit should be as easy as a clean setup.** +> The partnership is not a permanent commitment. `--offboard` works from either +> server at any time. Everything the mirror needs to run independently is already +> there. The only thing that stops on separation is the sync — and that's intentional. --- @@ -21,925 +20,173 @@ discipline as the rest of the ecosystem. ### 🔴 Two Servers, One Auth Stack, No Clean Way to Share It -The auth stack — NginxProxyManager, LLDAP, Authelia, MariaDB, Redis — runs on HOST1. -HOST2 serves his own domain to his own household. He needs his own auth. But maintaining -two independent auth stacks means double the work: two places to add users, two places to -update proxy rules, two places to renew certs, two configurations that inevitably drift -apart. One change made on HOST1 has to be manually replicated to HOST2 — or it isn't, -and the configurations diverge silently. +The auth stack — NginxProxyManager, LLDAP, Authelia, MariaDB, Redis — runs on +HOST1. HOST2 serves its own domain to its own household. It needs its own auth. +But maintaining two independent auth stacks means double the work: two places to +add users, two places to update proxy rules, two places to renew certs, two +configurations that inevitably drift apart. One change on HOST1 has to be manually +replicated to HOST2 — or it isn't, and the configurations diverge silently. -The real cost isn't the initial setup. It's the maintenance burden that accumulates over -months — every new user, every proxy rule change, every cert renewal, every configuration -update applied in one place and forgotten in the other. +The real cost isn't the initial setup. It's the maintenance burden that accumulates +over months — every new user, every proxy rule change, every config update applied +in one place and forgotten in the other. -The fix: one auth stack with a managed mirror. HOST1 owns the configuration. HOST2 runs -a warm copy that stays current via 15-minute sync. HOST2's operator makes zero auth -management decisions — he clicks an auth container, gets HOST1's WebUI via Tailscale, -changes happen there, propagate to his server in 15 minutes. One place to manage -everything for both households. +**The fix:** one auth stack with a managed mirror. HOST1 owns the configuration. +HOST2 runs a warm copy that stays current via 15-minute sync. HOST2's operator makes +zero auth management decisions — clicking an auth container opens HOST1's WebUI via +Tailscale. Changes happen there, propagate to HOST2 in 15 minutes. One place to +manage everything for both households. --- ### 🔴 No Structure Around the Relationship Itself -Setting up the mirror was a manual process. SSH in, reconfigure container WebUI URLs -one by one, copy auth config, verify connectivity, update state tracking. No defined -sequence. No dry-run capability. No verification that each step worked. If something -went wrong midway, the mirror was in an inconsistent state with no clear way to -understand what had and hadn't been done. +Setting up the mirror was a manual process. SSH in, reconfigure container WebUI +URLs one by one, copy auth config, verify connectivity, update state tracking. No +defined sequence. No dry-run capability. No verification that each step worked. If +something went wrong midway, the mirror was in an inconsistent state with no clear +way to understand what had and hadn't been done. Offboard was worse — it involves +stopping a sync that's been running for months, making a final copy of data, +reconfiguring WebUIs back to local addresses, removing Tailscale access, and +notifying both servers. A manual process with that many steps, taken under pressure, +leaves one or both parties in a bad state. -The same problem on offboard — worse, actually. Because offboard involves stopping a sync -that's been running for months, making a final copy of data, reconfiguring WebUIs back -to local addresses, removing Tailscale access, and notifying both servers. A manual -process with this many steps, taken under pressure at the moment of separation, -is the kind of process that leaves one or both parties in a bad state. - -The fix: `partnership_manage.sh` with explicit modes for each lifecycle phase. Each mode -is a defined sequence. Every step is verified. Dry-run shows exactly what will happen -before anything changes. State files make the current relationship status unambiguous -from either server. +**The fix:** `partnership_manager.sh` with explicit modes for each lifecycle phase. +Each mode is a defined sequence. Every step is verified. Dry-run shows exactly what +will happen before anything changes. State files make the current relationship status +unambiguous from either server. --- ### 🔴 No Safe Way to Check If the Other Server Has Gone Away -After months of operation, HOST2 goes quiet. Maybe a hardware failure. Maybe the -person moved. Maybe a forgotten reboot that never completed. The sync starts failing. -The offline counter increments. But nothing actually happens — the ecosystem just keeps -failing the same sync, incrementing the same counter, sending the same notifications. - +After months of operation, HOST2 goes quiet. The sync starts failing. The offline +counter increments. But nothing actually happens — the ecosystem just keeps failing +the same sync, incrementing the same counter, sending the same notifications. Without a defined threshold and an automated response, "partner gone for 30 days" -looks exactly like "partner gone for 3 years" from the ecosystem's perspective. -Both require manual intervention to clean up. +looks exactly like "partner gone for 3 years." -The fix: `PARTNERSHIP_OFFLINE_THRESHOLD`. After this many days of missed sync cycles, -both servers independently auto-offboard. HOST1 removes HOST2 from Tailscale, disables -critical sync, writes INACTIVE state. HOST2 — if it eventually comes back — reads HOST1's -INACTIVE state and cleans up its own side. The relationship is formally ended from both -sides without anyone needing to be physically present. +**The fix:** `PARTNERSHIP_OFFLINE_THRESHOLD`. After this many days of missed sync +cycles, both servers independently auto-offboard. HOST1 removes HOST2 from Tailscale, +disables critical sync, writes INACTIVE state. HOST2 — if it eventually comes back — +reads HOST1's INACTIVE state and cleans up its own side. The relationship is formally +ended from both sides without anyone needing to be present. --- ### 🔴 Ownership Transfer Had No Safe Path -The arrangement was always intended to be flexible — HOST1 owns the auth stack now, but -circumstances change. HOST2 operator might want to take ownership. Or HOST1 might move -and HOST2's location becomes more appropriate for the primary. Swapping ownership manually -meant reconfiguring WebUIs on both servers, swapping sync direction, updating master.conf -on both servers, and hoping the sequence was correct. +The arrangement was always intended to be flexible — HOST1 owns the auth stack now, +but circumstances change. Swapping ownership manually meant reconfiguring WebUIs on +both servers, swapping sync direction, updating master.conf on both, and hoping the +sequence was correct. A misstep — like flipping sync direction before the final sync +completed — leaves both servers with different auth configurations and no clear source +of truth. -A misstep midway through — like flipping sync direction before the final sync completed — -could leave both servers with different auth configurations and no clear source of truth. - -The fix: `--transfer` with a required confirmation string, a consecutive health check -system, and a strict sequence. The confirmation string is as long and specific as -`--i-know-what-im-doing` in the arr cleanup scripts — you cannot type it accidentally. -Health strikes require both servers to be healthy on multiple consecutive checks before -the transfer begins. The sequence guarantees a final sync in the current direction before -anything is flipped. +**The fix:** `--transfer` with a required confirmation string, a consecutive health +check system, and a strict sequence. The confirmation string cannot be typed +accidentally. Health strikes require both servers to be healthy on multiple +consecutive checks before the transfer begins. A final sync in the current direction +runs before anything is flipped. --- -## ━━━ THE RELATIONSHIP MODEL ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ━━━ WHAT THIS FOLDER DOES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -``` -HOST1 (owner) HOST2 (mirror) -──────────────── ──────────────────────────── -Source of truth Warm copy — always current -Auth stack config Auth containers running -NPM proxy rules NPM serving his domain -LLDAP users LLDAP — same users -Authelia policies Authelia — same policies -Certs Certs — mirrored, valid -Emby Emby — dirty-synced every 15min +**Onboarding** (`partnership_onboard.sh`) — one-time setup run from both servers. +Generates SSH keys, installs the auth stack and arr stack on the mirror from XML +templates, configures WebUI redirects to the owner, and bootstraps the arr library. +Role is detected automatically — no flags needed to declare which side you are. -Changes made here ──→ Propagated every 15 minutes -WebUI management ──→ Redirected to HOST1 via Tailscale -Config → git push ──→ Received via git pull on next cycle -``` +**Offboarding** (`partnership_offboard.sh`) — handles clean separation from either +role. Owner path: final sync, WebUI reconfigure, remote container + appdata cleanup +(auth/arr stack by XML array, fallback coverage by naming), SSH key revocation, +Tailscale removal. Mirror path: local WebUI reconfigure, remove owner-deployed +containers locally, disable sync, revoke Emby admin, SSH key revocation, signal owner. +Called by `partnership_manager.sh --offboard` but runnable directly. + +**Lifecycle management** (`partnership_manager.sh`) — dispatcher and monitor. +- Manually: `--onboard`, `--offboard` (delegates to offboard script), `--transfer`, `--status`, `--unblock` +- Automatically: `--check` called every 15 minutes by `critical_sync_maintenance.sh` + +**SSH management** (`ssh_setup.sh`) — generates the keypair for rsync automation, +installs it on the remote, and tracks auth failures with a configurable strike system. +Called by `partnership_onboard.sh` but runnable independently for validation and +re-keying. --- -### ── What the Mirror's Daily Experience Looks Like ─────────────────────────── +## ━━━ RELATIONSHIP TO OTHER FOLDERS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -```bash -# ───────────────────────────────────────────────────────────────────────────── -# HOST2 operator opens the unRAID Docker UI. -# Clicks NginxProxyManager. -# Browser opens HOST1's NPM WebUI automatically (via Tailscale redirect). -# Makes a proxy rule change. -# 15 minutes later it's live on HOST2. -# -# HOST2 operator never: -# Opens HOST2's NPM directly (WebUI points to HOST1) -# Manages LLDAP users on HOST2 (WebUI points to HOST1) -# Touches Authelia config on HOST2 (WebUI points to HOST1) -# Renews HOST2's certs manually (Critical-Data sync handles it) -# -# HOST2 operator always benefits from: -# Every proxy rule HOST1 adds -# Every user HOST1 adds to LLDAP -# Every security policy HOST1 configures -# Every cert HOST1 renews -# ───────────────────────────────────────────────────────────────────────────── -``` +**Orchestrators/** — `critical_sync_maintenance.sh` calls `partnership_manager.sh --check` +every 15 minutes, passing `--remote-seen` or `--remote-unseen` based on whether the +rsync to the partner succeeded. The rsync outcome is the connectivity signal — no +separate ping needed. + +**Rsync/** — Critical-Data rsync keeps the auth stack appdata current on the mirror +(NPM rules, Authelia config, LLDAP database, certs). Partnership manages the +relationship; Rsync delivers the actual data. On offboard, `rsync_stop.sh --rsync-only` +stops any running rsync before the final sync runs. + +**Media/** — `arr_sync.sh` bootstraps the mirror's arr library during onboard, ensuring +both servers have each other's full library from day one. Ongoing arr sync runs +independently at 4-hour cadence. --- -### ── What the Owner's Daily Experience Looks Like ──────────────────────────── +## ━━━ SCRIPTS IN THIS FOLDER ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -```bash -# ───────────────────────────────────────────────────────────────────────────── -# HOST1 operator manages auth as normal: -# Opens NPM → adds a proxy rule → 15min later live on both servers -# Opens LLDAP → adds a user → 15min later exists on both servers -# Renews a cert → 15min later current on both servers -# -# HOST1 operator never needs to: -# SSH to HOST2 to apply changes -# Coordinate with HOST2 operator on auth changes -# Know what HOST2's auth config currently looks like -# Think about HOST2 at all during normal operation -# ───────────────────────────────────────────────────────────────────────────── -``` +| Script | Role | When It Runs | +|--------|------|--------------| +| `partnership_onboard.sh` | One-time setup — SSH keys, stack deploy, arr bootstrap | Manually, once per server per partnership | +| `partnership_offboard.sh` | Clean separation — both paths, both roles | Via `partnership_manager.sh --offboard`; or directly | +| `partnership_manager.sh` | Dispatcher + monitor — onboard WebUIs, health check, transfer, status | `--check` every 15min; all other modes manually | +| `ssh_setup.sh` | SSH key generation, remote install, auth validation | Called by onboard; manually for re-keying or validation | --- -### ── What Makes the Redirect Transparent ───────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Three things work together to make the auth stack transparent to HOST2: -# -# 1. Auth stack warm on both servers always -# NPM, LLDAP, Authelia, MariaDB, Redis — all running on HOST2 continuously. -# When HOST2's domain gets traffic, auth is ready immediately. -# No failover delay, no cold start. -# -# 2. WebUI URLs pointing to HOST1 -# unRAID stores WebUI URLs in the Docker template. -# partnership_manage.sh --onboard edits these templates on HOST2. -# "NginxProxyManager|81" → URL becomes http://[HOST1-tailscale-ip]:81 -# HOST2 operator clicks the container → gets HOST1's WebUI. -# All changes land on HOST1, propagate to HOST2 automatically. -# -# 3. Tailscale permanent connection -# Both servers permanently connected via Tailscale encrypted mesh. -# The WebUI redirect works because Tailscale IP is always reachable. -# No VPN configuration needed. No open ports. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── The One WebUI That Stays Local ────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Tailscale WebUI is left pointing to HOST2's own local interface. -# When diagnosing connectivity issues between servers you need to see -# HOST2's network state from its own perspective — not HOST1's view. -# It is the one WebUI that genuinely needs to stay local. -# Every other auth container WebUI redirects to HOST1. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -## ━━━ STATE FILES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Each server writes ONLY its own state file. Neither server writes the other's. -# State propagates via SSH reads — no rsync, no shared filesystem. -# -# /boot/config/partnership_HOST1.db ← HOST1 writes only, HOST2 reads via SSH -# /boot/config/partnership_HOST2.db ← HOST2 writes only, HOST1 reads via SSH -# -# /boot/config — survives reboots, available before array starts, minimal flash wear. -# flock on all writes — prevents concurrent corruption from overlapping --check cycles. -# -# Example state file content: -state=ACTIVE -last_updated=2026-04-14 03:00:00 -last_seen_remote=2026-04-14 03:00:00 -owner_host=HOST1 -offline_days=0 -# ───────────────────────────────────────────────────────────────────────────── -``` - -**Why this design enables deferred offboard:** -HOST2 offboards while HOST1 is temporarily unreachable — writes its own state file to -INACTIVE. When HOST1 can next reach HOST2 (next `--check` cycle), it reads the INACTIVE -state and finalises the offboard from its side automatically. No coordination required. -No message passing. Each server acts on what it can observe. - ---- - -## ━━━ THE LIFECYCLE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🟢 --onboard — Establishing the Partnership -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Run from HOST1 (owner) only. Establishes the mirror relationship. After onboard, -HOST2's auth WebUIs redirect to HOST1 and Critical-Data sync begins keeping them current. - -> **Always dry-run first.** Onboard modifies container templates on HOST2 and verifies -> WebUI connectivity. The dry-run shows exactly which templates would be changed and -> what the redirect URLs would become — review before committing. - ---- - -### ── Onboard Sequence ───────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Step 1 — Pre-flight checks -# Both servers healthy (array started, Docker responding) -# Tailscale connected — HOST2 reachable via Tailscale IP -# unRAID version parity — both servers on compatible versions -# HOST2 auth containers exist (may be stopped — that's fine) -# → Any failure: abort, explain what failed, no changes made -# -# Step 2 — Reconfigure HOST2 WebUI templates -# For each container in PARTNERSHIP_AUTH_WEBUIS: -# Find template in /boot/config/plugins/dockerMan/templates-user/ -# Update WebUI URL → http://[HOST1-TAILSCALE-IP]:PORT -# Container restart not needed — template change takes effect on next open -# -# Step 3 — Verify WebUI connectivity (if PARTNERSHIP_ONBOARD_VERIFY=true) -# curl each configured WebUI via HOST1's Tailscale IP -# Confirms Tailscale routing is working and the service is responding -# A redirect that can't be reached is worse than no redirect -# -# Step 4 — Write state files -# Write partnership_HOST1.db=ACTIVE locally -# SSH to HOST2 → push state, write partnership_HOST2.db=ACTIVE -# Reset offline counter on both sides -# -# Step 5 — Notify (if PARTNERSHIP_ONBOARD_NOTIFY=true) -# Notification to both servers: partnership active -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Configuration ──────────────────────────────────────────────────────────── - -```bash -# master_host1.conf (or master_host2.conf for HOST2 side of the relationship) -# ───────────────────────────────────────────────────────────────────────────── -# Containers whose WebUI URLs are reconfigured on onboard. -# Format: "ContainerName|WebUIPort" -# The port here is what gets redirected — matches the container's WebUI port. -# -HOST1_PARTNERSHIP_AUTH_WEBUIS=( - "NginxProxyManager|81" # NPM admin interface — all proxy rules managed here - "Lldap-Gmer4Lfe|17170" # user directory — all user management here - "Authelia|9091" # SSO authentication — policy management here - "Authelia-Secondary|9092" # secondary Authelia instance -) - -# Paths the mirror should collect during the offboard grace window. -# Files here are accessible to HOST2 for PARTNERSHIP_GRACE_HOURS after offboard. -HOST1_PARTNERSHIP_MIRROR_BACKUPS=( - # "/mnt/user/appdata-Failover/Jayred365-Emby" # HOST2's Emby backup on HOST1 -) -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Always dry-run first — shows exact template changes and redirect URLs -# ───────────────────────────────────────────────────────────────────────────── -partnership_manage.sh --onboard --dry-run - -# ───────────────────────────────────────────────────────────────────────────── -# Live onboard — owner (HOST1) only -# ───────────────────────────────────────────────────────────────────────────── -partnership_manage.sh --onboard - -# Verbose — show each pre-flight check and template change -partnership_manage.sh --onboard --log -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🔴 --offboard — Clean Separation -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Either server can initiate offboard. The process differs slightly depending on who -runs it — but both paths end with both servers fully independent. The mirror leaves -with current auth config, working certs, and full control of its own services. - -> **Always dry-run first.** Offboard stops the sync, reconfigures WebUIs, and -> removes Tailscale access. The dry-run shows exactly what would happen, in order, -> before any of it does. - ---- - -### ── Owner-Initiated Offboard (HOST1 runs --offboard) ─────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# 1. 10-second countdown shown — Ctrl-C to abort if you changed your mind -# -# 2. Stop any running rsync -# rsync_stop.sh --rsync-only — stops the rsync but leaves container recovery -# to docker_watchdog.sh. Nothing else is interrupted. -# -# 3. Final sync -# Critical-Data pushed to HOST2 one last time — HOST2 leaves with current state -# Emby failover sync pushed — HOST2 has current watch states -# This is the cleanest possible state for HOST2 to inherit. -# -# 4. Reconfigure HOST2 WebUI templates → localhost -# SSH to HOST2 → update each auth container WebUI URL → http://localhost:PORT -# HOST2 operator now clicks NPM → gets HOST2's own NPM -# HOST2 manages its own auth from this point forward -# -# 5. Disable critical sync -# Write CRITICAL_RSYNC_ENABLED=false to master.conf -# Critical-Data sync stops at next 15-minute cycle -# -# 6. Write state files -# partnership_HOST1.db=INACTIVE -# SSH to HOST2 → partnership_HOST2.db=INACTIVE -# -# 7. Grace period — PARTNERSHIP_GRACE_HOURS (default 6 hours) -# HOST2's data on HOST1 remains accessible for this window -# HOST2 can collect anything it needs via Tailscale during this time -# Both access and Tailscale removal happen at the same moment (step 8) -# -# 8. Remove HOST2 from Tailscale tailnet -# API call to revoke HOST2's device — encrypted connection ends -# Backup access ends at the same time by design — no misleading window -# -# 9. Notify HOST2 — "offboard complete, access removed in Xh" -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Mirror-Initiated Offboard (HOST2 runs --offboard) ────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# HOST2 can offboard at any time, even if HOST1 is temporarily unreachable. -# The deferred finalisation mechanism ensures HOST1 catches up automatically. -# -# 1. Reconfigure own WebUI templates → localhost -# All auth containers now point to HOST2's own services -# HOST2 manages its own auth immediately — no waiting for HOST1 -# -# 2. Write partnership_HOST2.db=INACTIVE locally -# -# 3. SSH to HOST1 (if reachable) → push state file notification -# HOST1 reads INACTIVE → begins owner offboard sequence (steps 3-9 above) -# HOST1 runs final sync → sends current state to HOST2 one more time -# HOST1 handles Tailscale removal and grace period -# -# 4. If HOST1 unreachable — deferred finalisation -# HOST2 has already reconfigured to localhost — fully independent now -# STATE file written — HOST1 will read it on next --check when reachable -# Next time HOST1 runs --check and can reach HOST2: sees INACTIVE → finalises -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── What HOST2 Keeps After Offboard ──────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# What HOST2 leaves with: - Current auth config ✓ final sync was clean — NPM rules, users, policies - Own DDNS + updater ✓ always was his, unaffected by partnership - Own Git mirror ✓ full ecosystem — git pull always ran on HOST2 - Own anime source of truth ✓ Anime_Shows / Anime_Movies — always was his - Auth WebUIs → localhost ✓ managing own auth directly from this point - Full independence ✓ just stop the sync — ecosystem continues - Certs valid ~60-90 days ✓ cert_monitor.sh warns at 30 days — time to renew - -# What HOST1 keeps after offboard: - His hardware ✓ unaffected - His media shares ✓ Movies, Tv_Shows, Music — his source of truth - His services ✓ running on HOST1 as always - His domain ✓ DNS unaffected - -# What HOST1 loses: - Offsite backup ✗ was HOST2's hardware in a different location - Geographic redundancy ✗ cannot be scripted — that was the person -# ───────────────────────────────────────────────────────────────────────────── -# The arrangement was always mutual. Both parties provided something the scripts -# cannot replace — hardware on a different power utility in a different location. -# Everything else can be reconfigured. That cannot. -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Available from either server — always dry-run first -# ───────────────────────────────────────────────────────────────────────────── -partnership_manage.sh --offboard --dry-run # show the complete sequence, no changes - -partnership_manage.sh --offboard # live offboard — 10s countdown before starting -partnership_manage.sh --offboard --log # verbose — show each step as it executes -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🔄 --transfer — Flipping Ownership -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Transfers ownership from HOST1 to HOST2 (or back after a previous transfer). After -transfer, HOST2 becomes the auth source of truth and HOST1 receives syncs. -Only the current owner can initiate transfer. - -> **This is the most significant operation in the lifecycle.** It reconfigures both -> servers, flips sync direction, and changes master.conf permanently on both sides. -> The confirmation string and health strike system exist to prevent accidents. - ---- - -### ── Why This Is Protected ─────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Transfer involves: -# Modifying master.conf on both servers simultaneously -# Flipping sync direction (HOST1 → HOST2 becomes HOST2 → HOST1) -# Reconfiguring WebUI templates on both servers -# Making HOST2 the authoritative source for auth decisions -# -# A partial transfer — if interrupted mid-sequence — leaves both servers with -# different auth configurations and no clear source of truth. The health strike -# system prevents starting a transfer when either server is degraded. -# The confirmation string prevents accidental execution entirely. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Transfer Sequence ─────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# 1. Display current and future ownership clearly -# "Current: HOST1 → Mirror: HOST2" -# "After: HOST2 → Mirror: HOST1" -# No ambiguity about what is being confirmed. -# -# 2. Confirmation string check -# Must type exactly: i-understand-this-transfers-ownership -# Long and specific by design — cannot be typed accidentally. -# -# 3. Health strike system -# Both servers must pass PARTNERSHIP_TRANSFER_STRIKES consecutive health checks. -# Max PARTNERSHIP_TRANSFER_MAX_ATTEMPTS attempts (default 20) before aborting. -# Each strike: check both arrays started, Docker daemons responding, Tailscale connected. -# Protects against transferring during a degraded state. -# -# 4. Final sync in current direction -# Push Current-Data from current owner (HOST1) to current mirror (HOST2). -# New mirror (HOST1) starts with current state from the outgoing owner. -# -# 5. Reconfigure WebUI templates on HOST2 -# HOST2 WebUIs → localhost (HOST2 now manages its own auth directly) -# -# 6. Reconfigure WebUI templates on HOST1 -# HOST1 WebUIs → HOST2's Tailscale IP (HOST1 now redirects to HOST2) -# -# 7. Flip PARTNERSHIP_OWNER_HOST in master.conf on BOTH servers -# "HOST1" → "HOST2" written to both servers via SSH -# -# 8. Write updated state files on both servers -# -# 9. Notify both servers — transfer complete, new roles shown -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── After Transfer ─────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Before transfer: HOST1 = owner, HOST2 = mirror -# After transfer: HOST2 = owner, HOST1 = mirror -# -# HOST2 now: -# All auth changes made here directly -# Critical-Data pushed → to HOST1 every 15 minutes -# NPM, LLDAP, Authelia — HOST2's own instances are authoritative -# HOST1's WebUIs redirect to HOST2 via Tailscale -# -# HOST1 now: -# Receives Critical-Data sync from HOST2 -# Auth WebUIs point to HOST2 — clicking NPM opens HOST2's NPM -# Never makes auth configuration changes directly -# -# To transfer back — run --transfer from HOST2 (the new owner) -# PARTNERSHIP_OWNER_HOST="HOST2" after first transfer, "HOST1" after second -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Dry run — ALWAYS first. Shows the full sequence, health check results, -# and what master.conf changes would be made on both servers. -# ───────────────────────────────────────────────────────────────────────────── -partnership_manage.sh --transfer --dry-run - -# ───────────────────────────────────────────────────────────────────────────── -# Live transfer — owner only — requires exact confirmation string -# ───────────────────────────────────────────────────────────────────────────── -partnership_manage.sh --transfer --confirm=i-understand-this-transfers-ownership -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🔁 --check — Automated State Monitor -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Called automatically by `critical_sync_maintenance.sh` every 15 minutes. Never run -manually. Silent when healthy — produces output only when state changes or thresholds -are hit. - -```bash -# Called internally — do not run manually: -partnership_manage.sh --check --remote-seen # rsync succeeded, remote reachable -partnership_manage.sh --check --remote-unseen # rsync failed, remote unreachable -``` - ---- - -### ── What It Does Each Cycle ───────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# --remote-seen path (rsync succeeded → HOST2 was reachable): -# Reset offline counter to 0 -# Update last_seen_remote timestamp -# SSH to read remote state file -# Both ACTIVE → silent ✅ -# Remote INACTIVE → begin offboard finalisation from owner side -# -# --remote-unseen path (rsync failed → HOST2 unreachable): -# Increment offline counter by 1 -# Counter × 15min = approximate offline duration -# At PARTNERSHIP_OFFLINE_THRESHOLD days: auto-offboard begins -# Notify when counter hits threshold — not on every missed cycle -# -# Why the rsync outcome drives this: -# The rsync that runs before --check provides a reliable connectivity signal. -# If rsync to HOST2 succeeded, HOST2 is reachable — reset counter. -# If rsync failed, HOST2 may be unreachable — increment counter. -# No separate ping needed — the sync attempt is the connectivity test. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Auto-Offboard ──────────────────────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# After this many days of consecutive missed sync cycles, auto-offboard begins. -# Works both directions independently — neither server needs the other to act. -# - PARTNERSHIP_OFFLINE_THRESHOLD=30 # days — each missed cycle = ~15min -# -# HOST1 sees HOST2 offline for 30 days: -# AM_OWNER → full owner-initiated offboard -# Tailscale removal via API (API works regardless of HOST2 reachability) -# Critical sync disabled -# State: INACTIVE -# -# HOST2 sees HOST1 offline for 30 days: -# AM_MIRROR → mirror-initiated offboard -# Reconfigure own WebUIs → localhost -# Write HOST2.db=INACTIVE -# Fully independent immediately -# When HOST1 eventually comes back → reads INACTIVE → cleans its side -# -# 30 days covers: -# Extended hardware failure (people do fix things eventually) -# Someone moving and being offline for weeks -# But not: a permanent departure -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -## ━━━ ROLE-BASED ACCESS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Which server can run which mode: -# - HOST1 (owner) HOST2 (mirror) - --onboard ✅ ❌ ownership is granted, not taken - --offboard ✅ ✅ clean exit available to both parties - --transfer ✅ ❌ owner only — mirror cannot self-promote - --check ✅ ✅ both servers monitor state - --status ✅ ✅ status is always available -# -# AM_OWNER / AM_MIRROR flags set by detect_hosts() from PARTNERSHIP_OWNER_HOST. -# All routing decisions use these flags — no hostname string comparisons. -# Mirror attempting --onboard or --transfer: blocked with a clear error message. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -## ━━━ INTEGRATION WITH CRITICAL SYNC ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Partnership check runs inside critical_sync_maintenance.sh — the 15-minute -# orchestrator. The rsync outcome directly informs the partnership check. -# -# Schedule: */15 * * * * -# -# Execution order in critical_sync_maintenance.sh: -# 1. Critical-Data rsync → auth stack sync -# 2. emby-failover rsync → dirty Emby sync -# 3. partnership --check → state check, informed by rsync results -# → --remote-seen if rsync succeeded (HOST2 reachable) -# → --remote-unseen if rsync failed (HOST2 unreachable) -# -# When critical sync is disabled (CRITICAL_RSYNC_ENABLED=false): -# partnership --check is skipped automatically -# Offline counter does not increment from a sync that wasn't attempted -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -## ━━━ FULL CONFIGURATION REFERENCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── - -# ── Partnership Gate ─────────────────────────────────────────────────────── - PARTNERSHIP_ENABLED=false # set true once both servers are configured - PARTNERSHIP_OWNER_HOST="HOST1" # "HOST1" or "HOST2" — flips on --transfer - -# ── Tailscale Removal ────────────────────────────────────────────────────── - PARTNERSHIP_REMOVE_TAILSCALE=true # remove mirror from tailnet on offboard - TAILSCALE_API_KEY="" # tskey-api-... from tailscale.com/admin/settings/keys - TAILSCALE_TAILNET="" # your tailnet name (e.g. example.github) - # required when PARTNERSHIP_REMOVE_TAILSCALE=true - -# ── Timing ───────────────────────────────────────────────────────────────── - PARTNERSHIP_GRACE_HOURS=6 # hours before Tailscale removal after offboard - # backup access also expires at this time — by design - PARTNERSHIP_OFFLINE_THRESHOLD=30 # days unreachable before auto-offboard (both servers) - PARTNERSHIP_SYNC_INTERVAL=15 # informational — actual schedule in cron - -# ── Transfer Safety ──────────────────────────────────────────────────────── - PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership" - # must type this exactly to proceed — long by design - PARTNERSHIP_TRANSFER_STRIKES=3 # consecutive health checks required before transfer - PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20 # max health attempts before giving up - -# ── Onboard Behaviour ────────────────────────────────────────────────────── - PARTNERSHIP_ONBOARD_VERIFY=true # curl each WebUI after reconfigure to verify routing - PARTNERSHIP_ONBOARD_NOTIFY=true # notify both servers on successful onboard - -# master_host1.conf / master_host2.conf -# ───────────────────────────────────────────────────────────────────────────── - -# Containers whose WebUI URLs are reconfigured on onboard/offboard - HOST1_PARTNERSHIP_AUTH_WEBUIS=( - "NginxProxyManager|81" - "Lldap-Gmer4Lfe|17170" - "Authelia|9091" - "Authelia-Secondary|9092" - ) - -# Paths accessible to HOST2 during grace window after offboard - HOST1_PARTNERSHIP_MIRROR_BACKUPS=( - # "/mnt/user/appdata-Failover/Jayred365-Emby" - ) -``` - ---- - -## ━━━ INITIAL SETUP REQUIREMENTS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Before `--onboard` can run, five things must be in place: - ---- - -### 1. Tailscale Connected on Both Servers - -```bash -# Verify HOST2 is visible from HOST1: -tailscale ip -4 unRAID-Jayred365 # should return HOST2's Tailscale IP - -# Verify HOST1 is visible from HOST2: -tailscale ip -4 unRAID-Gmer4Lfe # should return HOST1's Tailscale IP - -# Verify reachability: -tailscale ping unRAID-Jayred365 -``` - -HOST1 and HOST2 hostnames must match their exact Tailscale device names — already -enforced by the ecosystem since `REMOTE_SERVER_NAME` is used everywhere. - ---- - -### 2. SSH Keys — Passwordless Both Directions - -```bash -# From HOST1 — should print HOST2's hostname without a password prompt: -ssh -i /root/.ssh/Gmer4Lfe-rsync-key root@[HOST2-tailscale-ip] "hostname" - -# From HOST2 — should print HOST1's hostname without a password prompt: -ssh -i /root/.ssh/Jayred365-rsync-key root@[HOST1-tailscale-ip] "hostname" -``` - ---- - -### 3. Auth Containers Exist on HOST2 - -```bash -# Containers must exist — they can be stopped: -ssh root@[HOST2-ip] "docker inspect NginxProxyManager --format '{{.State.Status}}'" -# Expected: created, exited, or running — NOT "no such container" - -# Template files must exist for WebUI reconfiguration: -ssh root@[HOST2-ip] "ls /boot/config/plugins/dockerMan/templates-user/" -``` - ---- - -### 4. Critical-Data rsync Profile Configured - -```bash -# master.conf — CRITICAL_SYNC_SHARES must include the auth stack path: -CRITICAL_SYNC_SHARES=( - "/mnt/user/appdata-Failover/Critical-Data" # auth config — NPM, Authelia, LLDAP, certs -) -# This is what keeps HOST2 current after onboard -``` - ---- - -### 5. Tailscale API Key (if removing from tailnet on offboard) - -```bash -# master.conf - PARTNERSHIP_REMOVE_TAILSCALE=true - TAILSCALE_API_KEY="tskey-api-..." # from tailscale.com/admin/settings/keys - TAILSCALE_TAILNET="example.github" # your tailnet name - -# Required scope: Devices write -# Generate at: https://login.tailscale.com/admin/settings/keys -``` - ---- - -## ━━━ TROUBLESHOOTING ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - ---- - -### 🔴 WebUI Still Pointing to Old IP After Onboard +## ━━━ HOW THE SCRIPTS RELATE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ``` -Check template was found and modified: - partnership_manage.sh --status shows current WebUI redirect state - ssh HOST2 "cat /boot/config/plugins/dockerMan/templates-user/NginxProxyManager.xml" - should show HOST1's Tailscale IP +INITIAL SETUP (run once) +───────────────────────────────────────────────────────────────────────────── +HOST2 (mirror) runs: + partnership_onboard.sh + └─ ssh_setup.sh generates keypair, copies to owner -Some containers need a restart to pick up new WebUI URL: - docker restart NginxProxyManager on HOST2 — picks up template change +HOST1 (owner) runs: + partnership_onboard.sh + ├─ ssh_setup.sh generates keypair, copies to mirror + ├─ [plugin install on mirror] FolderView3 if configured + ├─ [stop mirror auth stack] PARTNERSHIP_REPLACE_CONTAINERS via SSH + ├─ deploy_container_from_xml() pushes auth XMLs to mirror + starts containers + │ └─ wait_for_container_healthy() Mariadb/Redis health-checked before Authelia + ├─ [stop mirror arr stack] PARTNERSHIP_ARR_REPLACE_CONTAINERS via SSH + ├─ deploy_container_from_xml() pushes arr XMLs to mirror + starts containers + ├─ partnership_manager.sh --onboard reconfigures WebUIs, writes ACTIVE state + └─ arr_sync.sh bootstraps full library on both servers -Verify Tailscale routing is working: - curl http://[HOST1-tailscale-ip]:81 from HOST2 — should reach HOST1's NPM +ONGOING OPERATION (every 15min) +───────────────────────────────────────────────────────────────────────────── +critical_sync_maintenance.sh + ├─ Critical-Data rsync keeps auth appdata current on mirror + └─ partnership_manager.sh --check reads state files, tracks offline counter + ├─ --remote-seen path rsync succeeded → reset counter + └─ --remote-unseen path rsync failed → increment counter → auto-offboard at threshold + +OFFBOARD (manual or auto) +───────────────────────────────────────────────────────────────────────────── +partnership_manager.sh --offboard + └─ partnership_offboard.sh (exec'd — holds own lock) + ├─ [owner-initiated] stop rsync → final sync → reconfigure mirror WebUIs + │ → disable critical sync → write INACTIVE state + │ → local fallback cleanup → restart own stack + │ → remote: remove auth/arr stack + fallback containers + appdata + │ → restart mirror stack → Emby revoke → SSH key revocation + │ → Tailscale removal after grace window + └─ [mirror-initiated] stop rsync → reconfigure own WebUIs + → remove owner-deployed containers locally (reads owner's stack arrays) + → remove local fallback containers → disable critical sync + → revoke own Emby admin → restart own stack + → SSH key revocation → write INACTIVE → signal owner ``` - ---- - -### 🔴 Transfer Health Checks Failing - -``` -Both arrays must be fully started: - ls /mnt/user should show share directories on both servers - -Both Docker daemons responding: - docker ps should return a list, not hang - -Tailscale connected on both servers: - tailscale status should show the remote peer as online - -Max attempts before giving up: PARTNERSHIP_TRANSFER_MAX_ATTEMPTS (default 20) -Each attempt waits for next health window — may take several minutes total -``` - ---- - -### 🔴 Mirror's Domain Broken After Offboard - -``` -Auth WebUIs reconfigured to localhost — mirror manages auth directly now. -The containers are running and functional — they just no longer redirect to HOST1. - -Verify auth containers running: - docker ps | grep -E "NPM|Authelia|LLDAP" - -Check cert expiry — certs won't auto-renew anymore via HOST1: - cert_monitor.sh --dry-run shows days remaining on each domain's cert - -Set up cert auto-renewal on HOST2 independently if not already configured. -``` - ---- - -### 🔴 State Mismatch Between Servers - -``` -Run --status on both servers to see both state files: - partnership_manage.sh --status on HOST1 - partnership_manage.sh --status on HOST2 - -If one shows ACTIVE and other shows INACTIVE: - The INACTIVE side has already offboarded - Run --offboard on the ACTIVE side to sync the state - -If HOST2 unreachable: - HOST2 will self-resolve on next --check when it becomes reachable - HOST1 reads HOST2's INACTIVE state → finalises from owner side automatically -``` - ---- - -### 🔴 Auto-Offboard Triggered Unexpectedly - -``` -Check the offline counter: - cat /boot/config/partnership_offline_days.db - -Extended Tailscale connectivity issue may have incremented the counter: - tailscale status verify HOST2 appears in HOST1's peer list - -If partnership should continue — re-onboard: - partnership_manage.sh --onboard --dry-run # verify setup is correct - partnership_manage.sh --onboard # re-establish partnership - -Counter resets to 0 on successful --onboard. -``` - ---- - -## ━━━ DESIGN NOTES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -``` -Why HOST1/HOST2 instead of PARTNERSHIP_OWNER/MIRROR? - HOST1 and HOST2 are already defined in master.conf — SSH keys, Tailscale names, - all connection details. A single PARTNERSHIP_OWNER_HOST var flips ownership. - Duplicating everything as PARTNERSHIP_OWNER and PARTNERSHIP_MIRROR would require - maintaining identical values in two places. One source of truth for the host - definition, one var to flip direction. - -Why does transfer require a confirmation string? - Same philosophy as --i-know-what-im-doing in the arr cleanup scripts. - Transfer touches master.conf on both servers, reconfigures WebUIs, and flips - sync direction. The confirmation string makes accidental execution impossible, - not just unlikely. - -Why do Tailscale removal and backup access expire at the same time? - If the mirror can't reach the owner's server via Tailscale, the backup is also - unreachable. PARTNERSHIP_GRACE_HOURS controls both — one var, consistent behaviour, - no misleading "data available for 30 days" when access is gone in 6 hours. - -Why does either server auto-offboard after 30 days offline? - 30 consecutive days of missed sync cycles means the relationship has effectively - ended regardless of intent. Auto-offboard makes the state official. Each server - acts independently — they don't need to coordinate to offboard. - -Why does WebUI reconfiguration use the unRAID template system? - unRAID stores WebUI URLs in /boot/config/plugins/dockerMan/templates-user/. - Editing templates persists across container restarts and array reboots. - Direct Docker label manipulation is lost on next container recreation. -``` \ No newline at end of file diff --git a/Partnership/partnership_manager.sh b/Partnership/partnership_manager.sh index 7fb6c05..e7c7182 100644 --- a/Partnership/partnership_manager.sh +++ b/Partnership/partnership_manager.sh @@ -1,93 +1,187 @@ #!/bin/bash # ============================================================================================== -# ================================= Partnership Manager ======================================== +# ============================= Partnership Manager ============================================ # ============================================================================================== -# Manages the relationship lifecycle between two unRAID servers. -# HOST1 is always the owner (source of truth). HOST2 is always the mirror. +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Manages the full lifecycle of a two-server partnership — onboard, daily health +# monitoring, offboard, and ownership transfer. Called by partnership_onboard.sh +# during initial setup, and by critical_sync_maintenance.sh every 15 minutes for +# the --check mode. All other modes are run manually. +# # PARTNERSHIP_OWNER_HOST flips to "HOST2" after a successful --transfer. +# AM_OWNER / AM_MIRROR flags (set by detect_hosts) control all routing — +# no hostname string comparisons anywhere in this script. # -# ── MODES ───────────────────────────────────────────────────────────────────────────────────── -# --onboard ← owner only — establish mirror relationship -# Reconfigures HOST2 auth WebUIs → HOST1 Tailscale IP -# HOST2 clicks NPM → gets HOST1's NPM automatically +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# --offboard ← either server — clean separation -# Mirror-initiated: writes INACTIVE state, reconfigures own WebUIs → localhost -# Owner finalises on next --check: final sync, Tailscale removal -# Owner-initiated: final sync, reconfigures mirror WebUIs, Tailscale removal -# Both leave with current state, clean exit ✅ +# --onboard (owner only) +# Reconfigures mirror auth WebUIs → owner's Tailscale IP +# Mirror operator clicks NPM → gets owner's NPM via Tailscale automatically +# Writes ACTIVE state on both servers # -# --transfer ← owner only — flip ownership between servers -# Requires confirmation string + consecutive health strikes -# Reconfigures both servers, flips PARTNERSHIP_OWNER_HOST in master.conf +# --offboard (either server) +# Mirror-initiated: reconfigures own WebUIs → localhost, writes INACTIVE state +# Owner finalises on next --check: final sync, Tailscale removal +# Owner-initiated: final sync, reconfigures mirror WebUIs → localhost, removes +# mirror containers + appdata, SSH key revocation, Tailscale removal +# Both leave with clean state ✅ # -# --check ← called by critical_sync_maintenance.sh every 15min -# Reads both state files via SSH -# Detects offboard requests → finalises from owner side -# Updates last_seen_remote timestamp -# Increments offline counter → auto-offboards after threshold -# Silent when healthy ← never noisy on clean runs +# --transfer (owner only) +# Reconfigures both servers, flips PARTNERSHIP_OWNER_HOST in master.conf +# Requires confirmation string + consecutive health check passes # -# --status ← either server — show current state, both sides +# --check (called every 15min by critical_sync_maintenance.sh) +# --remote-seen: rsync succeeded → reset offline counter, read remote state +# --remote-unseen: rsync failed → increment offline counter → auto-offboard at threshold +# Silent when healthy — never noisy on clean runs # -# ── STATE FILES ─────────────────────────────────────────────────────────────────────────────── -# On /boot/config — survives reboots, available before array starts: -# /boot/config/partnership_HOST1.db ← HOST1 writes only -# /boot/config/partnership_HOST2.db ← HOST2 writes only -# Propagated via SSH — no rsync needed +# --status (either server) +# Show state files from both servers, blocklist, SSH key status # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# Root check — all operations require root -# validate_unraid_cmd — notify validated before use -# Role enforcement — mirror cannot run owner-only modes -# AM_OWNER / AM_MIRROR — all routing via these flags, not hostname strings -# version parity — onboard checks both servers match unRAID version -# remote docker daemon — onboard verifies remote daemon responsive -# SSH_TIMEOUT — all SSH calls timeout-protected -# flock on state writes — prevents concurrent state file corruption -# SIGTERM trap — grace period sleep interruptible -# Silent by default — only warns/errors produce output (--check is always silent healthy) +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── -# HOST*_PARTNERSHIP_AUTH_WEBUIS — containers reconfigured on onboard/offboard -# HOST*_PARTNERSHIP_MIRROR_BACKUPS — paths available after offboard -# All aliased by detect_hosts() — script uses PARTNERSHIP_AUTH_WEBUIS etc. +# Deferred offboard +# Either server can offboard without the other being reachable. The initiating +# server writes its state immediately and becomes independent. The other server +# reads the INACTIVE state on its next --check and finalises automatically. +# No message passing, no coordination required. # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# PARTNERSHIP_ENABLED — global enable gate -# PARTNERSHIP_OWNER_HOST — "HOST1" or "HOST2" — flips on --transfer -# PARTNERSHIP_REMOVE_TAILSCALE — remove mirror from tailnet on offboard -# PARTNERSHIP_GRACE_HOURS — hours before Tailscale removal after offboard -# PARTNERSHIP_OFFLINE_THRESHOLD — days unreachable before auto-offboard -# PARTNERSHIP_TRANSFER_CONFIRM — exact string required for --transfer -# PARTNERSHIP_TRANSFER_STRIKES — consecutive health checks required -# PARTNERSHIP_TRANSFER_MAX_ATTEMPTS — max attempts before giving up -# PARTNERSHIP_ONBOARD_VERIFY — verify WebUI connectivity after onboard -# PARTNERSHIP_ONBOARD_NOTIFY — notify both servers on onboard completion -# PARTNERSHIP_SYNC_INTERVAL — informational — actual schedule in cron -# TAILSCALE_API_KEY / TAILSCALE_TAILNET — required when PARTNERSHIP_REMOVE_TAILSCALE=true +# Appdata cleanup on offboard +# Partner containers are stopped and removed. Their appdata bind-mount paths +# (collected via docker inspect before removal) are also deleted. Safety gate: +# only paths matching /mnt/*/appdata* are deleted — media and config shares +# outside the appdata tree are never touched. # -# ── BLOCKLIST ───────────────────────────────────────────────────────────────────────────────── -# After offboard, the former partner's hostname is written to: -# /boot/config/partnership_blocklist.db (format: hostname|timestamp|reason) +# Blocklist is application-layer; Tailscale removal is network-layer +# Both happen on offboard. The blocklist prevents re-onboard until explicitly +# cleared with --unblock. Tailscale removal ends encrypted access at the network +# level. Grace period controls both simultaneously — one var, consistent behaviour. # -# --onboard is hard-blocked if the remote is on the blocklist — exits with error. -# --check silently skips remote state reads for blocklisted hosts (no noise every 15min). -# --unblock removes an entry to permit re-onboarding. -# --status shows the full blocklist. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# The blocklist persists until explicitly cleared — surviving reboots, array restarts, -# and Tailscale reconnections. Tailscale removal is a separate step at the network layer; -# the blocklist is the application-layer guard. +# Root check +# All operations require root. +# +# Role enforcement +# Mirror cannot run --onboard or --transfer. Blocked with a clear error message. +# +# Version parity check +# --onboard verifies both servers are on compatible unRAID versions. +# +# SSH_TIMEOUT on all remote calls +# Every ssh/scp call is timeout-protected. No operation hangs on an unreachable peer. +# +# flock on state writes +# Prevents concurrent state file corruption from overlapping --check cycles. +# +# SIGTERM trap on grace period sleep +# Offboard grace period is interruptible — Ctrl-C aborts cleanly. +# +# Silent by default +# --check produces no output when both servers are healthy. Only state changes +# and threshold crossings produce output. +# +# ============================================================================================== +# STATE FILES +# ============================================================================================== +# +# /boot/config/partnership_HOST1.db — HOST1 writes, HOST2 reads via SSH +# /boot/config/partnership_HOST2.db — HOST2 writes, HOST1 reads via SSH +# /boot/config/partnership_blocklist.db — hostname|timestamp|reason, persists until cleared +# +# On /boot/config — survives reboots, available before array starts, minimal flash wear. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# PARTNERSHIP_ENABLED +# Global gate — set true once both servers are configured (default: false) +# +# PARTNERSHIP_OWNER_HOST +# "HOST1" or "HOST2" — flips on --transfer (default: "HOST1") +# +# PARTNERSHIP_REMOVE_TAILSCALE +# Remove mirror from tailnet on offboard (default: true) +# +# PARTNERSHIP_GRACE_HOURS +# Hours before Tailscale removal after offboard — backup access also expires then (default: 6) +# +# PARTNERSHIP_OFFLINE_THRESHOLD +# Days of missed sync cycles before auto-offboard triggers (default: 30) +# +# PARTNERSHIP_TRANSFER_CONFIRM +# Exact string required for --transfer (default: "i-understand-this-transfers-ownership") +# +# PARTNERSHIP_TRANSFER_STRIKES +# Consecutive health checks required before transfer proceeds (default: 3) +# +# PARTNERSHIP_TRANSFER_MAX_ATTEMPTS +# Max health check attempts before giving up (default: 20) +# +# PARTNERSHIP_ONBOARD_VERIFY +# Curl-verify each WebUI after reconfigure to confirm Tailscale routing works (default: true) +# +# PARTNERSHIP_ONBOARD_NOTIFY +# Notify both servers on successful onboard (default: true) +# +# PARTNERSHIP_SYNC_INTERVAL +# Informational — actual schedule is in cron (default: 15) +# +# TAILSCALE_API_KEY / TAILSCALE_TAILNET +# Required when PARTNERSHIP_REMOVE_TAILSCALE=true +# +# master_host*.conf +# +# HOST*_PARTNERSHIP_AUTH_WEBUIS +# Containers reconfigured on onboard/offboard. Format: "ContainerName|WebUIPort" +# Aliased by detect_hosts() → PARTNERSHIP_AUTH_WEBUIS +# +# HOST*_PARTNERSHIP_MIRROR_BACKUPS +# Paths accessible to the partner during grace window after offboard. +# Aliased by detect_hosts() → PARTNERSHIP_MIRROR_BACKUPS +# +# HOST*_PARTNERSHIP_OWN_CONTAINERS +# Containers parked here during partnership, restarted on offboard. +# Aliased by detect_hosts() → PARTNERSHIP_OWN_CONTAINERS +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# partnership_manager.sh --onboard +# Establish mirror relationship — owner only +# +# partnership_manager.sh --offboard +# Clean separation — either server. 10-second countdown before executing. +# +# partnership_manager.sh --offboard --dry-run +# Show the complete offboard sequence without executing +# +# partnership_manager.sh --transfer --confirm=i-understand-this-transfers-ownership +# Flip ownership — owner only. Requires exact confirmation string. +# +# partnership_manager.sh --check --remote-seen|--remote-unseen +# Called by critical_sync_maintenance.sh every 15min — do not run manually +# +# partnership_manager.sh --status +# Show state files, blocklist, SSH key status from both servers +# +# partnership_manager.sh --unblock +# Remove hostname from blocklist to permit re-onboarding +# +# Any mode supports --dry-run and --log # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# partnership_manage.sh --onboard -# partnership_manage.sh --offboard -# partnership_manage.sh --transfer --confirm=i-understand-this-transfers-ownership -# partnership_manage.sh --check --remote-seen|--remote-unseen -# partnership_manage.sh --status -# partnership_manage.sh --unblock -# Any mode supports --dry-run and --log # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -152,10 +246,12 @@ MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" ) OWNER="${!OWNER_ID}" # hostname string MIRROR="${!MIRROR_ID}" -OWNER_SSH_KEY_VAR="${OWNER_ID}_SSH_KEY" -MIRROR_SSH_KEY_VAR="${MIRROR_ID}_SSH_KEY" -OWNER_SSH_KEY="${!OWNER_SSH_KEY_VAR}" -MIRROR_SSH_KEY="${!MIRROR_SSH_KEY_VAR}" +# SSH_KEY (set by detect_hosts) is this server's own private key. +# The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh. +# With sparse checkout, each server only has its own master_host{N}.conf — the other server's +# key path is never available here. Use SSH_KEY for all outbound SSH regardless of mode. +MIRROR_SSH_KEY="$SSH_KEY" +OWNER_SSH_KEY="$SSH_KEY" AM_OWNER=false AM_MIRROR=false @@ -169,34 +265,36 @@ OWNER_STATE_FILE="/boot/config/partnership_${OWNER}.db" MIRROR_STATE_FILE="/boot/config/partnership_${MIRROR}.db" OFFLINE_COUNTER="/boot/config/partnership_offline_days.db" -if [[ -z "$MODE" ]]; then - error "No mode specified" - echo "Usage:" - echo " partnership_manage.sh --onboard" - echo " partnership_manage.sh --offboard" - echo " partnership_manage.sh --transfer --confirm=..." - echo " partnership_manage.sh --check --remote-seen|--remote-unseen" - echo " partnership_manage.sh --status" - echo " partnership_manage.sh --unblock " - exit 1 +if [[ "${PARTNERSHIP_LIB_MODE:-}" != "1" ]]; then + if [[ -z "$MODE" ]]; then + error "No mode specified" + echo "Usage:" + echo " partnership_manager.sh --onboard" + echo " partnership_manager.sh --offboard" + echo " partnership_manager.sh --transfer --confirm=..." + echo " partnership_manager.sh --check --remote-seen|--remote-unseen" + echo " partnership_manager.sh --status" + echo " partnership_manager.sh --unblock " + exit 1 + fi + + # Role-based access control + if [[ "$AM_MIRROR" == true ]]; then + case "$MODE" in + onboard|transfer) + error "Only the owner ($OWNER / $OWNER_ID) can run --$MODE" + error "Run from $OWNER or use --offboard to separate cleanly" + exit 1 + ;; + esac + fi + + # Lock for all modes except --check (frequent) and --offboard (offboard script holds its own) + [[ "$MODE" != "check" && "$MODE" != "offboard" ]] && acquire_lock "strict" + + [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" fi -# Role-based access control -if [[ "$AM_MIRROR" == true ]]; then - case "$MODE" in - onboard|transfer) - error "Only the owner ($OWNER / $OWNER_ID) can run --$MODE" - error "Run from $OWNER or use --offboard to separate cleanly" - exit 1 - ;; - esac -fi - -# Lock for all modes except --check (check is called frequently, lock would pile up) -[[ "$MODE" != "check" ]] && acquire_lock "strict" - -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" - # ============================================================================================== # ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -678,19 +776,39 @@ start_own_stack() { done } -# Remove partnership containers on this server. +# Remove partnership containers on this server + their appdata bind-mount paths. # Uses FolderView3 folder if enabled (precise list), else falls back to FALLBACK_*_COVERS_* config. +# Appdata paths collected via docker inspect BEFORE removal — inspect fails on removed containers. +# Safety gate: only paths matching /mnt/*/appdata* are deleted. cleanup_partner_containers() { local folder_name="$1" + declare -a containers=() + gather_partner_fallback_containers containers + + if [[ ${#containers[@]} -eq 0 ]]; then + log "No partner containers found to remove" + if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then + folderview3_remove_partner_folder "$folder_name" + fi + return 0 + fi + + # Collect appdata paths BEFORE any removal — inspect fails once container is gone + local all_appdata_paths="" + for container in "${containers[@]}"; do + [[ -z "$container" ]] && continue + if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$container" >/dev/null 2>&1; then + local cpaths + cpaths=$(docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' \ + "$container" 2>/dev/null | awk -F: '{print $1}' | grep '^/mnt/.*/appdata') + [[ -n "$cpaths" ]] && all_appdata_paths+=$'\n'"$cpaths" + fi + done + + # Remove containers if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then folderview3_remove_partner_folder "$folder_name" else - declare -a containers=() - gather_partner_fallback_containers containers - if [[ ${#containers[@]} -eq 0 ]]; then - log "No partner containers found to remove" - return 0 - fi for container in "${containers[@]}"; do [[ -z "$container" ]] && continue if [[ "$DRY_RUN" == true ]]; then @@ -706,9 +824,22 @@ cleanup_partner_containers() { fi done fi + + # Delete appdata after containers are gone + while IFS= read -r path; do + [[ -z "$path" ]] && continue + if [[ "$DRY_RUN" == true ]]; then + warn " DRY RUN — would rm -rf $path" + continue + fi + rm -rf "$path" && log " Appdata removed: $path ✅" || warn " Failed to remove: $path" + done <<< "$all_appdata_paths" } -# SSH to mirror — remove all containers named *-${OWNER_SHORT} (owner's deployed containers). +# SSH to mirror — remove all containers named *-${OWNER_SHORT} (owner's deployed containers) +# and their appdata bind-mount paths. +# Appdata paths collected via SSH docker inspect before removal, then deleted via SSH. +# Safety gate: only paths matching /mnt/*/appdata* are deleted on the remote. cleanup_owner_containers_on_mirror() { local mirror_ip="$1" local owner_short @@ -716,7 +847,7 @@ cleanup_owner_containers_on_mirror() { log "Removing owner-deployed containers from $MIRROR..." if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would remove *-${owner_short} containers from $MIRROR" + warn "DRY RUN — would remove *-${owner_short} containers + appdata from $MIRROR" return 0 fi @@ -732,6 +863,14 @@ cleanup_owner_containers_on_mirror() { while IFS= read -r container; do [[ -z "$container" ]] && continue + + # Collect appdata paths before removal + local appdata_paths + appdata_paths=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \ + "docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$container' 2>/dev/null \ + | awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null) + timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \ "docker stop '$container' >/dev/null 2>&1 @@ -739,6 +878,16 @@ cleanup_owner_containers_on_mirror() { grep -q removed && \ log "$container removed from $MIRROR ✅" || \ warn "Failed to remove $container from $MIRROR" + + # Delete appdata on remote after container removal + while IFS= read -r path; do + [[ -z "$path" ]] && continue + timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \ + "rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \ + log " Appdata removed on $MIRROR: $path ✅" || \ + warn " Failed to remove appdata on $MIRROR: $path" + done <<< "$appdata_paths" done <<< "$container_list" } @@ -983,6 +1132,11 @@ update_master_conf() { fi } +# ── Library-mode guard — source only, skip all mode dispatch ───────────────────────────────── +# partnership_offboard.sh sources this file with PARTNERSHIP_LIB_MODE=1 to get helper +# functions without triggering any mode execution. +[[ "${PARTNERSHIP_LIB_MODE:-}" == "1" ]] && { return 0 2>/dev/null || exit 0; } + # ============================================================================================== # ━━━ Unblock ━━━ # ============================================================================================== @@ -1166,8 +1320,8 @@ if [[ "$MODE" == "check" ]]; then REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE") if [[ -z "$REMOTE_CONTENT" ]]; then # IP resolved but SSH returned nothing — could be auth failure, not just missing file - if [[ -f "$SCRIPT_DIR/../Initial_run/ssh_setup.sh" ]]; then - bash "$SCRIPT_DIR/../Initial_run/ssh_setup.sh" --validate 2>/dev/null || true + if [[ -f "$SCRIPT_DIR/ssh_setup.sh" ]]; then + bash "$SCRIPT_DIR/ssh_setup.sh" --validate 2>/dev/null || true fi log "Partnership check — remote state file not found" exit 0 @@ -1383,229 +1537,7 @@ fi # ━━━ Offboard ━━━ # ============================================================================================== if [[ "$MODE" == "offboard" ]]; then - echo "" - echo "━━━ $ICON_FALLBACK Offboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━" - - # Check already offboarded - if [[ -f "$LOCAL_STATE_FILE" ]]; then - CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state") - if [[ "$CURRENT_STATE" == "INACTIVE" ]]; then - warn "Partnership already INACTIVE — use --status to verify both servers agree" - exit 0 - fi - fi - - # ── Mirror-initiated offboard ───────────────────────────────────────────────────────────── - if [[ "$AM_MIRROR" == true ]]; then - warn "$MIRROR_ID ($MIRROR) is initiating offboard" - warn "Local auth WebUIs will be reconfigured → localhost" - warn "$OWNER_ID ($OWNER) will finalise on its next --check cycle" - - if [[ "$DRY_RUN" == false ]]; then - echo "" - echo "You have 10 seconds to cancel (Ctrl+C)..." - sleep 10 - fi - - echo "" - echo "━━━ $ICON_CONTAINERS Reconfigure Local WebUIs → localhost ━━━" - reconfigure_local_webuis "localhost" - - # Remove owner's deployed containers + restart own stack - echo "" - echo "━━━ $ICON_CONTAINERS Cleanup Partner Containers ━━━" - PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$OWNER") - cleanup_partner_containers "$PARTNER_FOLDER_NAME" - start_own_stack - - NOW=$(date '+%Y-%m-%d %H:%M:%S') - write_state_file "$LOCAL_STATE_FILE" \ - "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON" - log "Local state: INACTIVE ✅" - - [[ "$DRY_RUN" == false ]] && add_to_blocklist "$OWNER" "$REASON" - - OWNER_IP=$(resolve_tailscale_ip "$OWNER") - if [[ -n "$OWNER_IP" ]]; then - push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$MIRROR_SSH_KEY" - notify "Partnership offboard requested by $MIRROR — $OWNER will finalise on next check" \ - "Partnership" "normal" - else - warn "$OWNER unreachable — state written locally, owner will see it when reachable" - fi - - echo "" - echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY ━━━━━" - echo " Your WebUIs: reconfigured → localhost ✅" - echo " State: INACTIVE ✅" - echo " Blocklist: $OWNER blocked — re-onboard to permit access again ✅" - echo " Owner: will finalise + final sync on next --check ✅" - echo "━━━━━━━━━━━━━━━━━━━━━━━" - exit 0 - fi - - # ── Owner-initiated offboard ────────────────────────────────────────────────────────────── - warn "Offboarding $MIRROR_ID ($MIRROR) from partnership" - warn "Final sync will run — mirror leaves with current state" - - if [[ "$DRY_RUN" == false ]]; then - echo "" - echo "You have 10 seconds to cancel (Ctrl+C)..." - sleep 10 - echo "Proceeding..." - fi - - resolve_remote_ip - - # Stop any running rsync first - echo "" - echo "━━━ $ICON_STOP Stop Running Rsync ━━━" - bash "$SCRIPT_DIR/../Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true - - # Final sync - echo "" - echo "━━━ $ICON_SYNC Final Sync ━━━" - do_final_sync - - MIRROR_IP=$(resolve_tailscale_ip "$MIRROR") - MIRROR_REACHABLE=false - [[ -n "$MIRROR_IP" ]] && MIRROR_REACHABLE=true - - # Reconfigure mirror WebUIs → localhost - echo "" - echo "━━━ $ICON_CONTAINERS Reconfigure Mirror WebUIs → localhost ━━━" - - WEBUI_FAILURES=0 - if [[ "$MIRROR_REACHABLE" == true ]]; then - for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do - [[ -z "$entry" ]] && continue - container="${entry%%|*}" - port="${entry##*|}" - reconfigure_webui "$container" "$port" "localhost" \ - "$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ )) - done - else - warn "$MIRROR unreachable — WebUI reconfiguration skipped" - warn "$MIRROR will reconfigure its own WebUIs when it sees INACTIVE state on --check" - (( WEBUI_FAILURES++ )) - fi - - # Disable critical rsync - echo "" - echo "━━━ $ICON_GEAR Disable Critical Sync ━━━" - if [[ "$DRY_RUN" == false ]]; then - update_master_conf "CRITICAL_RSYNC_ENABLED" "false" - warn "CRITICAL_RSYNC_ENABLED=false ✅" - else - warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false" - fi - - # Write state files - echo "" - echo "━━━ $ICON_GEAR Write State ━━━" - NOW=$(date '+%Y-%m-%d %H:%M:%S') - - write_state_file "$LOCAL_STATE_FILE" \ - "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON" - log "Local state: INACTIVE ✅" - - [[ "$DRY_RUN" == false ]] && add_to_blocklist "$MIRROR" "$REASON" - - if [[ "$MIRROR_REACHABLE" == true ]]; then - push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY" - fi - - # Local: remove fallback-coverage containers for mirror, restart own stack - echo "" - echo "━━━ $ICON_CONTAINERS Local Container Cleanup ━━━" - PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$MIRROR") - cleanup_partner_containers "$PARTNER_FOLDER_NAME" - start_own_stack - - # Remote: remove owner's deployed containers from mirror, restart mirror's own stack - echo "" - echo "━━━ $ICON_CONTAINERS Remote Container Cleanup ━━━" - if [[ "$MIRROR_REACHABLE" == true ]]; then - cleanup_owner_containers_on_mirror "$MIRROR_IP" - start_mirror_own_stack "$MIRROR_IP" - else - warn "$MIRROR unreachable — remote container cleanup skipped" - warn "Run 'partnership_manager.sh --offboard' on $MIRROR to clean up manually" - fi - - # Emby admin revocation — before SSH key revocation while Emby still reachable - [[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP" - - # SSH key revocation — mutual, both directions - # Must run before Tailscale removal (SSH needs network) and after state is pushed - SSH_REVOKE_REMOTE_OK=false - SSH_REVOKE_LOCAL_OK=false - do_ssh_key_revocation "${MIRROR_IP:-}" - - # Tailscale removal - if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then - echo "" - echo "━━━ $ICON_NET Tailscale Separation ━━━" - if [[ "$MIRROR_REACHABLE" == true ]]; then - grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-6} * 3600 )) - warn "Waiting ${PARTNERSHIP_GRACE_HOURS:-6}hr grace — mirror can collect backups..." - if [[ "$DRY_RUN" == false ]]; then - trap 'warn "Offboard interrupted during grace sleep"; exit 0' SIGTERM SIGINT - sleep "$grace_seconds" - trap - SIGTERM SIGINT - fi - fi - remove_tailscale_device "$MIRROR" - fi - - # Backup notification - if [[ ${#PARTNERSHIP_MIRROR_BACKUPS[@]} -gt 0 ]]; then - echo "" - echo "━━━ $ICON_DISK Backup Handover ━━━" - log "Backups available for $MIRROR:" - for path in "${PARTNERSHIP_MIRROR_BACKUPS[@]}"; do - [[ -z "$path" ]] && continue - echo " $path" - done - notify "$MIRROR offboard complete — backups available for ${PARTNERSHIP_GRACE_HOURS:-6}hr. Tailscale access expires then." \ - "Partnership" "warning" - fi - - # Summary - echo "" - echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY ━━━━━" - echo " Owner: $OWNER_ID ($OWNER)" - echo " Mirror: $MIRROR_ID ($MIRROR)" - echo " Final sync: complete ✅" - echo " WebUI failures: $WEBUI_FAILURES" - echo " Critical rsync: disabled ✅" - echo " State: INACTIVE ✅" - echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅" - _revoke_status() { - if [[ "${SSH_REVOKE_REMOTE_OK:-false}" == true ]] && [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then - echo "both directions ✅" - elif [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then - echo "local only ✅ — remote failed (revoke manually on $MIRROR)" - else - echo "⚠️ failed — check warnings above" - fi - } - echo " Keys revoked: $(_revoke_status)" - [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]] && \ - echo " FolderView3: ${PARTNER_FOLDER_NAME:-} cleaned ✅" - [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]] && \ - echo " Tailscale: $MIRROR removed ✅" - echo "" - echo " $MIRROR leaves with:" - echo " ✓ Current auth config (final sync)" - echo " ✓ Auth WebUIs → localhost" - echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-6}hr to collect backups" - echo " ✓ Full ecosystem — just stop the sync" - echo "" - [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \ - warn "$ICON_DONE DONE — clean separation complete ✅" - echo "━━━━━━━━━━━━━━━━━━━━━━━" - exit 0 + exec bash "$SCRIPT_DIR/partnership_offboard.sh" "${FILTERED_ARGS[@]}" --reason="$REASON" fi # ============================================================================================== diff --git a/Partnership/partnership_offboard.sh b/Partnership/partnership_offboard.sh new file mode 100755 index 0000000..0073493 --- /dev/null +++ b/Partnership/partnership_offboard.sh @@ -0,0 +1,678 @@ +#!/bin/bash +# ============================================================================================== +# ============================= Partnership Offboard =========================================== +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Cleanly ends a partnership. Role is detected automatically — run on either server. +# Owner path runs the full sequence including remote cleanup and final sync. +# Mirror path handles the local side and signals the owner to complete its own cleanup. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# OWNER PATH (10 steps) +# Step 1: Stop rsync — halt any running sync before state changes +# Step 2: Final sync — mirror leaves with current Critical-Data state +# Step 3: Reconfigure WebUIs — mirror's auth WebUIs → localhost +# Step 4: Disable sync — CRITICAL_RSYNC_ENABLED=false in master.conf +# Step 5: Write state — INACTIVE locally + pushed to mirror, mirror blocklisted +# Step 6: Local cleanup — remove fallback coverage containers + appdata +# Step 7: Restart own stack — bring up owner's own parked containers +# Step 8: Remote cleanup — remove auth/arr stack + fallback containers from mirror +# Step 9: Restart mirror — bring up mirror's own parked containers +# Step 10: Revocation — Emby admin, SSH keys, Tailscale device +# +# MIRROR PATH (8 steps) +# Step 1: Stop rsync — halt any running sync +# Step 2: Reconfigure WebUIs — local auth WebUIs → localhost +# Step 3: Remote stack clean — remove owner-deployed containers locally (auth/arr stack) +# Step 4: Fallback cleanup — remove fallback coverage containers +# Step 5: Disable sync — CRITICAL_RSYNC_ENABLED=false in master.conf +# Step 6: Revoke Emby admin — remove own admin account from local Emby instance +# Step 7: Restart own stack — bring up own parked containers +# Step 8: SSH revocation — revoke keys both directions, write state, signal owner +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# +# HOST*_PARTNERSHIP_AUTH_STACK +# Auth container XMLs to push during onboard — used on offboard to identify what +# to remove. Owner's PARTNERSHIP_AUTH_STACK determines which containers get removed +# from the mirror on both owner-initiated and mirror-initiated offboard. +# +# HOST*_PARTNERSHIP_ARR_STACK +# Arr container XMLs — same cleanup logic as auth stack. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# Partnership/partnership_offboard.sh +# Full offboard — role detected automatically +# +# Partnership/partnership_offboard.sh --dry-run +# Preview all steps without executing +# +# Partnership/partnership_offboard.sh --log +# Verbose per-step output +# +# Partnership/partnership_offboard.sh --reason= +# Tag the offboard reason in state file and blocklist (default: manual) +# Called by partnership_manager.sh --offboard (reason passed through) +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPTS_ROOT="$SCRIPT_DIR/.." +TEMPLATES_DIR="/boot/config/plugins/dockerMan/templates-user" +SSH_TIMEOUT=15 + +source "$SCRIPTS_ROOT/load_config.sh" + +# ── Parse flags ─────────────────────────────────────────────────────────────────────────────── +REASON="manual" +FILTERED_ARGS=() + +for arg in "$@"; do + case "$arg" in + --reason=*) REASON="${arg#--reason=}" ;; + *) FILTERED_ARGS+=("$arg") ;; + esac +done + +parse_args "${FILTERED_ARGS[@]}" + +# ── Source partnership_manager.sh for shared helpers ────────────────────────────────────────── +# PARTNERSHIP_LIB_MODE=1 skips mode dispatch — functions are defined, nothing is executed. +PARTNERSHIP_LIB_MODE=1 source "$SCRIPT_DIR/partnership_manager.sh" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; } + +detect_hosts + +OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}" +MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" ) +OWNER="${!OWNER_ID}" +MIRROR="${!MIRROR_ID}" +MIRROR_SSH_KEY="$SSH_KEY" +OWNER_SSH_KEY="$SSH_KEY" + +AM_OWNER=false +AM_MIRROR=false +[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true +[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true + +LOCAL_STATE_FILE="/boot/config/partnership_${LOCAL_SERVER_NAME}.db" +REMOTE_STATE_FILE="/boot/config/partnership_${REMOTE_SERVER_NAME}.db" +OWNER_STATE_FILE="/boot/config/partnership_${OWNER}.db" +MIRROR_STATE_FILE="/boot/config/partnership_${MIRROR}.db" +OFFLINE_COUNTER="/boot/config/partnership_offline_days.db" + +acquire_lock "strict" + +# Check already offboarded +if [[ -f "$LOCAL_STATE_FILE" ]]; then + CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state") + if [[ "$CURRENT_STATE" == "INACTIVE" ]]; then + warn "Partnership already INACTIVE — use partnership_manager.sh --status to verify both servers agree" + exit 0 + fi +fi + +START=$(date +%s) + +echo "" +echo "━━━ $ICON_FALLBACK Partnership Offboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "" +echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )" +echo " This: $MY_ID ($LOCAL_SERVER_NAME)" +echo " Partner: $( [[ "$AM_OWNER" == true ]] && echo "$MIRROR_ID ($MIRROR)" || echo "$OWNER_ID ($OWNER)" )" +echo " Reason: $REASON" +echo "" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made" + +# ============================================================================================== +# ── HELPER: remove owner-deployed containers from a remote host ─────────────────────────────── +# +# Uses PARTNERSHIP_AUTH_STACK + PARTNERSHIP_ARR_STACK arrays (owner's conf) to derive +# container names from local XML templates. SSHes to remote to stop, remove, and delete +# appdata. Appdata paths are collected via docker inspect before removal so they aren't +# lost once the container is gone. Safety gate: only /mnt/*/appdata* paths are deleted. +# ============================================================================================== +cleanup_deployed_stack_on_remote() { + local remote_ip="$1" ssh_key="$2" + local -a xml_names=() + [[ ${#PARTNERSHIP_AUTH_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_AUTH_STACK[@]}") + [[ ${#PARTNERSHIP_ARR_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_ARR_STACK[@]}") + + if [[ ${#xml_names[@]} -eq 0 ]]; then + log "No auth/arr stack arrays configured — skipping deployed stack cleanup" + return 0 + fi + + log "Removing owner-deployed containers (auth/arr stacks) from $MIRROR..." + for xml_name in "${xml_names[@]}"; do + [[ -z "$xml_name" ]] && continue + local xml_file="${TEMPLATES_DIR}/${xml_name}" + if [[ ! -f "$xml_file" ]]; then + warn " $xml_name not found in local $TEMPLATES_DIR — skipping" + continue + fi + + local cname + cname=$(awk 'match($0,/([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file") + [[ -z "$cname" ]] && continue + + if [[ "$DRY_RUN" == true ]]; then + warn " DRY RUN — would stop + rm $cname on $MIRROR" + warn " DRY RUN — would delete appdata for $cname on $MIRROR" + continue + fi + + # Collect appdata paths via docker inspect before removal + local appdata_paths + appdata_paths=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \ + "docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$cname' 2>/dev/null \ + | awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null) + + timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ + -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \ + "docker stop '$cname' >/dev/null 2>&1 + docker rm '$cname' >/dev/null 2>&1 && echo removed" 2>/dev/null | \ + grep -q removed && \ + log " $cname removed from $MIRROR ✅" || \ + log " $cname not found on $MIRROR — skipping" + + while IFS= read -r path; do + [[ -z "$path" ]] && continue + timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ + -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \ + "rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \ + log " Appdata removed on $MIRROR: $path ✅" || \ + warn " Failed to remove appdata on $MIRROR: $path" + done <<< "$appdata_paths" + done +} + +# ============================================================================================== +# ── HELPER: remove owner-deployed containers locally (mirror-initiated offboard) ───────────── +# +# SSHes to owner to read PARTNERSHIP_AUTH_STACK + PARTNERSHIP_ARR_STACK, then uses the +# local templates-user/ copies (SCPed there during onboard) to get container names and +# appdata paths. Appdata collected before removal. Skips gracefully if owner unreachable. +# ============================================================================================== +cleanup_deployed_stack_locally() { + local owner_ip="$1" ssh_key="$2" + local -a xml_names=() + + if [[ -n "$owner_ip" ]]; then + local -a auth_arr arr_arr + mapfile -t auth_arr < <(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$owner_ip" \ + "source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null + detect_hosts 2>/dev/null + printf '%s\n' \"\${PARTNERSHIP_AUTH_STACK[@]:-}\"" 2>/dev/null | grep -v '^$') + mapfile -t arr_arr < <(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$owner_ip" \ + "source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null + detect_hosts 2>/dev/null + printf '%s\n' \"\${PARTNERSHIP_ARR_STACK[@]:-}\"" 2>/dev/null | grep -v '^$') + xml_names=("${auth_arr[@]}" "${arr_arr[@]}") + fi + + if [[ ${#xml_names[@]} -eq 0 ]]; then + log "Could not read deployed stack from owner — skipping auth/arr cleanup" + return 0 + fi + + log "Removing owner-deployed containers (auth/arr stacks) locally..." + for xml_name in "${xml_names[@]}"; do + [[ -z "$xml_name" ]] && continue + local xml_file="${TEMPLATES_DIR}/${xml_name}" + if [[ ! -f "$xml_file" ]]; then + warn " $xml_name not found locally — skipping" + continue + fi + + local cname + cname=$(awk 'match($0,/([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file") + [[ -z "$cname" ]] && continue + + if [[ "$DRY_RUN" == true ]]; then + warn " DRY RUN — would stop + rm $cname" + warn " DRY RUN — would delete appdata for $cname" + continue + fi + + local appdata_paths="" + if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$cname" >/dev/null 2>&1; then + appdata_paths=$(docker inspect \ + --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' \ + "$cname" 2>/dev/null | awk -F: '{print $1}' | grep '^/mnt/.*/appdata') + timeout "${DOCKER_TIMEOUT:-30}" docker stop "$cname" >/dev/null 2>&1 || true + timeout "${DOCKER_TIMEOUT:-30}" docker rm "$cname" >/dev/null 2>&1 && \ + log " $cname removed ✅" || warn " $cname rm failed" + else + log " $cname not found locally — skipping" + fi + + while IFS= read -r path; do + [[ -z "$path" ]] && continue + rm -rf "$path" && log " Appdata removed: $path ✅" || warn " Failed to remove: $path" + done <<< "$appdata_paths" + done +} + +# ============================================================================================== +# ── HELPER: revoke own admin account from local Emby instance ──────────────────────────────── +# +# Mirror-initiated path only. Called before start_own_stack so Emby is still running. +# Uses local EMBY_API_KEY and the mirror's own short name as the username to delete. +# ============================================================================================== +revoke_local_emby_admin() { + local emby_port="${PARTNERSHIP_EMBY_PORT:-8096}" + local emby_url="http://127.0.0.1:${emby_port}" + + echo "" + echo "━━━ $ICON_EMBY Emby Admin Revocation ━━━" + + if [[ "${PARTNERSHIP_PROVISION_EMBY_ADMIN:-false}" != true ]]; then + log "PARTNERSHIP_PROVISION_EMBY_ADMIN=false — skipping" + return 0 + fi + + if [[ -z "${EMBY_API_KEY:-}" ]]; then + warn "EMBY_API_KEY not set — skipping local Emby admin revocation" + return 1 + fi + + # The account to revoke is this server's own short name (the mirror user's account) + local username="${PARTNERSHIP_EMBY_ADMIN_USER:-$(derive_short_name "$LOCAL_SERVER_NAME")}" + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would delete Emby admin '$username' at $emby_url" + return 0 + fi + + log "Looking up Emby user '$username' at $emby_url..." + + local users_json user_id + users_json=$(curl -sf --max-time 15 \ + -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \ + "${emby_url}/Users" 2>/dev/null) + + user_id=$(echo "$users_json" | \ + grep -o "\"Id\":\"[^\"]*\"[^}]*\"Name\":\"${username}\"" | \ + grep -o '"Id":"[^"]*"' | cut -d'"' -f4 | head -1) + + if [[ -z "$user_id" ]]; then + warn "Emby user '$username' not found at $emby_url — may already be removed" + return 0 + fi + + local del_code + del_code=$(curl -sf --max-time 15 -w "%{http_code}" -o /dev/null \ + -X DELETE \ + -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \ + "${emby_url}/Users/${user_id}" 2>/dev/null) + + if [[ "$del_code" == "200" ]] || [[ "$del_code" == "204" ]] || [[ "$del_code" == "404" ]]; then + log "Emby admin '$username' removed ✅" + else + warn "Failed to delete Emby user '$username' (HTTP $del_code) — remove manually" + fi +} + +# ============================================================================================== +# ── MIRROR PATH ─────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +if [[ "$AM_MIRROR" == true ]]; then + warn "$MIRROR_ID ($MIRROR) is initiating offboard" + warn "Owner ($OWNER) will see INACTIVE state on its next --check cycle and finalize" + + if [[ "$DRY_RUN" == false ]]; then + echo "" + echo "You have 10 seconds to cancel (Ctrl+C)..." + sleep 10 + fi + + OWNER_IP=$(resolve_tailscale_ip "$OWNER") + OWNER_REACHABLE=false + [[ -n "$OWNER_IP" ]] && OWNER_REACHABLE=true + + STEP_STOP_RSYNC_OK=true + STEP_WEBUI_OK=true + STEP_STACK_CLEANUP_OK=true + STEP_FALLBACK_CLEANUP_OK=true + STEP_DISABLE_RSYNC_OK=true + STEP_EMBY_OK=true + SSH_REVOKE_REMOTE_OK=false + SSH_REVOKE_LOCAL_OK=false + + # ── Step 1: Stop rsync ──────────────────────────────────────────────────────────────────── + echo "" + echo "━━━ $ICON_STOP Step 1/8 — Stop Rsync ━━━" + + if [[ "$DRY_RUN" == false ]]; then + bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true + log "Rsync stopped ✅" + else + warn "DRY RUN — would stop rsync" + fi + + # ── Step 2: Reconfigure local WebUIs → localhost ────────────────────────────────────────── + echo "" + echo "━━━ $ICON_CONTAINERS Step 2/8 — Reconfigure Local WebUIs → localhost ━━━" + + reconfigure_local_webuis "localhost" || STEP_WEBUI_OK=false + + # ── Step 3: Remove owner-deployed containers (auth/arr stack) locally ───────────────────── + echo "" + echo "━━━ $ICON_CONTAINERS Step 3/8 — Remove Owner-Deployed Containers ━━━" + + if [[ "$OWNER_REACHABLE" == true ]]; then + cleanup_deployed_stack_locally "$OWNER_IP" "$OWNER_SSH_KEY" || STEP_STACK_CLEANUP_OK=false + else + warn "Owner unreachable — cannot read deployed stack list" + warn "Auth/arr containers will remain — remove manually or re-run when owner is reachable" + STEP_STACK_CLEANUP_OK=false + fi + + # ── Step 4: Remove fallback coverage containers ─────────────────────────────────────────── + echo "" + echo "━━━ $ICON_CONTAINERS Step 4/8 — Fallback Container Cleanup ━━━" + + PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$OWNER") + cleanup_partner_containers "$PARTNER_FOLDER_NAME" || STEP_FALLBACK_CLEANUP_OK=false + + # ── Step 5: Disable critical sync ───────────────────────────────────────────────────────── + echo "" + echo "━━━ $ICON_GEAR Step 5/8 — Disable Critical Sync ━━━" + + if [[ "$DRY_RUN" == false ]]; then + update_master_conf "CRITICAL_RSYNC_ENABLED" "false" && \ + warn "CRITICAL_RSYNC_ENABLED=false ✅" || \ + { warn "Failed to update CRITICAL_RSYNC_ENABLED"; STEP_DISABLE_RSYNC_OK=false; } + else + warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false" + fi + + # ── Step 6: Revoke Emby admin locally ───────────────────────────────────────────────────── + revoke_local_emby_admin || STEP_EMBY_OK=false + + # ── Step 7: Restart own stack ───────────────────────────────────────────────────────────── + start_own_stack + + # ── Step 8: SSH key revocation, write state, signal owner ──────────────────────────────── + echo "" + echo "━━━ $ICON_SHIELD Step 8/8 — SSH Revocation + State ━━━" + + do_ssh_key_revocation "${OWNER_IP:-}" + + NOW=$(date '+%Y-%m-%d %H:%M:%S') + if [[ "$DRY_RUN" == false ]]; then + write_state_file "$LOCAL_STATE_FILE" \ + "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON" + log "Local state: INACTIVE ✅" + add_to_blocklist "$OWNER" "$REASON" + else + warn "DRY RUN — would write INACTIVE state and blocklist $OWNER" + fi + + if [[ "$OWNER_REACHABLE" == true ]]; then + push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$OWNER_SSH_KEY" + notify "Partnership offboard requested by $MIRROR — $OWNER will finalise on next check" \ + "Partnership" "normal" + else + warn "$OWNER unreachable — state written locally, owner will see it when reachable" + fi + + # ── Summary ─────────────────────────────────────────────────────────────────────────────── + END=$(date +%s) + echo "" + echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY (Mirror) ━━━━━" + echo " Mirror: $MY_ID ($LOCAL_SERVER_NAME)" + echo " Owner: $OWNER_ID ($OWNER)" + echo " Reason: $REASON" + echo " Duration: $(format_duration $(( END - START )))" + echo "" + _ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; } + _skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; } + _revoke_status() { + if [[ "${SSH_REVOKE_REMOTE_OK:-false}" == true && "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then + echo "both directions ✅" + elif [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then + echo "local only ✅ — remote failed (revoke manually on $OWNER)" + else + echo "⚠️ failed — check warnings above" + fi + } + + echo " Step 1 — Stop rsync: $(_ok "$STEP_STOP_RSYNC_OK")" + echo " Step 2 — WebUIs: $(_ok "$STEP_WEBUI_OK")" + echo " Step 3 — Stack cleanup: $(_ok "$STEP_STACK_CLEANUP_OK")" + echo " Step 4 — Fallback cleanup: $(_ok "$STEP_FALLBACK_CLEANUP_OK")" + echo " Step 5 — Disable sync: $(_ok "$STEP_DISABLE_RSYNC_OK")" + echo " Step 6 — Emby revoke: $(_ok "$STEP_EMBY_OK")" + echo " Step 7 — Own stack: started" + echo " Step 8 — Keys revoked: $(_revoke_status)" + echo "" + echo " State: INACTIVE ✅" + echo " Blocklist: $OWNER blocked ✅" + echo " Owner: will finalise + final sync on next --check" + echo "" + [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \ + warn "$ICON_DONE DONE — mirror separation complete ✅" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ── OWNER PATH ──────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +warn "Offboarding $MIRROR_ID ($MIRROR) from partnership" +warn "Final sync will run — mirror leaves with current state" + +if [[ "$DRY_RUN" == false ]]; then + echo "" + echo "You have 10 seconds to cancel (Ctrl+C)..." + sleep 10 + echo "Proceeding..." +fi + +WEBUI_FAILURES=0 +SSH_REVOKE_REMOTE_OK=false +SSH_REVOKE_LOCAL_OK=false + +# ── Step 1: Stop rsync ──────────────────────────────────────────────────────────────────────── +echo "" +echo "━━━ $ICON_STOP Step 1/10 — Stop Rsync ━━━" + +if [[ "$DRY_RUN" == false ]]; then + bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true + log "Rsync stopped ✅" +else + warn "DRY RUN — would stop rsync" +fi + +# ── Step 2: Final sync ──────────────────────────────────────────────────────────────────────── +echo "" +echo "━━━ $ICON_SYNC Step 2/10 — Final Sync ━━━" + +do_final_sync + +# ── Step 3: Reconfigure mirror WebUIs → localhost ───────────────────────────────────────────── +echo "" +echo "━━━ $ICON_CONTAINERS Step 3/10 — Reconfigure Mirror WebUIs → localhost ━━━" + +MIRROR_IP=$(resolve_tailscale_ip "$MIRROR") +MIRROR_REACHABLE=false +[[ -n "$MIRROR_IP" ]] && MIRROR_REACHABLE=true + +if [[ "$MIRROR_REACHABLE" == true ]]; then + for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do + [[ -z "$entry" ]] && continue + container="${entry%%|*}" + port="${entry##*|}" + reconfigure_webui "$container" "$port" "localhost" \ + "$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ )) + done +else + warn "$MIRROR unreachable — WebUI reconfiguration skipped" + warn "$MIRROR will reconfigure its own WebUIs when it sees INACTIVE state on --check" + (( WEBUI_FAILURES++ )) +fi + +# ── Step 4: Disable critical sync ───────────────────────────────────────────────────────────── +echo "" +echo "━━━ $ICON_GEAR Step 4/10 — Disable Critical Sync ━━━" + +if [[ "$DRY_RUN" == false ]]; then + update_master_conf "CRITICAL_RSYNC_ENABLED" "false" + warn "CRITICAL_RSYNC_ENABLED=false ✅" +else + warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false" +fi + +# ── Step 5: Write state, push to mirror, blocklist ──────────────────────────────────────────── +echo "" +echo "━━━ $ICON_GEAR Step 5/10 — Write State ━━━" + +NOW=$(date '+%Y-%m-%d %H:%M:%S') + +if [[ "$DRY_RUN" == false ]]; then + write_state_file "$LOCAL_STATE_FILE" \ + "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON" + log "Local state: INACTIVE ✅" + add_to_blocklist "$MIRROR" "$REASON" + [[ "$MIRROR_REACHABLE" == true ]] && \ + push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY" +else + warn "DRY RUN — would write INACTIVE state, blocklist $MIRROR, push to remote" +fi + +# ── Step 6: Local container cleanup ─────────────────────────────────────────────────────────── +echo "" +echo "━━━ $ICON_CONTAINERS Step 6/10 — Local Container Cleanup ━━━" + +PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$MIRROR") +cleanup_partner_containers "$PARTNER_FOLDER_NAME" + +# ── Step 7: Restart own stack ───────────────────────────────────────────────────────────────── +start_own_stack + +# ── Step 8: Remote container cleanup ────────────────────────────────────────────────────────── +echo "" +echo "━━━ $ICON_CONTAINERS Step 8/10 — Remote Container Cleanup ━━━" + +if [[ "$MIRROR_REACHABLE" == true ]]; then + # Remove auth/arr stack containers deployed during onboard (by config array) + cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY" + # Remove fallback coverage containers (by *-owner_short naming pattern) + cleanup_owner_containers_on_mirror "$MIRROR_IP" +else + warn "$MIRROR unreachable — remote container cleanup skipped" + warn "Run 'partnership_manager.sh --offboard' on $MIRROR to clean up manually" +fi + +# ── Step 9: Restart mirror's own stack ──────────────────────────────────────────────────────── +echo "" +echo "━━━ $ICON_START Step 9/10 — Restart Mirror Stack ━━━" + +[[ "$MIRROR_REACHABLE" == true ]] && start_mirror_own_stack "$MIRROR_IP" + +# ── Step 10: Revocation ─────────────────────────────────────────────────────────────────────── +echo "" +echo "━━━ $ICON_SHIELD Step 10/10 — Revocation ━━━" + +# Emby admin — before SSH key revocation while Emby still reachable +[[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP" + +# SSH key revocation — mutual, both directions; must run while Tailscale still active +do_ssh_key_revocation "${MIRROR_IP:-}" + +# Tailscale removal — after SSH revocation, guard with grace window +if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then + echo "" + echo "━━━ $ICON_NET Tailscale Separation ━━━" + if [[ "$MIRROR_REACHABLE" == true ]]; then + grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-6} * 3600 )) + warn "Waiting ${PARTNERSHIP_GRACE_HOURS:-6}hr grace — mirror can collect backups..." + if [[ "$DRY_RUN" == false ]]; then + trap 'warn "Offboard interrupted during grace sleep"; exit 0' SIGTERM SIGINT + sleep "$grace_seconds" + trap - SIGTERM SIGINT + fi + fi + remove_tailscale_device "$MIRROR" +fi + +# Backup handover notification +if [[ ${#PARTNERSHIP_MIRROR_BACKUPS[@]} -gt 0 ]]; then + echo "" + echo "━━━ $ICON_DISK Backup Handover ━━━" + log "Backups available for $MIRROR:" + for path in "${PARTNERSHIP_MIRROR_BACKUPS[@]}"; do + [[ -z "$path" ]] && continue + echo " $path" + done + notify "$MIRROR offboard complete — backups available for ${PARTNERSHIP_GRACE_HOURS:-6}hr. Tailscale access expires then." \ + "Partnership" "warning" +fi + +# ── Summary ─────────────────────────────────────────────────────────────────────────────────── +END=$(date +%s) +echo "" +echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY (Owner) ━━━━━" +echo " Owner: $MY_ID ($LOCAL_SERVER_NAME)" +echo " Mirror: $MIRROR_ID ($MIRROR)" +echo " Reason: $REASON" +echo " Duration: $(format_duration $(( END - START )))" +echo "" +_ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; } +_revoke_status() { + if [[ "${SSH_REVOKE_REMOTE_OK:-false}" == true && "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then + echo "both directions ✅" + elif [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then + echo "local only ✅ — remote failed (revoke manually on $MIRROR)" + else + echo "⚠️ failed — check warnings above" + fi +} + +echo " Step 1 — Stop rsync: ✅" +echo " Step 2 — Final sync: ✅" +echo " Step 3 — WebUI failures: $WEBUI_FAILURES" +echo " Step 4 — Disable sync: ✅" +echo " Step 5 — State: INACTIVE ✅" +echo " Step 6 — Local cleanup: ✅" +echo " Step 7 — Own stack: started" +echo " Step 8 — Remote cleanup: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "✅" || echo "skipped (unreachable)" )" +echo " Step 9 — Mirror stack: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "started" || echo "skipped (unreachable)" )" +echo " Step 10 — Keys revoked: $(_revoke_status)" +echo "" +echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅" +[[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]] && \ + echo " FolderView3: ${PARTNER_FOLDER_NAME:-} cleaned ✅" +[[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]] && \ + echo " Tailscale: $MIRROR removed ✅" +echo "" +echo " $MIRROR leaves with:" +echo " ✓ Current auth config (final sync)" +echo " ✓ Auth WebUIs → localhost" +echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-6}hr to collect backups" +echo "" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \ + warn "$ICON_DONE DONE — clean separation complete ✅" +echo "━━━━━━━━━━━━━━━━━━━━━━━" +exit 0 diff --git a/Partnership/partnership_onboard.sh b/Partnership/partnership_onboard.sh new file mode 100755 index 0000000..28df983 --- /dev/null +++ b/Partnership/partnership_onboard.sh @@ -0,0 +1,640 @@ +#!/bin/bash +# ============================================================================================== +# ============================= Partnership Onboard ============================================ +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Runs once on both servers to establish a new partnership. Role is detected +# automatically via detect_hosts() — no flags needed to declare which side you are. +# Run on the mirror first (generates its SSH key), then on the owner to complete +# setup remotely. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# MIRROR PATH (1 step) +# Step 1: SSH key setup — generate keypair, copy to owner, update conf +# Owner completes the rest remotely. Mirror is done. +# +# OWNER PATH (8 steps) +# Step 1: SSH key setup — generate keypair, install on mirror, update conf +# Step 2: Plugin install — FolderView3 and required plugins on mirror +# Step 3: Stop mirror auth — stop mirror's existing auth containers before replacing +# Step 4: Deploy auth stack — push XMLs, pull images, create + start on mirror +# Mariadb/Redis health-checked before Authelia deploys +# Step 5: Stop mirror arr — stop mirror's existing arr containers before replacing +# Step 6: Deploy arr stack — push arr XMLs, pull images, create + start on mirror +# Step 7: Partnership onboard — configure WebUIs → owner IP, write state, FolderView3, Emby +# Step 8: Arr bootstrap — bidirectional library sync (arr_sync.sh) +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Credentials never in SSH command strings +# Auth stack containers hold API keys, DB passwords, etc. The deploy script is written +# locally, SCPed to the remote, and executed there. Command-line args are never used +# to pass credentials — they'd appear in `ps` output and shell history on both servers. +# +# XML templates are the single source of truth for deployed containers +# The owner's templates-user/ XMLs define every container deployed on the mirror. +# The same XMLs that Unraid's Docker Manager uses are what get SCPed — the mirror's +# Docker Manager can manage the containers after onboard without additional config. +# +# Dependency ordering in the auth stack is owner-enforced +# PARTNERSHIP_AUTH_STACK order matters: Mariadb and Redis must come before Authelia. +# The array is ordered correctly in master_host1.conf. After each Mariadb/Redis deploy, +# the script waits for the container to be healthy before continuing. This is a remote +# health check — the container must be running (or report healthy) before the next +# dependent is deployed. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root check +# All operations run as root — SSH key management, docker operations, conf updates. +# +# SSH timeout on all remote calls +# Every ssh/scp call uses SSH_TIMEOUT. No operation hangs indefinitely on a +# slow or unreachable mirror. +# +# --dry-run shows exact actions without executing +# Every step prints what it would do. SCP, deploy, plugin install, arr sync — +# all dry-run safe. +# +# Step skip flags for partial re-runs +# --skip-ssh, --skip-auth-stack, --skip-arr-stack, --skip-arr-sync allow +# resuming after a partial failure without re-running completed steps. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# +# HOST*_PARTNERSHIP_AUTH_STACK +# XML filenames (from this server's templates-user/) to push and deploy on the +# mirror as its auth stack. Order matters: database deps before Authelia. +# Aliased by detect_hosts() → PARTNERSHIP_AUTH_STACK +# +# HOST*_PARTNERSHIP_REPLACE_CONTAINERS +# Containers to stop on the mirror before deploying the auth stack. +# Defined in the MIRROR's own conf (master_host*.conf on HOST2) — never in HOST1's conf. +# Read live from the mirror via SSH during Step 3 (sources mirror's load_config.sh at +# the same $SCRIPTS_ROOT path — convention: both servers use the same repo location). +# Leave empty on HOST2 if no conflicting containers exist (fresh mirror: nothing to stop). +# Aliased by detect_hosts() → PARTNERSHIP_REPLACE_CONTAINERS (on the mirror) +# +# HOST*_PARTNERSHIP_ARR_STACK +# XML filenames to push and deploy on the mirror as its arr stack. +# Leave empty to skip arr stack deploy. +# Aliased by detect_hosts() → PARTNERSHIP_ARR_STACK +# +# HOST*_PARTNERSHIP_ARR_REPLACE_CONTAINERS +# Arr containers to stop on the mirror before deploying the arr stack. +# Same rule as PARTNERSHIP_REPLACE_CONTAINERS: defined in mirror's own conf, never HOST1's. +# Aliased by detect_hosts() → PARTNERSHIP_ARR_REPLACE_CONTAINERS (on the mirror) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# Partnership/partnership_onboard.sh +# Full onboard — role detected automatically +# +# Partnership/partnership_onboard.sh --dry-run +# Preview all steps without making changes +# +# Partnership/partnership_onboard.sh --log +# Verbose per-step output +# +# Partnership/partnership_onboard.sh --skip-ssh +# Skip SSH key setup (key already in place) +# +# Partnership/partnership_onboard.sh --skip-auth-stack +# Skip auth stack stop + deploy (Steps 3-4) +# +# Partnership/partnership_onboard.sh --skip-arr-stack +# Skip arr stack stop + deploy (Steps 5-6) +# +# Partnership/partnership_onboard.sh --skip-arr-sync +# Skip arr library bootstrap (Step 8) +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPTS_ROOT="$SCRIPT_DIR/.." +TEMPLATES_DIR="/boot/config/plugins/dockerMan/templates-user" +SSH_TIMEOUT=15 + +source "$SCRIPTS_ROOT/load_config.sh" + +# ── Parse flags ─────────────────────────────────────────────────────────────────────────────── +SKIP_SSH=false +SKIP_AUTH_STACK=false +SKIP_ARR_STACK=false +SKIP_ARR_SYNC=false +FILTERED_ARGS=() + +for arg in "$@"; do + case "$arg" in + --skip-ssh) SKIP_SSH=true ;; + --skip-auth-stack) SKIP_AUTH_STACK=true ;; + --skip-arr-stack) SKIP_ARR_STACK=true ;; + --skip-arr-sync) SKIP_ARR_SYNC=true ;; + *) FILTERED_ARGS+=("$arg") ;; + esac +done + +parse_args "${FILTERED_ARGS[@]}" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; } + +detect_hosts + +OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}" +MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" ) +OWNER="${!OWNER_ID}" +MIRROR="${!MIRROR_ID}" +# SSH_KEY (set by detect_hosts) is this server's own private key. +# The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh. +# HOST{N}_SSH_KEY lives in master_host{N}.conf — with sparse checkout, the other server's +# conf is never present here. Always use SSH_KEY (local private key) for outbound SSH. +MIRROR_SSH_KEY="$SSH_KEY" + +AM_OWNER=false +AM_MIRROR=false +[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true +[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true + +EXTRA_FLAGS=() +[[ "$DRY_RUN" == true ]] && EXTRA_FLAGS+=("--dry-run") +[[ "$LOG_MODE" == true ]] && EXTRA_FLAGS+=("--log") + +START=$(date +%s) + +echo "" +echo "━━━ $ICON_FALLBACK Partnership Onboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "" +echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )" +echo " This: $MY_ID ($LOCAL_SERVER_NAME)" +echo " Partner: $( [[ "$AM_OWNER" == true ]] && echo "$MIRROR_ID ($MIRROR)" || echo "$OWNER_ID ($OWNER)" )" +echo "" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made" + +# ============================================================================================== +# ── HELPER: deploy a container from a local Unraid template XML to a remote host ───────────── +# +# Parses Port / Path / Variable Config entries from the XML, SCPs the template and a +# self-contained deploy script to the remote, executes it, then cleans up both sides. +# Credentials are never passed as SSH command-line args — they stay in the SCPed script. +# ============================================================================================== +deploy_container_from_xml() { + local xml_file="$1" remote_ip="$2" ssh_key="$3" + local xml_name + xml_name=$(basename "$xml_file") + + # Extract top-level fields + local name repo network extra privileged + name=$( awk 'match($0,/([^<]+)<\/Name>/, a){print a[1];exit}' "$xml_file") + repo=$( awk 'match($0,/([^<]+)<\/Repository>/,a){print a[1];exit}' "$xml_file") + network=$( awk 'match($0,/([^<]+)<\/Network>/, a){print a[1];exit}' "$xml_file") + extra=$( awk 'match($0,/([^<]*)<\/ExtraParams>/,a){print a[1];exit}' "$xml_file") + privileged=$( awk 'match($0,/([^<]+)<\/Privileged>/,a){print a[1];exit}' "$xml_file") + + if [[ -z "$name" || -z "$repo" ]]; then + warn " Cannot parse Name/Repository from $xml_name — skipping" + return 1 + fi + + log "Deploying $name..." + + # SCP the XML so Unraid Docker Manager recognises and can manage the container + if [[ "$DRY_RUN" == false ]]; then + timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \ + "$xml_file" "root@${remote_ip}:${TEMPLATES_DIR}/${xml_name}" 2>/dev/null || { + warn " SCP failed for $xml_name — skipping $name" + return 1 + } + else + warn " DRY RUN — would SCP $xml_name → $MIRROR:${TEMPLATES_DIR}/" + fi + + # Build a self-contained deploy script locally. + # Writing to a temp file keeps credentials out of SSH command strings. + local tmp_script + tmp_script=$(mktemp /tmp/deploy_XXXXXX.sh) + chmod 600 "$tmp_script" + + { + echo "#!/bin/bash" + echo "set -e" + echo "" + printf "docker pull %q 2>/dev/null || true\n" "$repo" + printf "docker stop %q 2>/dev/null || true\n" "$name" + printf "docker rm %q 2>/dev/null || true\n" "$name" + echo "" + printf "docker create --name %q --restart=unless-stopped" "$name" + [[ -n "$network" ]] && printf " --network=%q" "$network" + [[ "$privileged" == "true" ]] && printf " --privileged" + [[ -n "$extra" ]] && printf " %s" "$extra" + + # Port mappings → -p host:container/proto + awk '/Type="Port"/ { + match($0, /Target="([^"]+)"/, t) + match($0, /Mode="([^"]+)"/, m) + match($0, />([^<]+)<\/Config>/, v) + if (t[1] != "" && v[1] != "") { + proto = (m[1] == "udp") ? "udp" : "tcp" + printf " -p %s:%s/%s", v[1], t[1], proto + } + }' "$xml_file" + + # Volume mappings → -v 'host:container:mode' + awk 'BEGIN{q=sprintf("%c",39)} /Type="Path"/ { + match($0, /Target="([^"]+)"/, t) + match($0, /Mode="([^"]+)"/, m) + match($0, />([^<]+)<\/Config>/, v) + if (t[1] != "" && v[1] != "") { + mode = (m[1] == "ro") ? "ro" : "rw" + printf " -v %s%s:%s:%s%s", q, v[1], t[1], mode, q + } + }' "$xml_file" + + # Environment variables → -e 'KEY=VALUE' (single-quoted to protect $ and special chars) + awk 'BEGIN{q=sprintf("%c",39)} /Type="Variable"/ { + match($0, /Target="([^"]+)"/, t) + match($0, />([^<]+)<\/Config>/, v) + if (t[1] != "" && v[1] != "") { + printf " -e %s%s=%s%s", q, t[1], v[1], q + } + }' "$xml_file" + + printf " %q\n" "$repo" + echo "" + printf "docker start %q && echo 'deployed:%s'\n" "$name" "$name" + } > "$tmp_script" + + if [[ "$DRY_RUN" == true ]]; then + warn " DRY RUN — would deploy $name on $MIRROR" + rm -f "$tmp_script" + return 0 + fi + + # SCP deploy script → remote, execute, clean up both sides + local remote_script="/tmp/deploy_${name//[^a-zA-Z0-9_]/_}.sh" + + if timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \ + "$tmp_script" "root@${remote_ip}:${remote_script}" 2>/dev/null && \ + timeout 120 ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \ + "bash '$remote_script' 2>&1; rc=\$?; rm -f '$remote_script'; exit \$rc" 2>/dev/null | \ + grep -q "deployed:${name}"; then + log " $name deployed ✅" + rm -f "$tmp_script" + return 0 + else + warn " $name deployment failed — check $MIRROR manually" + rm -f "$tmp_script" + return 1 + fi +} + +# ============================================================================================== +# ── HELPER: wait for a container on the remote to be healthy/running ───────────────────────── +# +# Polls docker inspect on the remote. Prefers the health status if a healthcheck is defined; +# falls back to the running state for containers with no healthcheck. Non-fatal after timeout +# — Authelia may take time to fully initialize but the deploy itself succeeded. +# ============================================================================================== +wait_for_container_healthy() { + local name="$1" remote_ip="$2" ssh_key="$3" + local max_wait=60 interval=5 elapsed=0 + + [[ "$DRY_RUN" == true ]] && return 0 + + log " Waiting for $name to be ready..." + while (( elapsed < max_wait )); do + local status + status=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \ + "h=\$(docker inspect --format '{{.State.Health.Status}}' '$name' 2>/dev/null) + r=\$(docker inspect --format '{{.State.Running}}' '$name' 2>/dev/null) + echo \${h:-\$r}" 2>/dev/null) + + case "$status" in + healthy|true) + log " $name ready ✅" + return 0 + ;; + starting|unhealthy|false|"") + sleep "$interval" + (( elapsed += interval )) + ;; + *) + sleep "$interval" + (( elapsed += interval )) + ;; + esac + done + + warn " $name not confirmed healthy after ${max_wait}s — continuing (may affect dependents)" + return 0 +} + +# ============================================================================================== +# ── HELPER: stop containers on the mirror by reading its own conf via SSH ──────────────────── +# +# SSHes to the mirror, sources its load_config.sh at the same $SCRIPTS_ROOT path (both servers +# use the same convention), and reads the named config array from the mirror's own conf. +# HOST2's container list stays in HOST2's master_host2.conf — not duplicated in HOST1's conf. +# Fails gracefully if scripts aren't present yet or the array is empty (nothing to stop). +# +# deploy_container_from_xml() already stops/removes containers with the same name as what's +# being deployed. This step handles containers with DIFFERENT names that conflict. +# ============================================================================================== +stop_mirror_stack() { + local config_var="$1" label="$2" + local -a to_stop=() + + mapfile -t to_stop < <( + timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \ + "source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null + detect_hosts 2>/dev/null + printf '%s\n' \"\${${config_var}[@]:-}\"" 2>/dev/null | grep -v '^$' + ) + + if [[ ${#to_stop[@]} -eq 0 ]]; then + log "No $label containers to stop on $MIRROR — skipping" + return 0 + fi + + log "Stopping $label on $MIRROR: ${to_stop[*]}" + for container in "${to_stop[@]}"; do + if [[ "$DRY_RUN" == true ]]; then + warn " DRY RUN — would stop + rm $container on $MIRROR" + continue + fi + timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" root@"$MIRROR_IP" \ + "docker stop '$container' 2>/dev/null + docker rm '$container' 2>/dev/null && echo removed" 2>/dev/null | \ + grep -q removed && \ + log " $container removed ✅" || \ + log " $container not found on $MIRROR — skipping" + done +} + +# ============================================================================================== +# ── HELPER: deploy a stack of XMLs to the mirror, health-checking db deps between batches ──── +# +# Sets globals _STACK_DEPLOYED and _STACK_FAILED rather than printing to stdout. +# This avoids the process-substitution capture problem: warn() writes to stdout, so any +# read -r X Y < <(func) would capture warn output as the count values. +# ============================================================================================== +_STACK_DEPLOYED=0 +_STACK_FAILED=0 + +deploy_xml_stack() { + local -n xml_array_ref="$1" + _STACK_DEPLOYED=0 + _STACK_FAILED=0 + + for xml_name in "${xml_array_ref[@]}"; do + local xml_file="${TEMPLATES_DIR}/${xml_name}" + if [[ ! -f "$xml_file" ]]; then + warn "$xml_name not found in $TEMPLATES_DIR — skipping" + (( _STACK_FAILED++ )) + continue + fi + + # Extract container name to use for health-wait matching + local cname + cname=$(awk 'match($0,/([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file") + + if deploy_container_from_xml "$xml_file" "$MIRROR_IP" "$MIRROR_SSH_KEY"; then + (( _STACK_DEPLOYED++ )) + # Health-check database deps before continuing — they must be ready before + # Authelia/app containers that depend on them can start cleanly. + if [[ -n "$cname" ]] && echo "$cname" | grep -qiE 'mariadb|redis|postgres|mysql'; then + wait_for_container_healthy "$cname" "$MIRROR_IP" "$MIRROR_SSH_KEY" + fi + else + (( _STACK_FAILED++ )) + fi + done +} + +# ============================================================================================== +# ── MIRROR PATH ─────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +if [[ "$AM_MIRROR" == true ]]; then + echo "━━━ Step 1/1 — SSH Key Setup (Mirror) ━━━" + echo "" + echo " Mirror only needs SSH keys ready." + echo " Owner ($OWNER) completes the rest remotely." + echo "" + + if [[ "$SKIP_SSH" == true ]]; then + warn "Skipping SSH setup (--skip-ssh)" + elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then + log "SSH key ready ✅" + else + error "SSH key setup failed" + exit 1 + fi + + echo "" + echo "━━━━━ $ICON_SUMMARY MIRROR SETUP COMPLETE ━━━━━" + echo " SSH key: ready" + echo " Next: Run Partnership/partnership_onboard.sh on $OWNER to complete setup" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ── OWNER PATH ──────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +MIRROR_IP=$(resolve_tailscale_ip "$MIRROR") +[[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP — is Tailscale running?"; exit 1; } +log "Mirror: $MIRROR ($MIRROR_IP)" +echo "" + +STEP_SSH_OK=false +STEP_PLUGINS_OK=true +STEP_STOP_AUTH_OK=true +STEP_AUTH_OK=true +AUTH_DEPLOYED=0 +AUTH_FAILED=0 +STEP_STOP_ARR_OK=true +STEP_ARR_OK=true +ARR_DEPLOYED=0 +ARR_FAILED=0 +ONBOARD_OK=false +ARR_SYNC_OK=false + +# ── Step 1: SSH ─────────────────────────────────────────────────────────────────────────────── +echo "━━━ Step 1/8 — SSH Key Setup ━━━" + +if [[ "$SKIP_SSH" == true ]]; then + warn "Skipping (--skip-ssh)" + STEP_SSH_OK=true +elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then + log "SSH keys ready ✅" + STEP_SSH_OK=true +else + error "SSH key setup failed — aborting" + error "Re-run or use --skip-ssh if key is already set up" + exit 1 +fi + +# ── Step 2: Plugins ─────────────────────────────────────────────────────────────────────────── +echo "" +echo "━━━ Step 2/8 — Plugin Install on Mirror ━━━" + +if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]] && [[ -n "${PARTNERSHIP_FOLDERVIEW3_URL:-}" ]]; then + FV3_PRESENT=$(timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \ + "test -d /usr/local/emhttp/plugins/folder.view3 && echo yes" 2>/dev/null) + + if [[ "$FV3_PRESENT" == "yes" ]]; then + log "FolderView3 already installed on $MIRROR ✅" + elif [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would install FolderView3 on $MIRROR" + else + log "Installing FolderView3 on $MIRROR..." + timeout 60 ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" root@"$MIRROR_IP" \ + "plugin install '$PARTNERSHIP_FOLDERVIEW3_URL' 2>/dev/null && echo installed" \ + 2>/dev/null | grep -q installed && \ + log "FolderView3 installed ✅" || { + warn "FolderView3 install failed — install manually from Community Applications" + STEP_PLUGINS_OK=false + } + fi +else + log "FolderView3 not configured — skipping" +fi + +# ── Step 3: Stop mirror's existing auth stack ───────────────────────────────────────────────── +echo "" +echo "━━━ Step 3/8 — Stop Mirror Auth Stack ━━━" + +if [[ "$SKIP_AUTH_STACK" == true ]]; then + warn "Skipping (--skip-auth-stack)" +else + stop_mirror_stack "PARTNERSHIP_REPLACE_CONTAINERS" "auth stack" +fi + +# ── Step 4: Deploy auth stack on mirror ─────────────────────────────────────────────────────── +echo "" +echo "━━━ Step 4/8 — Deploy Auth Stack on Mirror ━━━" + +if [[ "$SKIP_AUTH_STACK" == true ]]; then + warn "Skipping (--skip-auth-stack)" +elif [[ ${#PARTNERSHIP_AUTH_STACK[@]} -eq 0 ]]; then + warn "PARTNERSHIP_AUTH_STACK not set in ${MY_ID} conf — skipping auth stack deploy" + warn "Add HOST${MY_ID: -1}_PARTNERSHIP_AUTH_STACK to master_host${MY_ID: -1}.conf" + STEP_AUTH_OK=false +else + deploy_xml_stack PARTNERSHIP_AUTH_STACK + AUTH_DEPLOYED=$_STACK_DEPLOYED + AUTH_FAILED=$_STACK_FAILED + log "Auth stack: $AUTH_DEPLOYED deployed, $AUTH_FAILED failed" + [[ "$AUTH_FAILED" -gt 0 ]] && STEP_AUTH_OK=false +fi + +# ── Step 5: Stop mirror's existing arr stack ────────────────────────────────────────────────── +echo "" +echo "━━━ Step 5/8 — Stop Mirror Arr Stack ━━━" + +if [[ "$SKIP_ARR_STACK" == true ]]; then + warn "Skipping (--skip-arr-stack)" +elif [[ ${#PARTNERSHIP_ARR_STACK[@]} -eq 0 ]]; then + log "PARTNERSHIP_ARR_STACK not configured — skipping arr stack deploy" + SKIP_ARR_STACK=true +else + stop_mirror_stack "PARTNERSHIP_ARR_REPLACE_CONTAINERS" "arr stack" +fi + +# ── Step 6: Deploy arr stack on mirror ─────────────────────────────────────────────────────── +echo "" +echo "━━━ Step 6/8 — Deploy Arr Stack on Mirror ━━━" + +if [[ "$SKIP_ARR_STACK" == true ]]; then + warn "Skipping (--skip-arr-stack)" +else + deploy_xml_stack PARTNERSHIP_ARR_STACK + ARR_DEPLOYED=$_STACK_DEPLOYED + ARR_FAILED=$_STACK_FAILED + log "Arr stack: $ARR_DEPLOYED deployed, $ARR_FAILED failed" + [[ "$ARR_FAILED" -gt 0 ]] && STEP_ARR_OK=false +fi + +# ── Step 7: Partnership onboard ─────────────────────────────────────────────────────────────── +echo "" +echo "━━━ Step 7/8 — Partnership Onboard ━━━" + +if bash "$SCRIPTS_ROOT/Partnership/partnership_manager.sh" --onboard "${EXTRA_FLAGS[@]}"; then + log "Partnership onboard complete ✅" + ONBOARD_OK=true +else + error "Partnership onboard failed" + ONBOARD_OK=false +fi + +# ── Step 8: Arr library bootstrap ───────────────────────────────────────────────────────────── +echo "" +echo "━━━ Step 8/8 — Arr Library Bootstrap ━━━" + +if [[ "$ONBOARD_OK" == false ]]; then + warn "Skipping — onboard did not complete" +elif [[ "$SKIP_ARR_SYNC" == true ]]; then + warn "Skipping (--skip-arr-sync)" +elif [[ ! -f "$SCRIPTS_ROOT/Media/arr_sync.sh" ]]; then + warn "arr_sync.sh not found — run Media/arr_sync.sh manually once arrs are live" +elif bash "$SCRIPTS_ROOT/Media/arr_sync.sh" "${EXTRA_FLAGS[@]}"; then + log "Arr bootstrap complete ✅" + ARR_SYNC_OK=true +else + warn "Arr sync had errors — partnership still valid" + warn "Re-run Media/arr_sync.sh once all arr containers are live" +fi + +# ── Summary ─────────────────────────────────────────────────────────────────────────────────── +END=$(date +%s) +echo "" +echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━" +echo " Owner: $MY_ID ($LOCAL_SERVER_NAME)" +echo " Mirror: $MIRROR ($MIRROR_IP)" +echo " Duration: $(format_duration $(( END - START )))" +echo "" +_ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; } +_skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; } + +echo " Step 1 — SSH keys: $(_skip "$SKIP_SSH" "$STEP_SSH_OK")" +echo " Step 2 — Plugins: $(_ok "$STEP_PLUGINS_OK")" +echo " Step 3 — Stop auth: $(_skip "$SKIP_AUTH_STACK" "$STEP_STOP_AUTH_OK")" +echo " Step 4 — Auth stack: $( [[ "$SKIP_AUTH_STACK" == true ]] && echo "skipped" || echo "${AUTH_DEPLOYED} deployed, ${AUTH_FAILED} failed" )" +echo " Step 5 — Stop arr: $(_skip "$SKIP_ARR_STACK" "$STEP_STOP_ARR_OK")" +echo " Step 6 — Arr stack: $( [[ "$SKIP_ARR_STACK" == true ]] && echo "skipped" || echo "${ARR_DEPLOYED} deployed, ${ARR_FAILED} failed" )" +echo " Step 7 — Onboard: $(_ok "$ONBOARD_OK")" +echo " Step 8 — Arr bootstrap: $( [[ "$SKIP_ARR_SYNC" == true || "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARR_SYNC_OK")" )" +echo "" + +if [[ "$ONBOARD_OK" == true ]]; then + [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \ + log "$ICON_DONE DONE — partnership established ✅" + log "Verify with: Partnership/partnership_manager.sh --status" +else + error "Setup incomplete — resolve errors above and re-run" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━" +[[ "$ONBOARD_OK" == false ]] && exit 1 +exit 0 diff --git a/Initial_run/ssh_setup.sh b/Partnership/ssh_setup.sh similarity index 97% rename from Initial_run/ssh_setup.sh rename to Partnership/ssh_setup.sh index f182592..507d430 100755 --- a/Initial_run/ssh_setup.sh +++ b/Partnership/ssh_setup.sh @@ -30,10 +30,10 @@ # SSH_STRIKE_RESET_HRS — hours since last failure before counter resets (default 24) # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# Initial_run/ssh_setup.sh — initial setup (idempotent) -# Initial_run/ssh_setup.sh --force — regenerate + re-copy -# Initial_run/ssh_setup.sh --validate — health check + strike tracking -# Initial_run/ssh_setup.sh --status — show key and connectivity state +# Partnership/ssh_setup.sh — initial setup (idempotent) +# Partnership/ssh_setup.sh --force — regenerate + re-copy +# Partnership/ssh_setup.sh --validate — health check + strike tracking +# Partnership/ssh_setup.sh --status — show key and connectivity state # Any mode supports --dry-run and --log # ============================================================================================== @@ -262,8 +262,8 @@ if [[ "$MODE" == "validate" ]]; then if [[ "$STRIKES" -ge "$MAX_STRIKES" ]]; then error "SSH auth to $REMOTE_SERVER_NAME broken — ${STRIKES} consecutive failures" - error "Repair: run 'Initial_run/ssh_setup.sh --force' to regenerate and re-copy key" - notify "SSH auth broken to $REMOTE_SERVER_NAME ($MY_ID) — ${STRIKES} strikes, manual repair needed. Run: Initial_run/ssh_setup.sh --force" \ + error "Repair: run 'Partnership/ssh_setup.sh --force' to regenerate and re-copy key" + notify "SSH auth broken to $REMOTE_SERVER_NAME ($MY_ID) — ${STRIKES} strikes, manual repair needed. Run: Partnership/ssh_setup.sh --force" \ "SSH Setup" "warning" exit 2 fi diff --git a/Rsync/Manual-Rsync.md b/Rsync/Manual-Rsync.md new file mode 100644 index 0000000..ab6496c --- /dev/null +++ b/Rsync/Manual-Rsync.md @@ -0,0 +1,721 @@ +# ━━━━━ RSYNC — Manual ━━━━━ + +Config reference, procedures, operational workflows. +For overview see README-Rsync.md. For per-script detail see the rsync.sh header. + +This document also serves as the **initial setup guide** for standing up the +two-server ecosystem from scratch. + +--- + +## ━━━ WHAT YOU'RE BUILDING ━━━ + +``` +HOST1 (unRAID-Gmer4Lfe) HOST2 (unRAID-Jayred365) +────────────────────── ────────────────────── +Downloads to: Downloads to: + /mnt/user/Movies /mnt/user/Anime_Movies + /mnt/user/Tv_Shows /mnt/user/Anime_Shows + /mnt/user/Music + +Both servers' arrs track ALL content across both servers. +It does not matter who downloaded what or when. + +Daily 3-phase window (1am): + + Phase 1 — arr_sync.sh (runs first): + All arrs on all nodes reconcile libraries using external IDs + (TMDB, TVDB, MusicBrainz). Union model — any node that tracks + an item, all nodes get it. After this phase both servers' arrs + know about all content regardless of who downloaded it. + + Phase 2 — rsync (bidirectional, no --delete on media shares): + HOST1 pushes its shares ──────► HOST2 receives files + HOST1 receives files ◄────── HOST2 pushes its shares + Files arrive already tracked by the remote arr (arr_sync ran first). + No --delete — media files only spread. Arr cleanup handles deletions. + + Phase 3 — arr cleanup: + sonarr_cleanup / radarr_cleanup / lidarr_cleanup query the live arr + API and remove any files no longer tracked. Emby notified after. + +Note: profiled syncs (arrs_stack, emby, critical-data…) use their own +PROFILE_RSYNC_OPTS which include --delete. Only the no-profile media +shares use DEFAULT_RSYNC_OPTS (spread only). + +Weekly clean sync (2:30am Sunday, containers stopped): + Emby — full clean mirror ◄──────► Emby + Critical-Data ──────► Auth stack (HOST1 → HOST2) + +Every 30 minutes — dirty sync: + Emby watch states ──────► HOST2 stays current on playback +``` + +--- + +## ━━━ PREREQUISITES ━━━ + +Required on both servers before starting: + +``` +unRAID 7.x +Community Applications plugin — search "Community Applications" in unRAID plugins +User Scripts plugin — install via Community Applications +Tailscale plugin — install via Community Applications +Terminal access — unRAID UI → Tools → Terminal, or SSH +``` + +Optional but recommended: +``` +Gitea (Docker container on HOST1) — self-hosted git for the script repository +Working Emby installation — for transcode management and failover +``` + +--- + +## ━━━ STEP 1 — TAILSCALE ━━━ + +Tailscale provides the encrypted mesh network between servers. Scripts resolve the +remote server's IP via Tailscale at runtime — no hardcoded IPs, no VPN configuration, +no open ports. All server-to-server communication goes through Tailscale. + +### Install on Both Servers + +``` +1. Open Apps in the unRAID UI +2. Search "Tailscale" — install the plugin +3. Settings → Tailscale → Connect +4. Authenticate with your Tailscale account (browser opens on your machine) +5. Verify both servers appear: https://login.tailscale.com/admin/machines +``` + +### Verify Connectivity + +```bash +# From HOST1 — should return HOST2's 100.x.x.x Tailscale IP: +tailscale ip -4 unRAID-Jayred365 + +# Test actual connectivity: +tailscale ping unRAID-Jayred365 +``` + +> **Critical:** The hostnames in `master.conf` (`HOST1` and `HOST2`) must match the +> Tailscale machine names **exactly** — case sensitive. All remote IP resolution goes +> through `tailscale ip -4 HOSTNAME`. A name mismatch causes every remote operation to +> fail at the IP resolution step. + +--- + +## ━━━ STEP 2 — ENABLE SSH ━━━ + +unRAID 7.x has SSH disabled by default. Enable it on both servers. + +``` +Settings → Management Access → Secure Shell + SSH: Enabled + SSH port: 22 +Apply +``` + +SSH is only exposed on your local network and Tailscale interface. No ports are +opened to the public internet. + +--- + +## ━━━ STEP 3 — SSH KEYS ━━━ + +Two sets of keys needed: rsync automation keys (server-to-server) and a Gitea +access key (for script repository pulls). + +### 3a — Rsync Automation Keys (ssh_setup.sh) + +Run on **each server**. `ssh_setup.sh` generates the key, copies it to the remote, +and updates `master_host*.conf` with the key path automatically. + +```bash +# On HOST1 — after the repo is cloned: +bash /mnt/user/appdata/unraid_scripts/Partnership/ssh_setup.sh + +# On HOST2: +bash /mnt/user/appdata/unraid_scripts/Partnership/ssh_setup.sh +``` + +Key naming convention: hostname lowercased, `unraid-` prefix stripped. +``` +unRAID-Gmer4Lfe → /root/.ssh/gmer4lfe_rsync_automation +unRAID-Jayred365 → /root/.ssh/jayred365_rsync_automation +``` + +`master_host*.conf` is updated automatically with `HOST*_SSH_KEY` pointing to the +generated key. Run `--status` to verify: + +```bash +bash /mnt/user/appdata/unraid_scripts/Partnership/ssh_setup.sh --status +``` + +> **`ssh_setup.sh` is idempotent** — safe to re-run. Use `--force` to regenerate +> a key (e.g., after a security incident) and re-copy it to the remote. + +### 3b — Gitea SSH Key (manual) + +```bash +# On BOTH servers: +ssh-keygen -t ed25519 -f /root/.ssh/unraid_gitea -C "unraid-gitea" -N "" + +# Print the public key to add to Gitea: +cat /root/.ssh/unraid_gitea.pub + +# In Gitea: Settings → SSH / GPG Keys → Add Key → paste above +``` + +--- + +## ━━━ STEP 4 — CLONE THE REPOSITORY ━━━ + +```bash +# Create target directory: +mkdir -p /mnt/user/appdata/unraid_scripts + +# Clone: +GIT_SSH_COMMAND="ssh -i /root/.ssh/unraid_gitea" \ + git clone git@YOUR_GITEA_HOST:FailedProxy/Unraid_Scripts.git \ + /mnt/user/appdata/unraid_scripts + +# Make scripts executable: +find /mnt/user/appdata/unraid_scripts -name "*.sh" -exec chmod +x {} \; +``` + +Expected structure after clone: +``` +master.conf ← all shared configuration +master_host1.conf ← HOST1-specific configuration +master_host2.conf ← HOST2-specific configuration +common.sh ← shared library +load_config.sh ← config loader +Orchestrators/ +Rsync/ +Fallback/ +Docker_Essentials/ +... +``` + +--- + +## ━━━ STEP 5 — CONFIGURE MASTER.CONF ━━━ + +```bash +nano /mnt/user/appdata/unraid_scripts/master.conf +``` + +### Host Identity + +```bash +# These must match Tailscale machine names exactly — case sensitive. +HOST1="unRAID-Gmer4Lfe" # REQUIRED +HOST2="unRAID-Jayred365" # REQUIRED + +# SSH keys — each server uses its own key to authenticate to the other: +HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" +HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" +``` + +### Git Repository + +```bash +GITEA_CONTAINER="Gitea" +GITEA_REPO_PATH="FailedProxy/Unraid_Scripts.git" +TARGET_DIR="/mnt/user/appdata/unraid_scripts" +GITEA_SSH_KEY="/root/.ssh/unraid_gitea" +SSH_PORT=221 +``` + +### Daily Sync Shares + +```bash +# master_host1.conf +# Shares this server downloads to — pushed to the other server nightly. +# arr_sync.sh runs before rsync in the daily window, so the receiving +# server's arrs already track incoming files when they arrive. +# Never put the same path in both lists. +HOST1_DAILY_SYNC_SHARES=( + "/mnt/user/Movies" + "/mnt/user/Tv_Shows" + "/mnt/user/Music" + "/mnt/user/Kids_Movies" + "/mnt/user/Kids_Tv_Shows" + "/mnt/user/Sports" + "/mnt/user/stand-up_comedy" +) + +# master_host2.conf +HOST2_DAILY_SYNC_SHARES=( + "/mnt/user/Anime_Shows" + "/mnt/user/Anime_Movies" +) +``` + +### Weekly Sync Shares + +```bash +# master.conf +# Synced during the Sunday 2:30am window — containers stopped both sides. +WEEKLY_SYNC_SHARES=( + "/mnt/user/Media_Server/Emby" + "/mnt/user/appdata-Failover/Critical-Data" +) +``` + +--- + +## ━━━ STEP 6 — MASTER_HOST*.CONF ━━━ + +`detect_hosts()` reads which server is running and aliases `HOST*_` prefixed vars +to their unprefixed names. Scripts only ever reference the unprefixed name — they +work identically on both servers. + +```bash +nano /mnt/user/appdata/unraid_scripts/master_host1.conf # on HOST1 +nano /mnt/user/appdata/unraid_scripts/master_host2.conf # on HOST2 +``` + +Every variable is documented in the conf files. Key values to set: +- SSH key paths +- DAILY_SYNC_SHARES +- EMBY_URL / EMBY_API_KEY +- CERT_MONITOR_DOMAINS +- SMART_IGNORE_DRIVES +- ZFS_REPORT_IGNORE_POOLS + +--- + +## ━━━ RSYNC PROFILES ━━━ + +Profiles control per-share behavior. Profile key = directory basename lowercased. +Override with `--profile=name`. + +### Profile Matching + +```bash +# rsync.sh /mnt/user/appdata-Failover/Arrs_Stack +# basename: Arrs_Stack → lowercased: arrs_stack → matches [arrs_stack] profile +# +# rsync.sh /mnt/user/Movies +# basename: Movies → no profile match → global defaults (no containers stopped) +# +# rsync.sh /mnt/user/appdata-Failover/Critical-Data --profile=critical-data +# explicit override +``` + +### Current Profile Definitions + +```bash +# master.conf + +# ── arrs_stack ─────────────────────────────────────────────────────────────── +# Arr databases — stopped for clean SQLite snapshot +PROFILES["arrs_stack_CRITICAL_CONTAINER_NAMES"]=( + "Sonarr" "Radarr" "Lidarr" "Prowlarr" "Bazarr" "Pinchflat" +) + +# ── critical-data ───────────────────────────────────────────────────────────── +# Auth stack — stopped for clean database snapshot, Authelia delayed restart +PROFILES["critical-data_CRITICAL_CONTAINER_NAMES"]=( + "Mariadb-Authelia" "Redis-Authelia" + "NginxProxyManager" "Lldap-Gmer4Lfe" +) +PROFILES["critical-data_DELAYED_CONTAINERS"]=( + "Authelia" "Authelia-Secondary" +) +PROFILES["critical-data_CONTAINER_DELAY"]=30 + +# ── important-data ──────────────────────────────────────────────────────────── +# NextCloud + Postgres — stopped for clean snapshot +PROFILES["important-data_CRITICAL_CONTAINER_NAMES"]=( + "Postgres-NextCloud" +) +PROFILES["important-data_DELAYED_CONTAINERS"]=("NextCloud") + +# ── emby ────────────────────────────────────────────────────────────────────── +# Weekly full clean sync — both Emby instances stopped +PROFILES["emby_CRITICAL_CONTAINER_NAMES"]=("Emby") +PROFILES["emby_EXCLUDE_DIRS"]=( + "transcodes/" "logs/" "crash*" "cache/" +) + +# ── emby-failover ───────────────────────────────────────────────────────────── +# Every 30 minutes, Emby STAYS RUNNING — dirty sync of critical state only +PROFILES["emby-failover_CRITICAL_CONTAINER_NAMES"]=() # nothing stops +PROFILES["emby-failover_EXCLUDE_DIRS"]=( + "*.wal" "*.shm" # unsafe mid-write + "transcodes/" "logs/" "crash*" "cache/" +) +PROFILES["emby-failover_REMOTE_RESTART_CONTAINERS"]=("Emby") +``` + +### Two Emby Profiles — Why Both Exist + +**emby-failover** (every 30 minutes, Emby stays running): +- Syncs: users.db, library.db, authentication.db, config/ +- Skips: \*.wal, \*.shm, transcodes/, logs/, cache/ +- Why: WAL files are written while Emby runs — copying them would produce a corrupt database on HOST2 +- Result: HOST2 is always within 30 minutes of HOST1 on watch state and user activity + +**emby** (Sunday 2:30am, both Emby instances stopped): +- Syncs: everything except transcodes, logs, cache, crash files +- Includes: metadata, plugins, full database state, all config +- Why: WAL is checkpointed on clean shutdown — safe to copy everything +- Result: HOST2 gets a gold-standard Emby state once per week + +The two profiles work together. emby-failover keeps HOST2 current for immediate failover. +emby gives HOST2 full fidelity once per week. Neither alone is sufficient. + +--- + +## ━━━ PERSONAL ENCRYPTED SHARES ━━━ + +Personal shares sync to the remote for offsite backup. ZFS encrypts at the dataset +level — the remote receives encrypted blocks and cannot read the content without your +passphrase or keyfile. + +### Create an Encrypted ZFS Dataset + +```bash +# In the unRAID UI: +# Main → click your ZFS pool name → + Dataset +# Name: Gmer4Lfe-Personal +# Encryption: Enabled +# Passphrase: [your passphrase] +# Write your passphrase down — if lost, data is completely unrecoverable + +# Verify encryption is active before syncing: +zfs get encryption poolname/Gmer4Lfe-Personal +# Should show: encryption aes-256-gcm +``` + +### Auto-Unlock on Boot (Optional) + +```bash +# Keyfile approach — more convenient, but the keyfile itself is a secret +dd if=/dev/urandom bs=32 count=1 | base64 > /root/.zfs-keys/personal.key +chmod 600 /root/.zfs-keys/personal.key + +zfs change-key \ + -o keylocation=file:///root/.zfs-keys/personal.key \ + -o keyformat=raw \ + poolname/Gmer4Lfe-Personal + +# Add to array_start.sh or ramdisk_setup.sh: +zfs load-key poolname/Gmer4Lfe-Personal +zfs mount poolname/Gmer4Lfe-Personal + +# Manual unlock alternative (most secure): +zfs load-key poolname/Gmer4Lfe-Personal # prompts for passphrase +zfs mount poolname/Gmer4Lfe-Personal +``` + +### Add to master_host1.conf + +```bash +HOST1_PERSONAL_SHARES=( + "/mnt/user/Gmer4Lfe-Personal" +) +``` + +--- + +## ━━━ STEP 7 — USER SCRIPTS SETUP ━━━ + +Only a small number of User Scripts entries are needed — each one an orchestrator. +Individual scripts are never scheduled directly except the 30-minute Emby sync. + +### At Startup of Array + +```bash +#!/bin/bash +bash /mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh +# Schedule: At Startup of Array +# Run as: Background Task +``` + +This single entry launches everything defined in ARRAY_START_SCRIPTS from master.conf: +inotify_tuning, docker_syslog_filter, php_fpm_max_children, ramdisk_setup, +docker_network_connect, system_watchdog, docker_watchdog, failover. + +### Cron Schedule + +```bash +# Every 30 minutes — Emby dirty sync: +*/30 * * * * +bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ + /mnt/user/Media_Server/Emby --profile=emby-failover + +# Every 6 hours — failed import + stalled download recovery: +0 */6 * * * +bash /mnt/user/appdata/unraid_scripts/Orchestrators/arrs_failed_stalled_recovery.sh + +# 1am daily — full maintenance window: +0 1 * * * +bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh + +# 2:30am Sunday — weekly maintenance window: +30 2 * * 0 +bash /mnt/user/appdata/unraid_scripts/Orchestrators/weekly_sync_maintenance.sh + +# 8am daily — health digest: +0 8 * * * +bash /mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh + +# Every 6 hours — inotify + php-fpm snapshot: +0 */6 * * * +bash /mnt/user/appdata/unraid_scripts/Monitors/system_tuning_monitor.sh + +# Sunday morning — weekly reports: +0 6 * * 0 bash .../Monitors/zfs_memory_snapshot.sh +0 7 * * 0 bash .../Monitors/smart_health.sh +0 9 * * 0 bash .../Monitors/cert_monitor.sh +0 10 * * 0 bash .../Monitors/backup_verify.sh +0 11 * * 0 bash .../Monitors/emby_session_report.sh +0 11 * * 0 bash .../Monitors/bandwidth_monitor.sh --report +``` + +Set all entries to **Background Task** — non-background tasks can appear to hang +on long-running scripts. + +--- + +## ━━━ VERIFICATION ━━━ + +Test with `--dry-run` first — all pre-flight checks run, no changes made. + +### Test a Single Profile Sync + +```bash +bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ + /mnt/user/appdata-Failover/Arrs_Stack --dry-run --log +``` + +Expected output (healthy): +``` +━━━ Setup ━━━ + Host: HOST1 (unRAID-Gmer4Lfe) → HOST2 (unRAID-Jayred365) + Remote IP: 100.x.x.x + Profile: arrs_stack + +━━━ Pre-flight ━━━ + Remote reachable + version parity — both on unRAID X.Y.Z + Remote Docker daemon responding + Remote rootfs: 12% (threshold: 75%) + Remote share exists and not empty + All pre-flight checks passed +``` + +### Test the Daily Orchestrator + +```bash +bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh --dry-run +``` + +### Check Configuration Resolution + +```bash +bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ + /mnt/user/appdata-Failover/Arrs_Stack --status +``` + +--- + +## ━━━ INITIAL HOST2 SYNC ━━━ + +If HOST2 is being set up from scratch with empty shares: + +```bash +# Create share directories on HOST2: +bash /mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh + +# Initial push from HOST1 — first run may take several hours for large libraries: +bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh --log +``` + +The scheduled nightly sync will be incremental after the initial push. + +--- + +## ━━━ NAMING CONSISTENCY REQUIREMENT ━━━ + +The ecosystem uses one codebase on both servers. This only works if containers and +shares have identical names on both servers. **This is not configurable — it is a +design requirement.** + +``` +Container names must match exactly on both servers: + "Emby" ← both HOST1 and HOST2 + "NginxProxyManager" ← both HOST1 and HOST2 + "Mariadb-Authelia" ← both HOST1 and HOST2 + +Share paths must match exactly on both servers: + /mnt/user/Movies ← both HOST1 and HOST2 + /mnt/user/Tv_Shows ← both HOST1 and HOST2 +``` + +If a container has a different name on one server: the script skips it silently. +You only notice when the container is not stopped during a sync that requires it. + +If a share has a different path: rsync.sh aborts with "remote share missing." +Easier to catch — but still requires renaming the share to fix. + +--- + +## ━━━ TROUBLESHOOTING ━━━ + +### SSH Connection Refused / Timeout + +```bash +# Verify SSH is enabled on the remote: +# Settings → Management Access → Secure Shell → Enabled + +# Verify Tailscale is connected: +tailscale ip -4 unRAID-Jayred365 + +# Test SSH manually (key path from --status output): +ssh -i /root/.ssh/gmer4lfe_rsync_automation \ + root@$(tailscale ip -4 unRAID-Jayred365) "hostname" +# If password prompted: key not authorised — re-run ssh_setup.sh + +# Re-run setup (idempotent, re-copies key to remote): +bash /mnt/user/appdata/unraid_scripts/Partnership/ssh_setup.sh +``` + +### Pre-flight Aborts on Remote Rootfs + +```bash +# Check current usage on remote: +ssh -i /root/.ssh/Gmer4Lfe-rsync-key root@[HOST2-ip] "df /" + +# Common cause: array not started, drives not mounted +``` + +### Remote Share Missing + +```bash +# Verify the share exists on HOST2: +ssh -i /root/.ssh/Gmer4Lfe-rsync-key root@[HOST2-ip] "ls /mnt/user/" + +# If missing: create the share on HOST2, then run initial sync +``` + +### Containers Not Stopping / Starting + +```bash +# Verify container names match Docker exactly — case sensitive: +docker ps --format "{{.Names}}" + +# Test remote: +ssh -i /root/.ssh/Gmer4Lfe-rsync-key root@[HOST2-ip] "docker ps --format '{{.Names}}'" +``` + +### Profile Not Matching + +```bash +# Profile key = directory basename lowercased +# /mnt/user/appdata-Failover/Arrs_Stack → key: arrs_stack + +# Override explicitly: +bash rsync.sh /mnt/user/appdata-Failover/My_Stuff --profile=arrs_stack + +# Verify what profile resolved: +bash rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --status +``` + +--- + +## ━━━ FULL CONFIGURATION REFERENCE ━━━ + +### master.conf + +```bash +# Rsync engine +RSYNC_ENABLED=true +DEFAULT_RSYNC_OPTS="-az --no-perms --no-owner --no-group --inplace" +# No --delete in DEFAULT_RSYNC_OPTS — bidirectional media shares spread files only. +# Each server's arrs are source of truth for their content; arr cleanup scripts +# handle deletions. Profiles set PROFILE_RSYNC_OPTS with --delete explicitly. +BW_LIMIT=0 # KB/s, 0 = unlimited +RETRY_COUNT=3 +SLEEP=60 # seconds between retries +ROOTFS_WARN_PCT=75 # abort if remote rootfs above this % + +# Shared with Monitors/ +BANDWIDTH_LOG="/boot/config/bandwidth_history.db" +BANDWIDTH_LOG_RETENTION=90 +BANDWIDTH_WARN_GB=50 + +# Profile definitions (see RSYNC PROFILES section above) +declare -A PROFILES +declare -A PROFILE_BW_LIMIT +declare -A PROFILE_RETRY_COUNT +declare -A PROFILE_SLEEP +declare -A PROFILE_CONTAINER_DELAY +declare -A PROFILE_CRITICAL_CONTAINER_NAMES +declare -A PROFILE_DELAYED_CONTAINERS +declare -A PROFILE_EXCLUDE_DIRS +declare -A PROFILE_REMOTE_RESTART_CONTAINERS +``` + +### master_host*.conf + +```bash +# Daily sync shares — one list per server (mutually exclusive) +HOST1_DAILY_SYNC_SHARES=(...) +HOST2_DAILY_SYNC_SHARES=(...) + +# Personal encrypted shares +HOST1_PERSONAL_SHARES=(...) + +# SSH key for this server to authenticate to the remote +# Set automatically by ssh_setup.sh — do not edit manually +HOST1_SSH_KEY="/root/.ssh/gmer4lfe_rsync_automation" +HOST2_SSH_KEY="/root/.ssh/jayred365_rsync_automation" +``` + +--- + +## ━━━ FLAG REFERENCE ━━━ + +All flags work on `rsync.sh` and all orchestrators. + +### --dry-run + +Runs all pre-flight checks. Shows what rsync would transfer. No transfer, no container +stops, no bandwidth log entry. Safe to run at any time. + +```bash +rsync.sh /mnt/user/Movies --dry-run +rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --dry-run --log +daily_sync_maintenance.sh --dry-run +``` + +### --status + +Shows resolved configuration — profile, remote identity, all vars that would be +used — then exits. No pre-flight checks, no rsync. Use to verify configuration +loaded correctly. + +```bash +rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --status +``` + +### --log + +Verbose output throughout. Every decision, every container operation, every rsync +progress line. Use for first-time runs or when investigating issues. + +### --profile=name + +Override profile selection. Bypasses basename inference. Use when the directory +name doesn't match any profile key, or when testing a specific profile. + +```bash +rsync.sh /mnt/user/appdata-Failover/Critical-Data --profile=critical-data +rsync.sh /mnt/user/Media_Server/Emby --profile=emby-failover +``` diff --git a/Rsync/README-Rsync.md b/Rsync/README-Rsync.md new file mode 100644 index 0000000..a146733 --- /dev/null +++ b/Rsync/README-Rsync.md @@ -0,0 +1,133 @@ +# ━━━━━ RSYNC ━━━━━ + +The transfer engine for the two-server ecosystem. `rsync.sh` is the single script +called by every orchestrator that moves data between servers — it handles profiles, +pre-flight checks, container stops, the actual transfer, and bandwidth logging. +Orchestrators decide what to sync and when. `rsync.sh` decides how to do it safely. + +> **Never schedule rsync.sh directly for daily/weekly syncs.** Use the orchestrators +> in `Orchestrators/`. rsync.sh is called directly only for manual runs and the +> 30-minute Emby dirty sync, which needs its own cron entry. + +--- + +## ━━━ THE PROBLEM THAT BUILT THIS ━━━ + +**rsync Alone Isn't Safe Enough for Live Databases** +Running rsync against a share while SQLite databases are being written produces +corrupt snapshots on the remote. The arr databases, Emby library database, and +Authelia session store all write continuously. A plain rsync copies them mid-write. +The remote gets a file that opens cleanly but has internal inconsistencies. +Fix: profiles stop specific containers before syncing and restart them after. +The database is quiesced, rsync runs against a static snapshot, containers come back up. + +**Each Share Needs Different Behavior** +Media shares (Movies, TV) just spread files — nothing stops, no `--delete` +(arr cleanup scripts own deletions, and arr_sync ensures both arrs already +track incoming files before they arrive). Arr databases need containers stopped, +clean SQLite snapshot, restart. Emby has two modes: weekly full-stop clean mirror +and 30-minute dirty sync while Emby stays running (WAL files excluded). Critical-Data +stops the auth stack, waits for Authelia to come back after restart delay. +Fix: the profile system — one script, behavior defined entirely by the profile key. + +**A Failed Remote Shouldn't Corrupt a Live Sync** +If the remote's rootfs is nearly full, an rsync that starts will write partial files +then fail mid-transfer, leaving the remote in a worse state than before. If the remote's +backing disks are offline, rsync writes to an empty mount point and "succeeds." +Fix: pre-flight checks abort before touching anything if remote conditions are wrong. + +--- + +## ━━━ WHAT THIS FOLDER DOES ━━━ + +One script. One job: move data from this server to the remote safely. + +`rsync.sh` handles the full transfer lifecycle: +1. Infer or accept a profile for the given directory +2. Run pre-flight checks (connectivity, rootfs, disk temps, remote share exists) +3. Stop containers specified by the profile (both sides) +4. Run rsync with profile flags, excludes, and bandwidth limit +5. Restart containers (with delay if configured) +6. Restart remote containers if dirty-sync profile specifies it +7. Log the transfer to bandwidth_monitor.sh + +Everything else — deciding which shares to sync, in what order, on what schedule — +lives in the orchestrators. + +--- + +## ━━━ RELATIONSHIP TO OTHER FOLDERS ━━━ + +``` +Media/ + arr_sync.sh ── runs before rsync in daily window ──────────► all arrs agree on library + ↓ +Orchestrators/ ← decides what to sync, when, and in what order + daily_sync_maintenance.sh ──────────────────────────────────► rsync.sh (per share) + weekly_sync_maintenance.sh ──────────────────────────────────► rsync.sh (emby, critical-data) + critical_sync_maintenance.sh ────────────────────────────────► rsync.sh (partnership shares) + +Cron (direct): + */30 * * * * ──────────────────────────────────► rsync.sh --profile=emby-failover + +Monitors/ + bandwidth_monitor.sh ◄─── called by rsync.sh after each sync (--log-transfer) + +Fallback/ + fallback.sh ──── rsync writeback during handback ──► rsync.sh +``` + +rsync.sh never calls other scripts except `bandwidth_monitor.sh` at the end of a sync. +All orchestration logic lives in the callers. arr_sync.sh (Media/) is a peer that runs +before rsync in the daily window — it is not called by rsync.sh directly. + +--- + +## ━━━ THE PROFILE SYSTEM ━━━ + +Profile key = directory basename lowercased. `--profile=name` overrides. + +| Profile | What It Syncs | Containers Stopped | Notes | +|---------|--------------|-------------------|-------| +| *(none)* | Media shares (Movies, TV, Music…) | None | Bidirectional spread — no `--delete` in DEFAULT_RSYNC_OPTS. Arr cleanup scripts own deletions. | +| `arrs_stack` | Arr databases | Sonarr, Radarr, Lidarr, Prowlarr, Bazarr, Pinchflat | Clean SQLite snapshot | +| `critical-data` | Auth stack | Mariadb-Authelia, Redis-Authelia, NPM, Lldap | Authelia has restart delay | +| `important-data` | NextCloud + Postgres | Postgres-NextCloud | NextCloud has restart delay | +| `emby` | Full Emby mirror | Emby (both sides) | Weekly — Sunday 2:30am | +| `emby-failover` | Emby watch state delta | None | Dirty sync — Emby stays running | + +For full profile definitions see `Manual-Rsync.md`. + +--- + +## ━━━ SCRIPTS IN THIS FOLDER ━━━ + +| Script | Role | When It Runs | +|--------|------|-------------| +| `rsync.sh` | Core transfer engine — profile resolution, pre-flights, container management, transfer, bandwidth logging | Called by orchestrators; directly for manual and 30-min Emby dirty sync | + +--- + +## ━━━ HOW THE SCRIPTS RELATE ━━━ + +``` +Callers (Orchestrators/) ────────────────────────────────────────────────────── + daily_sync_maintenance.sh │ + weekly_sync_maintenance.sh rsync.sh /path/to/share [--profile=name] │ + critical_sync_maintenance.sh ─────────────────────────────────────────────► │ + fallback.sh (writeback) │ + Direct cron (emby-failover) │ + ▼ + ┌─────────────────────────────────────┐ + │ 1. Infer/accept profile │ + │ 2. Pre-flight checks │ + │ - connectivity │ + │ - rootfs / disk temp / disks │ + │ - remote share exists │ + │ 3. Stop containers (profile) │ + │ 4. rsync transfer │ + │ 5. Restart containers │ + │ 6. Remote restart (dirty sync) │ + │ 7. Log to bandwidth_monitor.sh │ + └─────────────────────────────────────┘ +``` diff --git a/Rsync/README-Rsync_Setup.md b/Rsync/README-Rsync_Setup.md deleted file mode 100644 index 46ea6bb..0000000 --- a/Rsync/README-Rsync_Setup.md +++ /dev/null @@ -1,993 +0,0 @@ -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# 🔄 RSYNC SETUP GUIDE -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -**Complete setup guide for the two-server rsync ecosystem.** By the end of this guide -both servers will have SSH keys configured, Tailscale connected, the git repository -cloned, and all scheduled operations running automatically. - -> **This is a setup guide, not a script reference.** For how rsync.sh works internally, -> profiles, safety checks, and operational details — those belong in the orchestrator -> and rsync script documentation. This guide is about standing the ecosystem up from -> scratch and verifying it works. - ---- - -## ━━━ WHAT YOU'RE BUILDING ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -``` -HOST1 (unRAID-Gmer4Lfe) HOST2 (unRAID-Jayred365) -────────────────────── ────────────────────── -Source of truth: Source of truth: - Movies, Tv_Shows, Music Anime_Shows, Anime_Movies - Critical-Data (auth stack) - Emby userdata - -Pushes to HOST2 daily: ──────→ Receives: - All HOST1 shares Mirror of HOST1 shares - Personal encrypted shares Personal (encrypted blocks) - -Receives from HOST2 daily: ←────── Pushes: - Anime_Shows, Anime_Movies All HOST2 shares - -Weekly clean sync (both sides stopped): - Emby — full clean mirror ←──────→ Emby - Critical-Data ──────→ Auth stack (HOST1 → HOST2) - -Every 30 minutes — dirty sync: - Emby watch states ──────→ HOST2 stays current on playback -``` - ---- - -## ━━━ PREREQUISITES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Both servers need these before starting: - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Required on both servers: - unRAID 7.x - Community Applications plugin — search "Community Applications" in unRAID plugins - User Scripts plugin — install via Community Applications - Tailscale plugin — install via Community Applications - Terminal access — unRAID UI → Tools → Terminal, or SSH - -# Optional but recommended: - Gitea (Docker container on HOST1) — self-hosted git for the script repository - Working Emby installation — for transcode management and failover -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -## ━━━ STEP 1 — TAILSCALE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Tailscale provides the encrypted mesh network between servers. Scripts resolve the -remote server's IP via Tailscale at runtime — no hardcoded IPs, no VPN configuration, -no open ports. All server-to-server communication goes through Tailscale. - ---- - -### ── Install on Both Servers ──────────────────────────────────────────────── - -```bash -# On each server: -# ───────────────────────────────────────────────────────────────────────────── -# 1. Open Apps in the unRAID UI -# 2. Search for "Tailscale" — install the plugin -# 3. Settings → Tailscale → Connect -# 4. Authenticate with your Tailscale account (browser opens on your machine) -# 5. Verify both servers appear: https://login.tailscale.com/admin/machines -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Verify Connectivity ───────────────────────────────────────────────────── - -```bash -# From HOST1 — should return HOST2's 100.x.x.x Tailscale IP: -tailscale ip -4 unRAID-Jayred365 - -# From HOST2 — should return HOST1's 100.x.x.x Tailscale IP: -tailscale ip -4 unRAID-Gmer4Lfe - -# Test actual connectivity: -tailscale ping unRAID-Jayred365 # run from HOST1 -``` - -> **Critical:** The hostnames in `master.conf` (`HOST1` and `HOST2`) must match the -> Tailscale machine names **exactly** — case sensitive. The ecosystem resolves all -> remote IPs via `tailscale ip -4 HOSTNAME` at runtime. A name mismatch means every -> script that touches the remote will fail at the IP resolution step. - ---- - -## ━━━ STEP 2 — ENABLE SSH ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -unRAID 7.x has SSH disabled by default. Enable it on both servers. - -```bash -# On each server: -# ───────────────────────────────────────────────────────────────────────────── -# Settings → Management Access → Secure Shell -# SSH: Enabled -# SSH port: 22 -# Apply -# ───────────────────────────────────────────────────────────────────────────── -``` - -> SSH is only exposed on your local network and Tailscale interface. Scripts connect -> via Tailscale IP — all traffic is encrypted end-to-end. No ports are opened to -> the public internet. - ---- - -## ━━━ STEP 3 — SSH KEYS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Two sets of keys needed: server-to-server for rsync and failover, and Gitea access -for script repository pull. Generate all keys before configuring anything else. - ---- - -### ── 3a — Server-to-Server Keys ───────────────────────────────────────────── - -```bash -# On HOST1 — generate HOST1's key pair: -ssh-keygen -t ed25519 -f /root/.ssh/Gmer4Lfe-rsync-key -C "gmer4lfe-rsync" -N "" - -# On HOST2 — generate HOST2's key pair: -ssh-keygen -t ed25519 -f /root/.ssh/Jayred365-rsync-key -C "jayred365-rsync" -N "" -``` - ---- - -### ── 3b — Authorise Keys Bidirectionally ───────────────────────────────────── - -```bash -# HOST1's public key must be authorised on HOST2 (so HOST1 can SSH into HOST2): -# ───────────────────────────────────────────────────────────────────────────── - -# On HOST1 — print the public key: -cat /root/.ssh/Gmer4Lfe-rsync-key.pub - -# On HOST2 — create authorized_keys and paste HOST1's public key: -mkdir -p /root/.ssh -echo "PASTE_HOST1_PUBLIC_KEY_HERE" >> /root/.ssh/authorized_keys -chmod 600 /root/.ssh/authorized_keys - -# HOST2's public key must be authorised on HOST1 (so HOST2 can SSH into HOST1): -# ───────────────────────────────────────────────────────────────────────────── - -# On HOST2 — print the public key: -cat /root/.ssh/Jayred365-rsync-key.pub - -# On HOST1 — append HOST2's public key: -echo "PASTE_HOST2_PUBLIC_KEY_HERE" >> /root/.ssh/authorized_keys -``` - ---- - -### ── 3c — Test Both Directions ──────────────────────────────────────────────── - -```bash -# From HOST1 — should print "connected" without a password prompt: -ssh -i /root/.ssh/Gmer4Lfe-rsync-key \ - root@$(tailscale ip -4 unRAID-Jayred365) \ - "echo connected" - -# From HOST2 — should print "connected" without a password prompt: -ssh -i /root/.ssh/Jayred365-rsync-key \ - root@$(tailscale ip -4 unRAID-Gmer4Lfe) \ - "echo connected" -``` - -``` -If prompted for a password: the key was not authorised correctly. - → Recheck Step 3b — the public key content must be on one line - → Check permissions: chmod 600 /root/.ssh/authorized_keys - → Check the key file referenced in the SSH command matches what was generated -``` - ---- - -### ── 3d — Gitea SSH Key ─────────────────────────────────────────────────────── - -```bash -# On BOTH servers — generate a key for Gitea access: -ssh-keygen -t ed25519 -f /root/.ssh/unraid_gitea -C "unraid-gitea" -N "" - -# Print the public key to add to Gitea: -cat /root/.ssh/unraid_gitea.pub - -# In Gitea: Settings → SSH / GPG Keys → Add Key → paste the output above -# Do this for both servers if they have separate Gitea accounts, or once if shared -``` - ---- - -## ━━━ STEP 4 — CLONE THE REPOSITORY ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Both servers clone from the same Gitea repository. Updates pushed to the repo -propagate to both servers automatically via `git_pull_execute.sh` at the start of -each daily maintenance window. - ---- - -### ── On Both Servers ───────────────────────────────────────────────────────── - -```bash -# Create the target directory: -mkdir -p /mnt/user/appdata/unraid_scripts - -# Clone the repository: -GIT_SSH_COMMAND="ssh -i /root/.ssh/unraid_gitea" \ - git clone git@YOUR_GITEA_HOST:FailedProxy/Unraid_Scripts.git \ - /mnt/user/appdata/unraid_scripts - -# Replace YOUR_GITEA_HOST with your Gitea server address and port -# Example: git@192.168.50.2:221 -``` - ---- - -### ── Verify the Structure ───────────────────────────────────────────────────── - -```bash -ls /mnt/user/appdata/unraid_scripts/ -``` - -``` -Expected output: - master.conf ← all user configuration — the only file you edit - master_host1.conf ← HOST1-specific configuration - master_host2.conf ← HOST2-specific configuration - common.sh ← shared library — functions used by all scripts - load_config.sh ← config loader - Orchestrators/ - Rsync/ - Failover/ - Docker_Essentials/ - unRAID_Essentials/ - Media/ - Transcodes/ - Monitors/ - Tools/ - Partnership/ -``` - ---- - -### ── Make Scripts Executable ───────────────────────────────────────────────── - -```bash -# Execute permission on all scripts — required once after clone: -find /mnt/user/appdata/unraid_scripts -name "*.sh" -exec chmod +x {} \; -``` - -> `array_start.sh` auto-fixes permissions on scripts that lost the execute bit — -> but this initial chmod ensures the first run works before that safeguard is active. - ---- - -## ━━━ STEP 5 — CONFIGURE MASTER.CONF ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -All user configuration lives in `master.conf`. Every value with a comment -`# REQUIRED` must be set before the first run. Everything else has working defaults. - -```bash -nano /mnt/user/appdata/unraid_scripts/master.conf -``` - ---- - -### ── Host Identity ───────────────────────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# These must match Tailscale machine names exactly — case sensitive. -# The ecosystem uses these to resolve remote IPs at runtime. -# - HOST1="unRAID-Gmer4Lfe" # REQUIRED — must match tailscale machine name - HOST2="unRAID-Jayred365" # REQUIRED — same - -# SSH key paths — each server's key for authenticating to the other: - HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" # HOST1 uses this to SSH to HOST2 - HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" # HOST2 uses this to SSH to HOST1 -``` - ---- - -### ── Emby API Keys ───────────────────────────────────────────────────────────── - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# Used by: emby_session_report.sh, sunday_morning_coffee_report.sh, -# sonarr/radarr cleanup (notify_emby_scan after deletion) -# -# Get from: Emby Dashboard → Settings → API Keys → + New API Key -# - HOST1_EMBY_URL="http://192.168.50.2:8096" - HOST1_EMBY_API_KEY="your-host1-emby-api-key" # REQUIRED for Emby features - HOST1_EMBY_CONTAINER="Emby" - -# master_host2.conf - HOST2_EMBY_URL="http://localhost:8096" - HOST2_EMBY_API_KEY="your-host2-emby-api-key" - HOST2_EMBY_CONTAINER="Emby" -``` - ---- - -### ── Git Repository ──────────────────────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# Used by git_pull_execute.sh — pulls latest scripts at start of each daily window. -# - GITEA_CONTAINER="Gitea" # exact Docker container name - GITEA_REPO_PATH="FailedProxy/Unraid_Scripts.git" - TARGET_DIR="/mnt/user/appdata/unraid_scripts" - GITEA_SSH_KEY="/root/.ssh/unraid_gitea" - SSH_PORT=221 # your Gitea SSH port -``` - ---- - -### ── Daily Sync Shares ──────────────────────────────────────────────────────── - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# Shares HOST1 is source of truth for — pushed to HOST2 every night at 1am. -# HOST2 treats these as read-only mirrors. Never put the same share in both lists. -# -HOST1_DAILY_SYNC_SHARES=( - "/mnt/user/Movies" # HOST1 manages this — Radarr runs here - "/mnt/user/Tv_Shows" # HOST1 manages this — Sonarr runs here - "/mnt/user/Music" # HOST1 manages this — Lidarr runs here - "/mnt/user/Kids_Movies" - "/mnt/user/Kids_Tv_Shows" - "/mnt/user/Sports" - "/mnt/user/stand-up_comedy" -) - -# master_host2.conf -HOST2_DAILY_SYNC_SHARES=( - "/mnt/user/Anime_Shows" # HOST2 manages this — his Sonarr runs here - "/mnt/user/Anime_Movies" # HOST2 manages this — his Radarr runs here -) -``` - ---- - -### ── Weekly Sync Shares ─────────────────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# Synced during the Sunday 2:30am window — containers stopped both sides. -# Do NOT add these to a separate cron schedule — they run via weekly_sync_maintenance.sh. -# -WEEKLY_SYNC_SHARES=( - "/mnt/user/Media_Server/Emby" # full clean Emby mirror - "/mnt/user/appdata-Failover/Critical-Data" # auth stack clean state -) -``` - ---- - -## ━━━ STEP 6 — MASTER_HOST*.CONF ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Per-host configuration lives in `master_host1.conf` and `master_host2.conf`. -`detect_hosts()` in `common.sh` reads which server is running and aliases the correct -`HOST*_` prefixed variables to their unprefixed names. Scripts only ever reference the -unprefixed name — they work identically on both servers. - -```bash -# master_host1.conf is only sourced on HOST1 -# master_host2.conf is only sourced on HOST2 -# Changes go in the right file for the right server - -nano /mnt/user/appdata/unraid_scripts/master_host1.conf # on HOST1 -nano /mnt/user/appdata/unraid_scripts/master_host2.conf # on HOST2 -``` - -See each conf file's comments — every variable is documented with its purpose -and the reasoning behind the value. - ---- - -## ━━━ STEP 7 — RSYNC PROFILES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Profiles control per-share behaviour — which containers to stop, rsync flags, -bandwidth limits, what to exclude. Profile is matched by directory basename -(lowercased). Override with `--profile=name`. - ---- - -### ── How Profile Matching Works ────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# rsync.sh /mnt/user/appdata-Failover/Arrs_Stack -# basename: Arrs_Stack -# lowercased: arrs_stack -# matched profile: [arrs_stack] -# -# rsync.sh /mnt/user/Movies -# basename: Movies -# lowercased: movies -# no matching profile → global defaults apply (no containers stopped) -# -# rsync.sh /mnt/user/appdata-Failover/Critical-Data --profile=critical-failover -# explicit override → uses [critical-failover] profile regardless of path -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Current Profiles ───────────────────────────────────────────────────────── - -```bash -# master.conf — profile definitions -# ───────────────────────────────────────────────────────────────────────────── -# Each profile defines which containers to stop, rsync flags, excludes, etc. -# Containers in PROFILE_CRITICAL_CONTAINER_NAMES are stopped on BOTH servers. -# PROFILE_DELAYED_CONTAINERS restart after PROFILE_CONTAINER_DELAY seconds. - -# ── arrs_stack ───────────────────────────────────────────────────────────── -# Arr databases — stopped for clean SQLite snapshot - PROFILES["arrs_stack_CRITICAL_CONTAINER_NAMES"]=( - "Sonarr" "Radarr" "Lidarr" "Prowlarr" "Bazarr" "Pinchflat" - ) - -# ── critical-data ────────────────────────────────────────────────────────── -# Auth stack — stopped for clean database snapshot, delayed restart - PROFILES["critical-data_CRITICAL_CONTAINER_NAMES"]=( - "Mariadb-Authelia" "Redis-Authelia" - "NginxProxyManager" "Lldap-Gmer4Lfe" - ) - PROFILES["critical-data_DELAYED_CONTAINERS"]=( - "Authelia" "Authelia-Secondary" # auth services restart after delay - ) - PROFILES["critical-data_CONTAINER_DELAY"]=30 # seconds before delayed containers start - -# ── important-data ───────────────────────────────────────────────────────── -# NextCloud + Postgres — stopped for clean snapshot - PROFILES["important-data_CRITICAL_CONTAINER_NAMES"]=( - "Postgres-NextCloud" - ) - PROFILES["important-data_DELAYED_CONTAINERS"]=("NextCloud") - -# ── emby ─────────────────────────────────────────────────────────────────── -# Weekly full clean sync — both Emby instances stopped - PROFILES["emby_CRITICAL_CONTAINER_NAMES"]=("Emby") - PROFILES["emby_EXCLUDE_DIRS"]=( - "transcodes/" "logs/" "crash*" "cache/" - ) - -# ── emby-failover ────────────────────────────────────────────────────────── -# Every 30 minutes, Emby STAYS RUNNING — dirty sync of critical state only -# WAL and SHM excluded — safe to copy while Emby is writing - PROFILES["emby-failover_CRITICAL_CONTAINER_NAMES"]=() # empty — nothing stops - PROFILES["emby-failover_EXCLUDE_DIRS"]=( - "*.wal" "*.shm" # WAL files — unsafe mid-write - "transcodes/" "logs/" "crash*" "cache/" # volatile data — skip - ) - PROFILES["emby-failover_REMOTE_RESTART_CONTAINERS"]=("Emby") - # Emby on HOST2 restarts after sync to pick up config changes -``` - ---- - -### ── Two Emby Profiles — Why Both Exist ────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# emby-failover — every 30 minutes, Emby stays running: -# What syncs: users.db, library.db, authentication.db, config/ -# What skips: *.wal *.shm transcodes/ logs/ cache/ -# Why: WAL files are being written while Emby runs — copying them -# would produce a corrupt database on HOST2 -# Result: HOST2 is always within 30 minutes of HOST1 on watch state -# and user activity. Failover is seamless — nobody notices. -# -# emby — Sunday 2:30am, both Emby instances stopped: -# What syncs: everything except transcodes, logs, cache, crash files -# What includes: metadata, plugins, full database state, all config -# Why: WAL is checkpointed on clean shutdown — safe to copy everything -# Full consistent mirror including metadata and plugin state -# Result: HOST2 has a gold-standard Emby state once per week -# Image cache warm for 6 days — only reset Sunday when users sleep -# -# The two profiles work together: -# emby-failover: keeps HOST2 current on what matters for immediate failover -# emby: gives HOST2 full fidelity once per week -# Neither alone is sufficient — both are needed. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -## ━━━ STEP 8 — PERSONAL ENCRYPTED SHARES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Personal shares are synced to the remote server for offsite backup. ZFS encrypts at -the dataset level — the remote server receives encrypted blocks and cannot read the -content without your passphrase or keyfile. - ---- - -### ── Create an Encrypted ZFS Dataset ──────────────────────────────────────── - -```bash -# In the unRAID UI: -# ───────────────────────────────────────────────────────────────────────────── -# Main → click your ZFS pool name → + Dataset -# Name: Gmer4Lfe-Personal -# Encryption: Enabled -# Passphrase: [your passphrase] -# ⚠️ Write your passphrase down — if lost, data is completely unrecoverable -# -# Settings → Shares → Add Share -# Share path: point to the new encrypted dataset -# Use cache: Only — keeps data on ZFS pool, not array -# ───────────────────────────────────────────────────────────────────────────── - -# Verify encryption is active before syncing: -zfs get encryption poolname/Gmer4Lfe-Personal -# Should show: encryption aes-256-gcm -``` - ---- - -### ── Auto-Unlock on Boot (Optional) ───────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Keyfile approach — passphrase stored in a file, loaded at boot. -# More convenient but the keyfile is a secret that must be protected. -# Never sync the keyfile to the remote server. -# -# Create keyfile — on HOST1 only: -dd if=/dev/urandom bs=32 count=1 | base64 > /root/.zfs-keys/personal.key -chmod 600 /root/.zfs-keys/personal.key - -# Set dataset to use keyfile instead of passphrase: -zfs change-key \ - -o keylocation=file:///root/.zfs-keys/personal.key \ - -o keyformat=raw \ - poolname/Gmer4Lfe-Personal - -# Add to ramdisk_setup.sh or array_start.sh custom scripts: -zfs load-key poolname/Gmer4Lfe-Personal -zfs mount poolname/Gmer4Lfe-Personal -# ───────────────────────────────────────────────────────────────────────────── -# Manual unlock alternative (most secure — passphrase only in your head): -zfs load-key poolname/Gmer4Lfe-Personal # prompts for passphrase -zfs mount poolname/Gmer4Lfe-Personal -``` - ---- - -### ── Add to master_host1.conf ──────────────────────────────────────────────── - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# Personal shares append to the daily sync after DAILY_SYNC_SHARES. -# Remote server receives encrypted blocks — cannot read content without your key. -# -HOST1_PERSONAL_SHARES=( - "/mnt/user/Gmer4Lfe-Personal" -) -``` - ---- - -## ━━━ STEP 9 — USER SCRIPTS SETUP ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -The ecosystem is designed so the User Scripts plugin has only a small number of entries — -each one an orchestrator. Individual scripts are never scheduled directly. - ---- - -### ── At Startup of Array ───────────────────────────────────────────────────── - -```bash -# Create one script entry named "array start": -# ───────────────────────────────────────────────────────────────────────────── -#!/bin/bash -bash /mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh -# ───────────────────────────────────────────────────────────────────────────── -# Schedule: At Startup of Array -# Run as: Background Task -# -# This is the ONLY "At Startup of Array" entry needed. -# It launches everything in ARRAY_START_SCRIPTS from master.conf: -# inotify_tuning.sh — raise inotify limits before containers start -# docker_syslog_filter.sh — suppress veth log noise -# php_fpm_max_children.sh — WebGUI tuning -# ramdisk_setup.sh — create ramdisk before Emby starts -# docker_network_connect.sh — connect containers to extra networks -# system_watchdog.sh — continuous system health monitor -# docker_watchdog.sh — continuous container health monitor -# failover.sh — continuous mutual failover -``` - ---- - -### ── Cron Schedule ──────────────────────────────────────────────────────────── - -```bash -# Create one script entry per cron schedule below. -# All entries: Run as Background Task -# ───────────────────────────────────────────────────────────────────────────── - -# Every 3 minutes — transcode cleanup + manager: -*/3 * * * * -bash /mnt/user/appdata/unraid_scripts/Orchestrators/transcode_management.sh - -# Every 30 minutes — Emby dirty sync (watch states, library delta): -*/30 * * * * -bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ - /mnt/user/Media_Server/Emby --profile=emby-failover - -# Every 6 hours — failed import + stalled download recovery: -0 */6 * * * -bash /mnt/user/appdata/unraid_scripts/Orchestrators/arrs_failed_stalled_recovery.sh - -# 1am daily — full maintenance window: -# git pull → rsync all shares → permissions → cleaners → arr cleanup → docker restart -0 1 * * * -bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh - -# 2:30am Sunday — weekly maintenance window: -# stop containers → pull updates → clean sync → start containers → weekly restarts -30 2 * * 0 -bash /mnt/user/appdata/unraid_scripts/Orchestrators/weekly_sync_maintenance.sh - -# 8am daily — health digest (DIGEST_PROFILE in master.conf controls when it notifies): -0 8 * * * -bash /mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh - -# Every 6 hours — inotify + php-fpm utilisation snapshot: -0 */6 * * * -bash /mnt/user/appdata/unraid_scripts/Monitors/system_tuning_monitor.sh - -# Sunday morning — weekly reports: -0 6 * * 0 bash .../Monitors/zfs_memory_snapshot.sh -0 7 * * 0 bash .../Monitors/smart_health.sh -0 9 * * 0 bash .../Monitors/cert_monitor.sh -0 10 * * 0 bash .../Monitors/backup_verify.sh -0 11 * * 0 bash .../Monitors/emby_session_report.sh -0 11 * * 0 bash .../Monitors/bandwidth_monitor.sh --report -# ───────────────────────────────────────────────────────────────────────────── -``` - -> **Set all entries to "Background Task"** — output streams correctly to the User -> Scripts log rather than buffering in the browser tab. Non-background tasks can -> appear to hang on long-running scripts. - ---- - -## ━━━ STEP 10 — VERIFY THE SETUP ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Before relying on scheduled jobs, test manually from the terminal on HOST1. -Test with `--dry-run` first — no changes made, but the full pre-flight and -configuration resolution runs. - ---- - -### ── Test a Single Profile Sync ───────────────────────────────────────────── - -```bash -# Dry run with verbose output — shows every decision the script makes: -bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ - /mnt/user/appdata-Failover/Arrs_Stack --dry-run --log -``` - -``` -Expected output (healthy): - ━━━ ⚙️ Setup ━━━ - Host: HOST1 (unRAID-Gmer4Lfe) → HOST2 (unRAID-Jayred365) - Remote IP: 100.x.x.x - Profile: arrs_stack - - ━━━ 🛡️ Pre-flight Checks ━━━ - ✅ Remote reachable - ✅ version parity — both on unRAID X.Y.Z - ✅ Remote Docker daemon responding - ✅ Remote rootfs: 12% (threshold: 75%) - ✅ Remote share exists and not empty - ✅ All pre-flight checks passed -``` - ---- - -### ── Test the Daily Orchestrator ───────────────────────────────────────────── - -```bash -# Dry run of the full daily window — shows every job that would run: -bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh --dry-run -``` - -``` -If any pre-flight check fails, the script aborts with a clear error message -before touching anything. Fix the reported issue and re-run --dry-run. - -Common pre-flight failures and their causes: - "Remote not reachable" → Tailscale not connected on HOST2 - "Version mismatch" → different unRAID versions — update before syncing - "Remote rootfs above X%" → HOST2's root filesystem nearly full - "Remote share missing" → share doesn't exist on HOST2 yet (see Step 11) - "Docker daemon not responding" → HOST2's Docker service not started -``` - ---- - -### ── Check the Configuration Resolved Correctly ────────────────────────────── - -```bash -# --status shows how master.conf resolved for this server and profile: -bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ - /mnt/user/appdata-Failover/Arrs_Stack --status -``` - ---- - -## ━━━ STEP 11 — INITIAL HOST2 SYNC ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -If HOST2 is being set up from scratch with empty shares: - ---- - -### ── Create Share Structure on HOST2 ──────────────────────────────────────── - -```bash -# On HOST2 — start the array and create shares via the unRAID UI. -# Or use the share recreation tool to create disk directories from HOST1's cfg files: -bash /mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh -``` - ---- - -### ── Initial Push From HOST1 ───────────────────────────────────────────────── - -```bash -# On HOST1 — push all shares to HOST2 for the first time: -# Use --log for verbose output on first run -bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh --log -``` - -``` -First run may take several hours for large libraries — this is normal. -The scheduled nightly sync will be incremental after the initial push. -Progress shows per-share throughout. -``` - ---- - -## ━━━ NAMING CONSISTENCY — THIS IS REQUIRED ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -The ecosystem uses one codebase on both servers. This only works if containers and -shares have identical names on both servers. This is not configurable — it is a -design requirement. - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Container names must match exactly on both servers: - "Emby" ← both HOST1 and HOST2 - "NginxProxyManager" ← both HOST1 and HOST2 - "Mariadb-Authelia" ← both HOST1 and HOST2 - -# Share paths must match exactly on both servers: - /mnt/user/Movies ← both HOST1 and HOST2 (HOST2 has a mirror) - /mnt/user/Tv_Shows ← both HOST1 and HOST2 -# ───────────────────────────────────────────────────────────────────────────── -# If a container has a different name on one server: the script skips it -# without error. It silently does the wrong thing. You only notice when -# the container is not stopped during a sync that requires it to stop. -# -# If a share has a different path: rsync.sh aborts with "remote share missing". -# Easier to catch — but still requires renaming the share to fix. -# -# Keep names consistent and one codebase covers both servers automatically. -# Diverge and every script that touches containers or shares needs custom logic. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -## ━━━ REPOSITORY STRUCTURE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -``` -Unraid_Scripts/ -├── master.conf ← All shared configuration — edit this file -├── master_host1.conf ← HOST1-specific configuration -├── master_host2.conf ← HOST2-specific configuration -├── common.sh ← Shared library — functions used by all scripts -├── load_config.sh ← Config loader — sources all conf files -│ -├── Orchestrators/ -│ ├── array_start.sh ← Single "At Startup of Array" entry point -│ ├── daily_sync_maintenance.sh ← 1am daily window orchestrator -│ ├── weekly_sync_maintenance.sh ← Sunday 2:30am window orchestrator -│ ├── critical_sync_maintenance.sh ← Every 15 minutes — critical sync + partnership -│ ├── media_management.sh ← Permissions + cleaners + arr cleanup -│ ├── transcode_management.sh ← Transcode cleanup then manager -│ └── arrs_failed_stalled_recovery.sh ← Failed import + stalled download recovery -│ -├── Rsync/ -│ └── rsync.sh ← Core rsync script — called per share -│ -├── Failover/ -│ ├── failover.sh ← Mutual container failover — continuous loop -│ ├── failover_test.sh ← Controlled iptables failover simulation -│ └── failover_state_reset.sh ← Reset failover state manually -│ -├── Docker_Essentials/ -│ ├── docker_watchdog.sh ← Two-tier container monitor — continuous loop -│ ├── docker_daily_restart.sh ← Nightly container restarts -│ ├── docker_weekly_restart.sh ← Weekly container restarts -│ ├── docker_network_connect.sh ← Ensure networks + connections at array start -│ └── watchdog_skip_list_manager.sh ← Skip list inspection and recovery -│ -├── unRAID_Essentials/ -│ ├── system_watchdog.sh ← Three-tier system health monitor — continuous -│ ├── ramdisk_setup.sh ← Creates ramdisk + symlink at array start -│ ├── inotify_tuning.sh ← Raise inotify limits at array start -│ ├── docker_syslog_filter.sh ← Suppress veth log noise -│ ├── php_fpm_max_children.sh ← WebGUI performance tuning -│ ├── server_reboot.sh ← Graceful reboot with pre-flight warnings -│ ├── mover_stop.sh ← Stop mover cleanly with wall warning -│ ├── clear_logs.sh ← Size-threshold log cleanup -│ ├── webgui_restart.sh ← nginx → php-fpm → emhttp escalation -│ └── git_pull_execute.sh ← Pull latest scripts from Gitea -│ -├── Media/ -│ ├── media_shares_permissions.sh ← Apply permissions to media shares -│ ├── media_cleaner.sh ← Remove junk files from media shares -│ ├── lidarr_cleanup.sh ← Remove orphaned music files (HOST1 only) -│ ├── sonarr_cleanup.sh ← Remove orphaned TV files (host-aware) -│ └── radarr_cleanup.sh ← Remove orphaned movie files (host-aware) -│ -├── Transcodes/ -│ ├── transcode_manager.sh ← Ramdisk/SSD symlink management -│ └── transcode_cleanup.sh ← Remove stale segment files -│ -├── Monitors/ -│ ├── cert_monitor.sh ← SSL cert expiry via live TLS connection -│ ├── backup_verify.sh ← rsync mirror MD5 checksum verification -│ ├── smart_health.sh ← Drive SMART attribute monitoring -│ ├── zfs_memory_snapshot.sh ← ZFS health + ARC + memory report -│ ├── bandwidth_monitor.sh ← rsync transfer logging + weekly report -│ ├── weekly_health_digest.sh ← Full ecosystem health aggregation -│ ├── emby_session_report.sh ← Emby streaming usage statistics -│ ├── system_tuning_monitor.sh ← inotify + php-fpm utilisation tracking -│ └── continuous_scripts_status.sh ← Live dashboard for background processes -│ -├── Partnership/ -│ └── partnership_manage.sh ← Two-server relationship lifecycle manager -│ -└── Tools/ - ├── recreate_shares.sh ← Create share directories from cfg files - ├── bulk_permissions_repair.sh ← One-shot permission repair - ├── rsync_stop.sh ← Stop active rsync jobs cleanly - ├── user_scripts_stop.sh ← Stop running user script processes - ├── server_reboot.sh ← Graceful scheduled reboot - ├── zfs_pool_scrub.sh ← Trigger ZFS pool scrub - └── container_data_export.sh ← Export container configuration -``` - ---- - -## ━━━ TROUBLESHOOTING ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - ---- - -### 🔴 SSH Connection Refused / Timeout - -```bash -# Verify SSH is enabled on the remote: -# Settings → Management Access → Secure Shell → Enabled - -# Verify Tailscale is connected: -tailscale ip -4 unRAID-Jayred365 # should return 100.x.x.x - -# Test SSH manually with the key: -ssh -i /root/.ssh/Gmer4Lfe-rsync-key root@$(tailscale ip -4 unRAID-Jayred365) "hostname" -# Expected: unRAID-Jayred365 -# If password prompted: key not authorised — recheck Step 3b -``` - ---- - -### 🔴 Pre-flight Aborts on Remote Rootfs - -```bash -# Remote rootfs above ROOTFS_WARN threshold -# Check current usage on remote: -ssh -i /root/.ssh/Gmer4Lfe-rsync-key root@[HOST2-ip] "df /" - -# Common cause: array not started, drives not mounted -# Verify array is started on HOST2 before running syncs -``` - ---- - -### 🔴 Remote Share Missing - -```bash -# Share exists locally but not on remote -# Verify the share exists on HOST2: -ssh -i /root/.ssh/Gmer4Lfe-rsync-key root@[HOST2-ip] "ls /mnt/user/" - -# If missing — create the share on HOST2 first, then run initial sync (Step 11) -# Or run recreate_shares.sh on HOST2 to create directories from HOST1's cfg files -``` - ---- - -### 🔴 Containers Not Stopping / Starting - -```bash -# Verify container names in master_host*.conf match Docker exactly — case sensitive -# Check what Docker actually calls the container: -docker ps --format "{{.Names}}" - -# Test Docker commands to remote manually: -ssh -i /root/.ssh/Gmer4Lfe-rsync-key root@[HOST2-ip] "docker ps --format '{{.Names}}'" -``` - ---- - -### 🔴 Profile Not Matching - -```bash -# Profile key = directory basename lowercased -# /mnt/user/appdata-Failover/Arrs_Stack → basename: Arrs_Stack → key: arrs_stack - -# Override explicitly if basename doesn't match a profile name: -bash rsync.sh /mnt/user/appdata-Failover/My_Stuff --profile=arrs_stack - -# Verify what profile resolved for a path: -bash rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --status -``` - ---- - -### 🔴 Script Not Found - -```bash -# Verify repo was cloned to the correct location: -ls /mnt/user/appdata/unraid_scripts/master.conf - -# Make scripts executable: -find /mnt/user/appdata/unraid_scripts -name "*.sh" -exec chmod +x {} \; -``` - ---- - -## ━━━ AVAILABLE FLAGS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -All scripts support these flags. Use `--dry-run` before any live operation. - -```bash -# ───────────────────────────────────────────────────────────────────────────── - --dry-run run without making any changes — pre-flight still runs ✅ - --log verbose output — show every decision made - --status show resolved configuration and exit — no rsync, no sync - --no-log suppress verbose output (some scripts) - -# Examples: -bash rsync.sh /mnt/user/Movies --dry-run --log # preview a media sync -bash rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --status # check profile resolution -bash daily_sync_maintenance.sh --dry-run # preview full daily window -bash docker_watchdog.sh --status # check watchdog state -# ───────────────────────────────────────────────────────────────────────────── -``` \ No newline at end of file diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index 6fdd217..c1e7327 100644 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -1,64 +1,133 @@ #!/bin/bash # ============================================================================================== -# ================================= Rsync Core Script ========================================== +# ============================= Rsync Core Script ============================================== # ============================================================================================== -# Core rsync script — called per share or per appdata profile. -# Called by orchestrators (daily/weekly/critical sync) and directly for manual syncs. # -# ── PROFILE SYSTEM ──────────────────────────────────────────────────────────────────────────── -# Profile is inferred from the directory basename (lowercased). -# Override with --profile=name for explicit profile selection. -# If no profile match found → all settings fall back to global defaults in master.conf. +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Core rsync engine for the two-server ecosystem. Called per share or per +# appdata profile by orchestrators (daily_sync_maintenance, weekly_sync_maintenance, +# critical_sync_maintenance) and directly for manual or scheduled dirty syncs. # -# Profiles define: -# PROFILE_RSYNC_OPTS — rsync flags (does NOT inherit DEFAULT_RSYNC_OPTS) -# PROFILE_BW_LIMIT — bandwidth limit in KB/s -# PROFILE_RETRY_COUNT — retry attempts before giving up -# PROFILE_SLEEP — seconds between retry attempts -# PROFILE_CRITICAL_CONTAINER_NAMES — containers stopped both sides before sync -# PROFILE_DELAYED_CONTAINERS — containers needing delay before starting after sync -# PROFILE_CONTAINER_DELAY — seconds before starting delayed containers -# PROFILE_EXCLUDE_DIRS — paths excluded from transfer +# Profile is inferred from the directory basename (lowercased). Override with +# --profile=name for explicit selection. If no profile matches, global defaults +# from master.conf apply and no containers are stopped. +# +# After each sync, logs transfer data to bandwidth_monitor.sh for the weekly +# bandwidth report. Silent on success — only failures produce visible output. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Profiles define per-share behavior: +# PROFILE_CRITICAL_CONTAINER_NAMES — containers stopped on both servers before sync +# PROFILE_DELAYED_CONTAINERS — containers with a delay before restart after sync +# PROFILE_CONTAINER_DELAY — seconds before delayed containers start +# PROFILE_RSYNC_OPTS — rsync flags (does not inherit DEFAULT_RSYNC_OPTS) +# PROFILE_BW_LIMIT — bandwidth limit in KB/s +# PROFILE_RETRY_COUNT — retry attempts on failure +# PROFILE_SLEEP — seconds between retry attempts +# PROFILE_EXCLUDE_DIRS — paths excluded from transfer # PROFILE_REMOTE_RESTART_CONTAINERS — containers restarted on remote after dirty sync -# (critical-fallback, emby-fallback profiles) # Was running → restart. Was stopped → leave stopped. # -# ── RSYNC ENABLE/DISABLE ────────────────────────────────────────────────────────────────────── -# Two-tier toggle system — checked at entry: -# Tier 1: RSYNC_ENABLED=false → all rsync stops -# Tier 2: Per-orchestrator flag (DAILY_RSYNC_ENABLED etc.) — checked by caller -# Direct calls to rsync.sh only check Tier 1 +# Two-tier rsync enable/disable: +# Tier 1: RSYNC_ENABLED=false → all rsync stops immediately (checked by this script) +# Tier 2: per-orchestrator flag (DAILY_RSYNC_ENABLED etc.) → checked by caller # -# ── BANDWIDTH LOGGING ───────────────────────────────────────────────────────────────────────── -# After each sync logs to bandwidth_monitor.sh --log-transfer: -# profile | duration_seconds | status | bytes_transferred -# Bytes captured from rsync --stats output — version-proof parsing. -# bandwidth_monitor.sh flags syncs exceeding BANDWIDTH_WARN_GB. +# Bandwidth logging: after each sync, logs profile/duration/status/bytes to +# bandwidth_monitor.sh --log-transfer. Bytes captured from rsync --stats via awk +# using version-stable field names. # -# ── DIRTY SYNC REMOTE RESTART ───────────────────────────────────────────────────────────────── -# Profiles using dirty sync (critical-fallback, emby-fallback) define -# PROFILE_REMOTE_RESTART_CONTAINERS — containers restarted on remote after sync completes. -# This ensures the remote picks up config changes synced during the dirty window. -# Was running → restart. Was stopped → leave stopped. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# check_rsync_enabled() — Tier 1 gate before any operation -# check_unraid_version_parity — refuses if servers on incompatible unRAID versions -# check_remote_docker_daemon — verifies remote daemon before container operations -# check_local_disk_temps() — temp check before transfer (exit 1=skip, 2=abort all) -# check_connectivity() — verifies remote reachable -# check_remote_rootfs() — aborts if remote rootfs nearly full -# check_remote_share() — aborts if target directory missing or empty -# check_remote_disks() — verifies all backing disks online on remote -# acquire_rsync_lock() — per-profile lock + global concurrent limit -# validate_unraid_cmd — notify validated before use -# Silent by default — only failures produce visible output +# Global Rsync Gate +# check_rsync_enabled() — RSYNC_ENABLED=false exits cleanly before any operation. +# +# Partnership Blocklist +# Refuses to sync if REMOTE_SERVER_NAME appears in the partnership blocklist. +# Written at offboard — prevents stale access after a partnership ends. +# +# Version Parity +# check_unraid_version_parity — refuses sync if servers on incompatible unRAID versions. +# +# Remote Health Pre-flights +# check_connectivity() — Tailscale IP reachable before any SSH +# check_remote_rootfs() — aborts if remote rootfs exceeds ROOTFS_WARN_PCT +# check_remote_share() — aborts if target directory missing or empty on remote +# check_remote_disks() — verifies all backing disks online on remote +# +# Drive Temperature Check +# check_local_disk_temps() — runs before any transfer. Exit 1 = skip this profile, +# exit 2 = abort all remaining syncs (CRITICAL temperature). +# +# Remote Docker Daemon Check +# check_remote_docker_daemon — verified before any container stop/start operations. +# If daemon unresponsive: container operations skipped, rsync proceeds without stopping. +# +# Per-Profile Concurrency Lock +# acquire_rsync_lock() — per-profile lock prevents parallel runs of the same profile. +# Global concurrent limit prevents too many simultaneous rsync processes. +# +# Notification Validated +# validate_unraid_cmd confirms the notify script is present before use. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# RSYNC_ENABLED +# Global on/off toggle for all rsync operations. (default: true) +# +# DEFAULT_RSYNC_OPTS +# Base rsync flags for unproiled shares. Note: --delete is intentionally absent — +# media shares spread files only, arr cleanup scripts own deletions. Profile-specific +# opts set --delete explicitly where needed. +# +# BW_LIMIT +# Default bandwidth cap in KB/s when no PROFILE_BW_LIMIT is set. (default: 0 = unlimited) +# +# RETRY_COUNT +# Default retry attempts on rsync failure. (default: 3) +# +# SLEEP +# Default seconds between retry attempts. (default: 60) +# +# ROOTFS_WARN_PCT +# Abort threshold for remote rootfs percentage full. (default: 75) +# +# PROFILES["profile_KEY"] +# Profile definitions — one entry per PROFILE_* key per profile name. +# See OPERATIONAL MODEL above for all supported keys. +# +# BANDWIDTH_LOG / BANDWIDTH_WARN_GB +# Shared with bandwidth_monitor.sh — set once, used by both. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# rsync.sh /path/to/share +# Sync the given path to the remote. Profile inferred from directory basename. +# +# rsync.sh /path/to/share --profile=name +# Sync with explicit profile override — bypasses basename inference. +# +# rsync.sh /path/to/share --dry-run +# Run all pre-flight checks and show what rsync would transfer. No transfer, +# no container stops. +# +# rsync.sh /path/to/share --status +# Show resolved profile, remote identity, and configuration. Then exit. +# +# rsync.sh /path/to/share --log +# Verbose output throughout — every decision logged. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# rsync.sh /mnt/user/Movies — media share, global defaults -# rsync.sh /mnt/user/appdata-Fallback/Arrs_Stack — matched to [arrs_stack] profile -# rsync.sh /mnt/user/appdata-Fallback/Arrs_Stack --dry-run --log -# rsync.sh /mnt/user/appdata-Fallback/Critical-Data --profile=critical-fallback # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Tools/Manual-Tools.md b/Tools/Manual-Tools.md new file mode 100644 index 0000000..edd42ed --- /dev/null +++ b/Tools/Manual-Tools.md @@ -0,0 +1,600 @@ +# ━━━━━ TOOLS — Manual ━━━━━ + +Configuration reference, usage procedures, and field guides for every script +in `Tools/`. Run any script with `--status` first — it shows current state before +making any changes. + +--- + +## ━━━ CONTENTS ━━━ + +- [failover_state_reset.sh](#failover_state_resetsh) +- [watchdog_skip_list_manager.sh](#watchdog_skip_list_managersh) +- [bulk_permissions_repair.sh](#bulk_permissions_repairsh) +- [container_data_export.sh](#container_data_exportsh) +- [emby_database_repair.sh](#emby_database_repairsh) +- [zfs_pool_scrub.sh](#zfs_pool_scrubsh) +- [recreate_shares.sh](#recreate_sharessh) +- [continuous_scripts_status.sh](#continuous_scripts_statussh) +- [claude_startup.sh](#claude_startupsh) +- [Adding a New Tool](#adding-a-new-tool) + +--- + +## failover_state_reset.sh + +Resets the fallback state file to NORMAL and clears all tier flags. State file only — +does NOT start or stop any containers. + +### When to Use + +``` +After failover_test.sh didn't complete cleanly + → state left in FALLBACK but containers are actually back to normal + +After a failed handback + → state shows FALLBACK but remote is back up and containers are split + +After killing fallback.sh directly (not via User Scripts Abort) + → state is unknown, cycle was interrupted mid-operation + +After a dev/debug session + → state left in a non-NORMAL state from testing +``` + +### Verify Before Resetting + +Run `--status` first and check each of these before writing: + +```bash +# Right containers on right server? +continuous_scripts_status.sh # shows failover current state + +# DDNS pointing correctly? +nslookup Gmer4Lfe.com # confirm it resolves to the right IP + +# fallback.sh not running? +pgrep -f "fallback.sh" # empty output = not running + +# Both servers Tailscale connected? +tailscale status # both hosts should show active +``` + +Resetting during an actual failover causes fallback.sh to think everything is normal +and stop covering the remote — services go offline until the next detection cycle. + +### Usage + +```bash +fallback_state_reset.sh --status # show current state file — always check first +fallback_state_reset.sh --dry-run # show what would be written, no write +fallback_state_reset.sh # interactive reset — prompts for YES to confirm +fallback_state_reset.sh --force # non-interactive — for scripts, no terminal +``` + +### What Gets Written + +```bash +# New state file after reset: +state=NORMAL +fallback_start=0 +handback_strikes=0 +tier2_started=false +tier3_started=false +tier4_started=false +``` + +--- + +## watchdog_skip_list_manager.sh + +View and manage the persistent container skip list used by `docker_watchdog.sh`. + +### When to Use + +``` +docker_watchdog.sh restarts the same container N times within the rolling window +→ container added to skip list on /boot/config/ +→ critical notification sent +→ watchdog stops touching it entirely + +You fix the underlying problem (database, config, dependencies). +You need to clear the container from the skip list so monitoring resumes. +``` + +### Recovery Workflow + +```bash +# 1. Understand the situation — always start here: +watchdog_skip_list_manager.sh --status +# Shows: skip list contents, which are running vs. stopped, restart history + +# 2. Fix the underlying problem first +# Check logs: docker logs ContainerName --tail 100 +# Check disk: df -h /mnt/user +# Check db: docker exec ContainerName sqlite3 /path/to.db ".tables" + +# 3. Clear from skip list + restart history: +watchdog_skip_list_manager.sh --clear ContainerName + +# 4. Start the container manually — confirm your fix worked: +docker start ContainerName + +# 5. Watchdog resumes normal monitoring on next cycle — no further action needed +``` + +### State Files Managed + +```bash +# Both live on /boot/config — survive reboots intentionally. +# A container that was skip-listed before a reboot is still broken after it. + +$SYS_WATCHDOG_FAILED_FILE # persistent skip list +$WATCHDOG_CONTAINER_RESTART_LOG # restart loop tracking +``` + +### Configuration (master.conf) + +```bash +WATCHDOG_CONTAINER_RESTART_LIMIT=3 # restarts before skip-listing +WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours +``` + +### Usage + +```bash +watchdog_skip_list_manager.sh # show status (default) +watchdog_skip_list_manager.sh --status # explicit status +watchdog_skip_list_manager.sh --clear ContainerName # clear specific + restart history +watchdog_skip_list_manager.sh --clear ContainerName --force # no confirmation prompt +watchdog_skip_list_manager.sh --clear-all # clear everything +watchdog_skip_list_manager.sh --clear-all --force # non-interactive +watchdog_skip_list_manager.sh --dry-run # preview any clear action +``` + +--- + +## bulk_permissions_repair.sh + +Applies correct ownership and permissions to specific paths. Faster than running +`media_shares_permissions.sh` which processes every configured share — use this when +you know exactly what needs fixing and don't want to wait for a full library walk. + +### When to Use + +``` +Admin copy left root:root files — scp, cp, direct file transfer +New share needs permissions now — can't wait for nightly run +Container wrote as root — before PUID/PGID was fixed +Specific directory has wrong perms — targeted fix, not a full library walk +``` + +Use the full `media_shares_permissions.sh` instead for: +- Regular nightly maintenance (already scheduled in daily_sync_maintenance.sh) +- After confirming a container's PUID/PGID is now correct +- Initial permissions setup on a new server + +### Diagnosing High Wrong-Owner Counts + +The script counts files with wrong ownership before applying the fix. A high count +on a share that was recently written means a container has wrong PUID/PGID. + +```bash +# Fix: add to the container's Docker template: +PUID=99 +PGID=100 + +# Common culprits writing as root: +# SABnzbd, qBittorrent, slskd — check each one's Docker env vars +``` + +### Configuration (master.conf) + +```bash +PERMISSIONS_OWNER="nobody:users" # matches PUID=99 PGID=100 +PERMISSIONS_DIR_MODE="755" # directories — enter, list, no world-write +PERMISSIONS_FILE_MODE="664" # files — owner+group rw, others read-only +``` + +### Usage + +```bash +# Single path: +bulk_permissions_repair.sh /mnt/user/Movies + +# Multiple paths — all corrected in one run: +bulk_permissions_repair.sh /mnt/user/Movies /mnt/user/Tv_Shows /mnt/user/Music + +# Dry run first — shows count of files with wrong ownership per path: +bulk_permissions_repair.sh /mnt/user/Movies --dry-run + +# Verbose — show each corrected file: +bulk_permissions_repair.sh /mnt/user/Movies --log +``` + +--- + +## container_data_export.sh + +Exports a container's appdata directory to a compressed tar archive. Stops the +container first for a clean consistent backup, verifies the archive after creation, +then restarts the container. + +### When to Use + +``` +Before major container updates — especially "database migration — no rollback" changelogs +Before pool migrations — clean backup before moving appdata to a new pool +Before removing a container from the stack — archive its data before deletion +Manual point-in-time backup before risky config changes +``` + +### Export Sequence + +``` +1. Space check + Estimates required space from appdata size × 1.1 + Aborts if output directory doesn't have enough free space + Container is NOT stopped until the space check passes + +2. Stop container cleanly + docker stop ContainerName — graceful shutdown + +3. Create archive + tar -czf ContainerName_YYYY-MM-DD_HH-MM.tar.gz /path/to/appdata + +4. Verify archive integrity + tar --test-file archive.tar.gz — confirms archive is valid and complete + If verification fails → restart container anyway, report error + +5. Restart container + docker start ContainerName — always happens, even if archiving failed +``` + +### Usage + +```bash +# Syntax: container_data_export.sh ContainerName AppDataPath OutputDir + +# Emby backup: +container_data_export.sh \ + Emby \ + /mnt/media-servers/Media_Server/Emby \ + /mnt/user/Backups/ + +# Dry run — verify space and paths without stopping anything: +container_data_export.sh \ + Emby \ + /mnt/media-servers/Media_Server/Emby \ + /mnt/user/Backups/ \ + --dry-run + +# Output filename: Emby_2026-05-14_02-30.tar.gz +# Timestamped — safe to run multiple times, no overwrite +``` + +--- + +## emby_database_repair.sh + +Stops Emby, runs SQLite `PRAGMA integrity_check` on every Emby database, and restarts. +Reports per-database — does NOT automatically repair. Recovery requires judgment. + +### When to Use + +``` +Emby logs show database errors → run this first +Emby crashing repeatedly with no clear cause → likely database corruption +Playback history or user data behaving strangely → users.db or library.db issue +After a hard shutdown or power loss with Emby running → check for WAL corruption +``` + +### Recovery Guide by Database + +``` +library.db — media library metadata: titles, seasons, episodes, artwork + CORRUPT → safe to delete — Emby fully rebuilds from media files on next start + Rebuild takes time (hours on large libraries) but loses nothing permanent + +users.db — user accounts, watch history, playback positions, settings + CORRUPT → deleting resets ALL user accounts and watch history + Check for a recent backup (weekly_sync_maintenance.sh mirrors Emby/) + before deleting — restore from remote if available + +authentication.db — API keys, session tokens + CORRUPT → safe to delete — API keys regenerated on restart + Any connected clients will need to re-authenticate once + +activity.db — activity/access log + CORRUPT → safe to delete — it's a log, losing it is acceptable + +library.db-wal — write-ahead log (uncommitted transactions) + PRESENT + CORRUPT → check library.db first; WAL corruption usually means + the main library.db is also affected +``` + +### Configuration (master_host*.conf) + +```bash +HOST1_EMBY_CONTAINER="Emby" # aliased by detect_hosts() → EMBY_CONTAINER +HOST2_EMBY_CONTAINER="Emby" +``` + +Emby's config path is detected automatically from Docker volume mounts — no manual +path configuration needed. + +### Usage + +```bash +emby_database_repair.sh # stop Emby, check all databases, restart +emby_database_repair.sh --dry-run # show what would be checked, no Emby stop +emby_database_repair.sh --log # verbose — show SQLite output per database +emby_database_repair.sh --status # show Emby config path and database locations +``` + +--- + +## zfs_pool_scrub.sh + +Triggers ZFS scrub on all pools (or a specific named pool) and waits for completion. +Notifies when done with a summary of any errors found. + +### Why Run ZFS Scrub + +ZFS stores a checksum with every block of data. Scrub reads every block and verifies +the checksum matches the stored hash. Silent data corruption can sit on disk for months +without triggering any error — until you try to read that specific file. By then: +- It may already be mirrored to HOST2 in its corrupted state +- The original source may no longer exist +- ZFS can self-repair during scrub if redundancy exists (RAIDZ or mirrors) + +Run monthly. Also run after any disk replacement or power event. +Safe to run while the system is in use — scrub runs at low I/O priority. + +### Configuration (master_host*.conf) + +```bash +HOST1_ZFS_REPORT_IGNORE_POOLS=( + "disk10" # JBOD member — no redundancy, skipped from default scrub + "disk9" + "disk8" +) + +HOST2_ZFS_REPORT_IGNORE_POOLS=( + "cache" # example — single-disk pool excluded from default +) +``` + +To scrub a pool in the ignore list, specify it by name explicitly. + +### Usage + +```bash +# Scrub all pools except those in ZFS_REPORT_IGNORE_POOLS: +zfs_pool_scrub.sh + +# Scrub a specific pool by name — bypasses the ignore list: +zfs_pool_scrub.sh gaming + +# Check current scrub status without starting a new one: +zfs_pool_scrub.sh --status + +# Dry run — show which pools would be scrubbed: +zfs_pool_scrub.sh --dry-run + +# Verbose — show scrub progress every 60s poll: +zfs_pool_scrub.sh --log +``` + +--- + +## recreate_shares.sh + +Creates share directories on the correct disks after a fresh unRAID install or disk +rebuild. Run once on HOST2 before the first rsync from HOST1. + +### When to Use + +``` +After a fresh unRAID install where /boot/config/shares/*.cfg were restored: + The share definitions exist → UI shows shares → directories are missing on disk + rsync.sh tries to write to /mnt/user/Movies → path doesn't exist → aborts + +After a disk replacement or rebuild where share folders were lost: + Replacement disk is blank → no share directories on the new disk + unRAID won't create them automatically +``` + +### What It Does + +``` +For each .cfg file in /boot/config/shares/: + 1. Read the share name (e.g., Movies) + 2. Read the shareInclude list (e.g., disk1,disk2,disk5) + 3. Create /mnt/disk1/Movies, /mnt/disk2/Movies, /mnt/disk5/Movies + 4. Place a .recovery marker in /mnt/user/Movies/ + +The .recovery marker tells rsync.sh this is a fresh share: + .recovery present → rsync WITHOUT --delete (safe — new files only, nothing removed) + .recovery absent → rsync WITH --delete (normal mirror mode) + +Self-cleaning: after the first successful rsync, the source side has no .recovery file, +so the second nightly run deletes it from the mirror and normal --delete resumes. +No manual cleanup needed. +``` + +### Usage + +```bash +recreate_shares.sh # create all missing share directories + .recovery markers +recreate_shares.sh --dry-run # show what would be created without creating +recreate_shares.sh --log # verbose — show each directory created per disk +recreate_shares.sh --status # show share configs and current directory state +``` + +--- + +## continuous_scripts_status.sh + +Live status dashboard for all continuously running scripts. Read-only — makes no +changes to any running process, container, or state file. + +### What It Shows + +``` +system_watchdog + Running state, PID, uptime, approximate cycle count + Active strikes, recent restart history + Live snapshot: rootfs, RAM, ZFS ARC, load, zombie count, CPU temp + +docker_watchdog + Running state, PID, uptime + Running / stopped / unhealthy container counts + Required containers status + Memory-monitored containers + Recent restart history + skip list + +failover (fallback.sh) + Current state (NORMAL / FALLBACK / HANDBACK) + Tier flags and timestamps + Remote Tailscale visibility +``` + +State files are read as-is — if a script is mid-cycle, the display reflects the last +completed cycle, not the current in-progress state. + +### Usage + +```bash +continuous_scripts_status.sh # show full dashboard +continuous_scripts_status.sh --log # verbose output with additional detail per section +``` + +--- + +## claude_startup.sh + +Restores Claude Code's persistent data after an unRAID reboot and optionally launches +Claude. Standalone script — no common.sh dependency. + +### Why This Exists + +unRAID's root filesystem lives in RAM — `/root/.claude` and `/root/.local` are wiped on +every reboot. This script symlinks both directories back to persistent appdata storage +at `/mnt/user/appdata/claude-code/` before launching Claude. + +### First Run Migration + +On first run, if persistent storage is empty, the script migrates from current live locations: + +``` +/root/.claude → /mnt/user/appdata/claude-code/.claude +/root/.local/share/claude → /mnt/user/appdata/claude-code/local/share/claude +``` + +Subsequent runs skip the migration and only create the symlinks. + +### Calling from array_started.sh + +To auto-restore Claude data on every boot without launching an interactive session: + +```bash +# In /boot/config/go or array_started.sh: +/path/to/Tools/claude_startup.sh --setup +``` + +### Usage + +```bash +claude_startup.sh # set up persistent symlinks and launch Claude +claude_startup.sh --setup # set up symlinks only — no launch (for array_started.sh) +``` + +--- + +## Adding a New Tool + +Write the tool when you solve a problem manually with bash commands. You'll face it again. +The cost of writing the tool is 30 minutes. The cost of reconstructing the commands at 2am +is much higher. + +### Checklist + +``` +✓ Header explains the specific situation that requires this tool +✓ Root check — most tools need root +✓ --dry-run support — always +✓ --status support — show current state before acting +✓ Confirmation for destructive operations (interactive YES or --force flag) +✓ Notify on completion — success and failure +✓ Leave system in clean state on any exit — trap for cleanup +✓ Add to README-Tools.md scripts table and HOW THE SCRIPTS RELATE diagram +``` + +### Minimal Skeleton + +```bash +#!/bin/bash +# ============================================================================================== +# ============================= Your Tool Name ================================================ +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# One sentence: what situation this solves and when to use it. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root Required +# chown / docker / etc. require root. +# +# Confirmation Required +# Interactive mode prompts for YES. Use --force to bypass in scripts. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# your_tool.sh +# Normal run. +# +# your_tool.sh --dry-run +# Preview without making changes. +# +# your_tool.sh --status +# Show current state and exit. +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../load_config.sh" +parse_args "$@" + +if [[ "$EUID" -ne 0 ]]; then error "Must be run as root"; exit 1; fi + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" "unRAID notify script" || warn "notify not found — notifications disabled" + +acquire_lock +detect_hosts + +if [[ "$SHOW_STATUS" == true ]]; then + log "Current state: ..." + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" + +if [[ "$FORCE" != true ]]; then + read -r -p "Type YES to proceed: " CONFIRM + [[ "$CONFIRM" != "YES" ]] && { warn "Aborted."; exit 0; } +fi + +# Do the work +# ... + +notify "Tool completed on $(hostname) ($MY_ID)" "Tool Name" "normal" +``` diff --git a/Tools/README-Tools.md b/Tools/README-Tools.md index 0c6009e..f6f8c50 100644 --- a/Tools/README-Tools.md +++ b/Tools/README-Tools.md @@ -1,834 +1,153 @@ -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# 🔧 TOOLS -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# ━━━━━ TOOLS ━━━━━ **Situational utilities — run when something needs fixing, not on a schedule.** -Recovery, repair, migration, cleanup, and one-time tasks that don't fit the scheduled -maintenance model. These scripts sit ready for the moment you actually need them. - -``` -unRAID_Essentials/ ← regular system maintenance — scheduled -Docker_Essentials/ ← regular container management — scheduled -Monitors/ ← regular health reporting — scheduled -Orchestrators/ ← regular maintenance windows — scheduled -Tools/ ← situational utilities — run when needed -``` +Recovery, repair, migration, and inspection tools for situations that arise outside +the scheduled maintenance model. These scripts sit ready for the moment you need them. > **None of these scripts run on a schedule.** A script belongs here when it solves -> a specific operational situation rather than ongoing maintenance — something you run -> in response to a problem, a planned migration, or a one-time task. Having a dedicated -> folder keeps the other folders clean and makes it obvious what runs routinely vs what -> runs situationally. +> a specific operational situation — something you run in response to a problem, before +> a risky operation, or during a one-time setup task. Having a dedicated folder keeps +> the other folders clean and makes it obvious what runs routinely vs. situationally. --- -## ━━━ THE PROBLEM THAT BUILT THIS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ━━━ THE PROBLEM THAT BUILT THIS ━━━ -Every tool here exists because a specific situation arose that required bash commands -to resolve — and that situation is guaranteed to arise again. +**Failover State Stuck After Testing** +Run a failover test, something exits uncleanly, state file shows `FALLBACK`. +`fallback.sh` resumes and reads FALLBACK — starts containers it shouldn't, makes +decisions based on a state that doesn't reflect reality. Manual recovery means +knowing the exact file format and every field to reset. At 2am after a failed test. +Fix: `fallback_state_reset.sh` — shows current state, prompts for confirmation, +resets cleanly to NORMAL. + +**Container Stuck on Watchdog Skip List After Fixing the Problem** +Authelia hit the restart loop limit — went on the skip list. Problem fixed. But +the watchdog still isn't monitoring it because the skip list persists on `/boot/config` +across reboots. Where's the file? What format? How do you clear restart history? +Fix: `watchdog_skip_list_manager.sh` — shows the skip list and which containers are +running vs. stopped, clears specific containers with confirmation. + +**Emby Crashing With No Clear Cause After a Power Cut** +Server lost power with Emby running. Emby comes back, runs for 20 minutes, crashes. +Logs show database errors. Which database? library.db? users.db? Each has different +recovery implications — deleting the wrong one resets all user watch history. +Fix: `emby_database_repair.sh` — stops Emby, runs `PRAGMA integrity_check` on every +database, reports per-database with specific guidance on what to do about each one. + +**Files Owned by Root After an Admin Copy** +`scp` a file into a media share. File arrives as `root:root`. Radarr fails to import — +permission denied. The daily permissions script won't run for another 20 hours. Running +`media_shares_permissions.sh` on the whole library takes 30 minutes just to fix one dir. +Fix: `bulk_permissions_repair.sh` — takes specific paths, applies correct ownership and +permissions in seconds. + +**No Way to Back Up a Container Before a Risky Update** +Major version update, changelog says "database migration — no rollback." You want a +point-in-time backup. But `cp -r` while the container is running produces an +inconsistent backup, and tar without stopping the container is equally unreliable. +Fix: `container_data_export.sh` — stops the container cleanly, archives appdata to a +timestamped `.tar.gz`, verifies archive integrity, restarts the container. + +**Fresh HOST2 Has Shares Configured But Directories Missing** +Fresh install on HOST2. Restored `/boot/config/shares/` from backup. Array starts. +Shares show in the UI. But the actual `/mnt/diskN/sharename` directories don't exist — +unRAID created the share definitions but not the directories. rsync.sh aborts. +Fix: `recreate_shares.sh` — reads every `.cfg` file, creates directories on each +included disk, places `.recovery` markers so the first rsync won't delete anything. --- -### 🔴 Failover State Stuck After Testing +## ━━━ WHAT THIS FOLDER DOES ━━━ -Run `failover_test.sh`, something goes wrong mid-test, script exits uncleanly. State -file shows `FAILOVER`. `failover.sh` resumes and reads FAILOVER — starts containers -it shouldn't start, makes decisions based on a state that doesn't reflect reality. -Or the test completed but handback didn't finish — state is partially reset. +One role: hold scripts for situations the scheduled maintenance model can't handle. -Manual recovery: edit the state file by hand? Know the exact format? Know which -fields to reset? At 2am after a failed test, none of that is obvious. +Every script here was written because a specific situation arose that required bash +commands to resolve — and that situation is guaranteed to arise again. When you encounter +something new, write the tool. Store it here. Find it at 2am next time. -The tool: `failover_state_reset.sh` — one command, shows you the current state before -asking for confirmation, resets cleanly to NORMAL, explains exactly what it changed. +**Recovery Tools** — Restore known-good state after a failure +`fallback_state_reset.sh`, `watchdog_skip_list_manager.sh` + +**Diagnostic Tools** — Inspect and verify before acting +`emby_database_repair.sh`, `continuous_scripts_status.sh` + +**Repair Tools** — Fix a specific known problem +`bulk_permissions_repair.sh`, `zfs_pool_scrub.sh` + +**Lifecycle Tools** — Backup, setup, and migration support +`container_data_export.sh`, `recreate_shares.sh`, `claude_startup.sh` --- -### 🔴 Container Stuck on Watchdog Skip List After Fixing the Problem +## ━━━ RELATIONSHIP TO OTHER FOLDERS ━━━ -Authelia hit the restart loop limit — three restarts in an hour — went on the skip list. -You fixed the underlying database issue. But the watchdog still isn't monitoring it -because it's on the skip list and you don't know where that file lives or what format -it's in. You restart Authelia manually, it runs fine, but the watchdog has no idea -it recovered and still thinks it's broken. +``` +unRAID_Essentials/ ← regular system maintenance — scheduled +Docker_Essentials/ ← regular container management — scheduled +Monitors/ ← regular health reporting — scheduled +Orchestrators/ ← regular maintenance windows — scheduled +Fallback/ ← automated failover/handback — event-driven +Tools/ ← situational utilities — run when needed +``` -The tool: `watchdog_skip_list_manager.sh` — shows the skip list, shows which containers -are stopped vs running, clears specific containers with a single command, clears restart -history so the loop protection window starts fresh. +Some tools interact with state written by other folders: + +``` +Fallback/ + fallback.sh ──────── writes FALLBACK_STATE_FILE ──► fallback_state_reset.sh reads/writes it + +Docker_Essentials/ + docker_watchdog.sh ── writes skip list + history ──► watchdog_skip_list_manager.sh manages them + +Docker_Essentials/ + unRAID_Essentials/ + Fallback/ + All continuous scripts ──────────────────────────► continuous_scripts_status.sh reads their state +``` + +Tools never call scripts in other folders. Other folders never call Tools scripts. +The relationship is one-way: Tools act on state that other scripts have written. --- -### 🔴 Emby Crashing With No Clear Cause After a Power Cut - -Server lost power with Emby running. Emby comes back up, runs for 20 minutes, crashes. -Comes back up, crashes again. Logs show database errors. Which database? library.db? -users.db? authentication.db? They're all SQLite, they all need a different recovery -approach, and the errors aren't always obvious from the log output alone. - -The tool: `emby_database_repair.sh` — stops Emby, runs `PRAGMA integrity_check` on -every database, reports per database what's clean and what's corrupted with specific -guidance on what to do about each one. - ---- - -### 🔴 Files Owned by Root After an Admin Copy - -`scp` a file from another machine directly into a media share. File arrives as -`root:root`. Radarr tries to move it and fails — permission denied. The daily -permissions script won't run for another 20 hours. Running the full -`media_shares_permissions.sh` on the whole share takes 30 minutes on a large library -just to fix one directory. - -The tool: `bulk_permissions_repair.sh` — takes a specific path or list of paths, -applies correct ownership and permissions in seconds, done. - ---- - -### 🔴 No Way to Back Up a Container Before a Risky Update - -Container has a major version update. The changelog says "database migration — no -rollback." You want a point-in-time backup before you proceed. But the container's -appdata is scattered across dozens of files and `cp -r` while it's running produces -an inconsistent backup. - -The tool: `container_data_export.sh` — stops the container cleanly, archives the -entire appdata directory to a timestamped `.tar.gz`, verifies the archive integrity, -restarts the container. The backup is valid and complete before anything else happens. -If the update goes wrong you have a clean restore point. - ---- - -### 🔴 Fresh HOST2 Has Shares Configured But Directories Missing - -Fresh install on HOST2. Restored `/boot/config/shares/` cfg files from backup. Array -starts. Shares show in the UI. But the actual `/mnt/diskN/sharename` directories don't -exist on the individual disks — unRAID created the share definitions but not the -directories. rsync.sh tries to write, finds the path doesn't exist, aborts. - -The tool: `recreate_shares.sh` — reads every `.cfg` file from `/boot/config/shares/`, -parses the `shareInclude` list, creates the correct directory on each included disk. -Run once after fresh setup, directories exist, rsync works. - ---- - -## ━━━ SCRIPTS AT A GLANCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ━━━ SCRIPTS IN THIS FOLDER ━━━ | Script | What It Fixes | When to Run | |--------|--------------|-------------| -| `failover_state_reset.sh` | State file stuck in FAILOVER after test or failed handback | After failover testing or manual intervention | -| `watchdog_skip_list_manager.sh` | Container stuck on watchdog skip list | After fixing a container that hit restart loop limit | -| `bulk_permissions_repair.sh` | Files owned by wrong user after admin copy or bad container | When arr operations fail due to permissions | -| `container_data_export.sh` | Need a clean backup before a risky container update | Before major updates, migrations, or removals | -| `emby_database_repair.sh` | Emby crashing with database errors after power loss | When Emby logs show corruption or repeated crashes | -| `zfs_pool_scrub.sh` | Verify ZFS pool integrity — catch silent corruption | Monthly, or after any disk/power event | -| `recreate_shares.sh` | Share directories missing on fresh install or rebuild | After fresh unRAID install or disk replacement | -| `rsync_stop.sh` | rsync stuck or needs emergency stop | When rsync is running and must be stopped cleanly | -| `user_scripts_stop.sh` | User Scripts running mid-cycle and need stopping | Before planned reboots, emergency stop | -| `server_reboot.sh` | Graceful reboot with pre-flight warnings and clean shutdown | Planned maintenance reboots | +| `fallback_state_reset.sh` | State file stuck in FALLBACK after test or failed handback | After failover testing or manual intervention | +| `watchdog_skip_list_manager.sh` | Container stuck on watchdog skip list after fixing root cause | After fixing a container that hit the restart loop limit | +| `bulk_permissions_repair.sh` | Files owned by wrong user after admin copy or bad container config | When arr operations fail due to permissions | +| `container_data_export.sh` | Need a clean backup before a risky container update or migration | Before major updates, appdata migrations, or container removals | +| `emby_database_repair.sh` | Emby crashing with database errors after power loss or crash | When Emby logs show corruption or repeated crashes | +| `zfs_pool_scrub.sh` | Verify ZFS pool integrity — catch silent corruption before it spreads | Monthly, or after any disk or power event | +| `recreate_shares.sh` | Share directories missing after fresh install or disk rebuild | After fresh unRAID install or disk replacement on HOST2 | +| `continuous_scripts_status.sh` | Need a live view of watchdog and fallback state | Any time — manual dashboard, no schedule | +| `claude_startup.sh` | Claude Code session setup after reboot — symlinks persistent storage | After each unRAID reboot, or called by array_started.sh | --- -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🔀 failover_state_reset.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ━━━ HOW THE SCRIPTS RELATE ━━━ -Resets the failover state file to NORMAL and clears all tier flags. State file only — -does NOT start or stop any containers. - -> Full documentation in `README-Failover.md` — `failover_state_reset.sh` section. -> This entry is a quick reference. - ---- - -### ── When to Use ────────────────────────────────────────────────────────────── +All Tools scripts are independent — none call each other, none are called by other Tools. ``` -After failover_test.sh didn't complete cleanly - → state left in FAILOVER but containers are actually back to normal - -After a failed handback - → state shows FAILOVER but remote is back up and containers are split - -After killing failover.sh directly (not via User Scripts Abort) - → state is unknown, cycle was interrupted mid-operation - -After a dev/debug session - → state left in a non-NORMAL state from testing +Situation arises + │ + ▼ +┌──────────────────────────────────────────────────────────────────────┐ +│ Tools/ Run directly when needed │ +│ │ +│ fallback_state_reset.sh ◄── after failover test / failed handback│ +│ watchdog_skip_list_manager ◄── after fixing a crash-looping container│ +│ bulk_permissions_repair ◄── wrong ownership after copy or rsync │ +│ container_data_export ◄── before a risky update or migration │ +│ emby_database_repair ◄── Emby logs show corruption │ +│ zfs_pool_scrub ◄── monthly integrity check / post-event │ +│ recreate_shares ◄── fresh HOST2 setup or disk rebuild │ +│ continuous_scripts_status ◄── manual status check at any time │ +│ claude_startup ◄── after each unRAID reboot │ +└──────────────────────────────────────────────────────────────────────┘ + │ + ▼ + State files in other folders (Fallback/, Docker_Essentials/) may be read or written. + No other scripts call into Tools/. ``` - ---- - -### ── Verify Before Resetting ──────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Verify reality matches what you're about to declare as NORMAL: -# -# Right containers on right server? -# DDNS pointing correctly? nslookup Gmer4Lfe.com -# failover.sh not running? pgrep -f "failover.sh" -# Both servers Tailscale connected? tailscale status -# -# Resetting during an actual failover causes failover.sh to think everything -# is normal and stop covering the remote — services go offline until the next -# detection cycle catches it again. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -failover_state_reset.sh --status # show current state file contents — always first -failover_state_reset.sh --dry-run # show what would be written, no write -failover_state_reset.sh # interactive reset — prompts "YES" to confirm -failover_state_reset.sh --force # non-interactive — for scripts, no terminal -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🐳 watchdog_skip_list_manager.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -View and manage the persistent container skip list used by `docker_watchdog.sh`. - -> Full documentation in `README-Docker_Essentials.md` — `watchdog_skip_list_manager.sh` -> section including the full recovery workflow. This entry is a quick reference. - ---- - -### ── When to Use ────────────────────────────────────────────────────────────── - -``` -docker_watchdog.sh restarts the same container N times within the rolling window -→ container added to skip list on /boot/config/ -→ critical notification sent -→ watchdog stops touching it entirely - -You fix the underlying problem (database, config, dependencies). -You need to clear the container from the skip list so monitoring resumes. -``` - ---- - -### ── Recovery Workflow ──────────────────────────────────────────────────────── - -```bash -# 1. Understand the situation — always start here: -watchdog_skip_list_manager.sh --status -# Shows: skip list contents, which are running vs stopped, restart history - -# 2. Fix the underlying problem first -# Check logs: docker logs ContainerName --tail 100 -# Check disk: df -h /mnt/user -# Check db: docker exec ContainerName sqlite3 /path/to.db ".tables" - -# 3. Clear from skip list + restart history: -watchdog_skip_list_manager.sh --clear ContainerName - -# 4. Start the container manually — confirm your fix worked: -docker start ContainerName - -# 5. Watchdog resumes normal monitoring on next cycle — no further action needed -``` - ---- - -### ── Files Managed ──────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── - /boot/config/system_watchdog_failed.db # persistent skip list - /boot/config/container_restart_history.db # restart loop tracking - -# Both live on /boot/config — survive reboots intentionally. -# A container that was skip-listed before a reboot is still broken after it. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -watchdog_skip_list_manager.sh # show status (default) -watchdog_skip_list_manager.sh --status # explicit status -watchdog_skip_list_manager.sh --clear ContainerName # clear specific + restart history -watchdog_skip_list_manager.sh --clear ContainerName --force # no confirmation prompt -watchdog_skip_list_manager.sh --clear-all # clear everything -watchdog_skip_list_manager.sh --clear-all --force # non-interactive -watchdog_skip_list_manager.sh --dry-run # preview any clear action -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🔐 bulk_permissions_repair.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Applies correct ownership and permissions to specific paths. Faster than running -`media_shares_permissions.sh` which processes every configured share — use this when -you know exactly what needs fixing and don't want to wait for a full library walk. - ---- - -### ── When to Use ────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Use instead of the full permissions script when: -# -# Admin copy left root:root files — scp, cp, direct file transfer -# New share needs permissions now — can't wait for nightly run -# Container wrote as root — before PUID/PGID was fixed -# Specific directory has wrong perms — targeted fix, not a full library walk -# -# The full media_shares_permissions.sh is the right tool for: -# Regular nightly maintenance (already scheduled in daily_sync_maintenance.sh) -# After confirming a container's PUID/PGID is now correct -# Initial permissions setup on a new server -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── What It Applies ────────────────────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# Same values as media_shares_permissions.sh — consistent permissions everywhere - PERMISSIONS_DIR_MODE="755" # directories — enter, list, no world-write - PERMISSIONS_FILE_MODE="664" # files — owner+group rw, others read-only - PERMISSIONS_OWNER="nobody:users" # matches PUID=99 PGID=100 in containers -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Single path: -bulk_permissions_repair.sh /mnt/user/Movies - -# Multiple paths — all corrected in one run: -bulk_permissions_repair.sh /mnt/user/Movies /mnt/user/Tv_Shows /mnt/user/Music - -# Dry run first — shows count of files that would be corrected: -bulk_permissions_repair.sh /mnt/user/Movies --dry-run - -# Verbose — show each corrected file: -bulk_permissions_repair.sh /mnt/user/Movies --log -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 📦 container_data_export.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Exports a container's appdata directory to a compressed tar archive. Stops the -container first for a clean consistent backup, verifies the archive after creation, -then restarts the container. - ---- - -### ── When to Use ────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Before major container updates — especially those with "no rollback" database migrations -# Before pool migrations — clean backup before moving appdata to a new pool -# Before removing a container from the stack — archive its data before deletion -# Manual point-in-time backup before risky config changes -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Sequence ───────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# 1. Space check -# Estimates required space from appdata size × 1.1 -# Aborts if output directory doesn't have enough free space -# Container is NOT stopped until the space check passes -# -# 2. Stop container cleanly -# docker stop ContainerName — graceful shutdown -# -# 3. Create archive -# tar -czf ContainerName_YYYY-MM-DD_HH-MM.tar.gz /path/to/appdata -# -# 4. Verify archive integrity -# tar -tzf archive.tar.gz — confirms archive is valid and complete -# If verification fails → restart container anyway, report error -# -# 5. Restart container -# docker start ContainerName — always restarted, even if archiving failed -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -# Syntax: container_data_export.sh ContainerName AppDataPath OutputDir - -# Emby backup example: -container_data_export.sh \ - Emby \ - /mnt/media-servers/Media_Server/Emby \ - /mnt/user/Backups/ - -# Dry run — verify space and paths without stopping anything: -container_data_export.sh \ - Emby \ - /mnt/media-servers/Media_Server/Emby \ - /mnt/user/Backups/ \ - --dry-run - -# Output filename format: ContainerName_YYYY-MM-DD_HH-MM.tar.gz -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🎬 emby_database_repair.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Stops Emby, runs SQLite `PRAGMA integrity_check` on every Emby database, and restarts. -Reports per-database with specific guidance on what to do if corruption is found. - ---- - -### ── When to Use ────────────────────────────────────────────────────────────── - -``` -Emby logs show database errors → run this first -Emby crashing repeatedly with no clear cause → likely database corruption -Playback history or user data behaving strangely → users.db or library.db issue -After a hard shutdown or power loss with Emby running → check for WAL corruption -``` - ---- - -### ── Databases Checked ──────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Each database has different recovery implications: -# -# library.db — media library metadata: titles, seasons, episodes, artwork -# CORRUPT → safe to delete — Emby fully rebuilds from media files on next start -# Rebuild takes time but loses nothing permanent -# -# users.db — user accounts, watch history, playback positions, settings -# CORRUPT → deleting resets ALL user accounts and watch history -# Check if you have a recent backup (weekly_sync_maintenance.sh) -# before deleting -# -# authentication.db — API keys, session tokens -# CORRUPT → safe to delete — API keys regenerated on restart -# Any connected clients will need to re-authenticate -# -# activity.db — activity/access log -# CORRUPT → safe to delete — it's a log, losing it is acceptable -# -# ───────────────────────────────────────────────────────────────────────────── -# This script checks and reports ONLY. It does NOT automatically delete or repair -# corrupted databases. Recovery requires judgment — and potentially a backup restore. -# The summary provides specific guidance per database type. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Normal run — stops Emby, checks all databases, restarts: -emby_database_repair.sh - -# Dry run — detect config path and show what would be checked, no Emby stop: -emby_database_repair.sh --dry-run - -# Verbose — show SQLite output for each database: -emby_database_repair.sh --log - -# Status — show Emby config path and database file locations: -emby_database_repair.sh --status -# ───────────────────────────────────────────────────────────────────────────── -# Config path is detected automatically from Docker volume mounts. -# No configuration needed — just run it. -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🗄️ zfs_pool_scrub.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Triggers ZFS scrub on all pools (or a specific named pool) and waits for completion. -Notifies when done with a summary of any errors found. - ---- - -### ── What ZFS Scrub Does ────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# ZFS stores a checksum with every block of data. Scrub reads every block on -# every pool and verifies the checksum matches the stored hash. -# -# Why this matters: -# Silent data corruption can sit on disk for months without triggering any -# error — until you try to read that specific file. By then: -# - It may already be mirrored to HOST2 in its corrupted state -# - The original source may no longer exist -# - The corruption may have spread if it was a drive issue -# -# ZFS can self-repair during scrub if redundancy exists — RAIDZ or mirrors. -# It cannot repair if you have a single-disk pool (JBOD). -# But it will tell you corruption exists before you find out the hard way. -# -# Recommended: run monthly or after any disk replacement / power event. -# Safe to run while the system is in use — scrub runs at low I/O priority. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Pool Filtering ─────────────────────────────────────────────────────────── - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# Single-disk JBOD members can be excluded from all-pool scrubs. -# To scrub a pool that's in the ignore list: specify it by name explicitly. -# -HOST1_ZFS_REPORT_IGNORE_POOLS=( - "disk10" # JBOD member — no redundancy, scrub still useful but excluded from default - "disk9" - "disk8" -) -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -# Scrub all pools except those in ZFS_REPORT_IGNORE_POOLS: -zfs_pool_scrub.sh - -# Scrub a specific pool by name — ignores the ignore list: -zfs_pool_scrub.sh gaming - -# Check current scrub status without starting a new one: -zfs_pool_scrub.sh --status - -# Dry run — show which pools would be scrubbed: -zfs_pool_scrub.sh --dry-run -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 📁 recreate_shares.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Creates share directories on the correct disks after a fresh install or disk rebuild. -Reads `.cfg` files from `/boot/config/shares/` and creates the corresponding -`/mnt/diskN/sharename` directory on each disk listed in the `shareInclude` setting. - ---- - -### ── When to Use ────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# After a fresh unRAID install where /boot/config/shares/*.cfg were restored: -# The share definitions exist → UI shows shares → directories are missing -# rsync.sh tries to write to /mnt/user/Movies → path doesn't exist → aborts -# -# After a disk replacement or rebuild where share folders were lost: -# Replacement disk is blank → no share directories on the new disk -# unRAID won't create them automatically -# -# Run once on HOST2 after fresh setup, before the first rsync from HOST1. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── What It Does ───────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# For each .cfg file in /boot/config/shares/: -# 1. Read the share name (e.g. Movies) -# 2. Read the shareInclude list (e.g. disk1,disk2,disk5) -# 3. Create /mnt/disk1/Movies, /mnt/disk2/Movies, /mnt/disk5/Movies -# 4. Set correct ownership: nobody:users -# -# Does not create content — just the directories. -# rsync.sh can then write into them normally. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -recreate_shares.sh # create all missing share directories -recreate_shares.sh --dry-run # show what would be created without creating -recreate_shares.sh --log # verbose — show each directory created -recreate_shares.sh --status # show share configs and current directory state -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🔄 rsync_stop.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Stops rsync intelligently on both local and remote servers. Auto-detects if an -orchestrator is running and chooses the safest stop strategy automatically. - ---- - -### ── Two Modes ──────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Smart (default — auto-detected): -# Orchestrator detected → kill rsync subprocess only -# Orchestrator sees rsync died → moves to next share or exits cleanly -# No orphaned lock files, orchestrator exits naturally -# -# --full-stop: -# Kill orchestrator first, then rsync -# Orchestrator will NOT continue to next share -# Use when: you need everything stopped immediately -# -# Why smart is usually correct: -# Killing the orchestrator directly (daily_sync_maintenance.sh) leaves it -# mid-execution. Containers may be stopped but not restarted. Lock files -# may not be released. The smart approach lets the orchestrator clean up -# after itself — fewer side effects. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── When to Use ────────────────────────────────────────────────────────────── - -``` -rsync running during a window where it shouldn't be → smart stop -rsync stuck with no progress → smart stop -Need to start a manual sync that conflicts → smart stop first -Everything must stop NOW (emergency) → --full-stop -Called by partnership_manage.sh --offboard → --rsync-only flag -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -rsync_stop.sh # smart stop — auto-detect orchestrator -rsync_stop.sh --full-stop # kill orchestrator + rsync -rsync_stop.sh --rsync-only # stop rsync, skip container recovery -rsync_stop.sh --dry-run # preview without stopping anything -rsync_stop.sh --status # show what's currently running -rsync_stop.sh --full-stop --dry-run # preview full stop -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🛑 user_scripts_stop.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Stops all running User Script processes spawned by the unRAID User Scripts plugin. -Identifies processes by their `/tmp/user.scripts` path signature, shows script names -not just PIDs, uses SIGTERM → SIGKILL sequence with verification. - ---- - -### ── When to Use ────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# A script is stuck and won't respond to the Abort button in the User Scripts UI -# → the UI button sends a signal that the script may have trapped or ignored -# → user_scripts_stop.sh finds the process by path signature, not by UI state -# -# Before a planned reboot to ensure scripts exit cleanly -# → server_reboot.sh calls this automatically as part of its shutdown sequence -# -# Emergency stop of all background ecosystem scripts -# → stops system_watchdog, docker_watchdog, failover, and any running maintenance -# → use when you need to take manual control immediately -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Self-Exclusion ──────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# If this script is run via the User Scripts plugin it would find its own PID. -# Self-exclusion prevents the script from killing itself mid-execution. -# Own PID and parent PID are excluded before any killing begins. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -user_scripts_stop.sh # stop all — SIGTERM → verify → SIGKILL if needed -user_scripts_stop.sh --dry-run # show which scripts would be stopped, by name -user_scripts_stop.sh --status # show currently running scripts with PIDs and runtime -user_scripts_stop.sh --log # verbose — show each signal and verification step -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🔁 server_reboot.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Graceful reboot with pre-flight warnings, wall message, unRAID notification, clean -shutdown sequence, and VM graceful shutdown before stopping services. - ---- - -### ── When to Use ────────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Planned maintenance reboots — gives users notice and shuts down cleanly -# After kernel or firmware updates that require a reboot -# As an alternative to the unRAID UI reboot — more visibility into state -# -# NOT needed for: system_watchdog.sh triggered reboots (those use /sbin/reboot -# directly after their own shutdown sequence) -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Shutdown Sequence ──────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# 1. Pre-flight warnings (warn not block): -# rsync running → "partial files possible, consider rsync_stop.sh" -# mover running → "files may be left mid-move, consider mover_stop.sh" -# Emby sessions active → "N streams will be interrupted" -# -# 2. Wall message → terminal users -# 3. unRAID notification → dashboard -# 4. Wait REBOOT_SLEEP seconds (default 30) — users can save work -# 5. virsh shutdown each running VM → wait REBOOT_VM_WAIT seconds -# 6. Stop libvirt (VM Manager) -# 7. Stop Docker service -# 8. sync — flush filesystem buffers -# 9. /sbin/reboot -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -server_reboot.sh # reboot with 30s warning -server_reboot.sh --dry-run # walk through sequence without rebooting -server_reboot.sh --status # show running processes that would be affected -server_reboot.sh --reason="maintenance" # include reason in wall + notification -server_reboot.sh --log # verbose output per shutdown step -``` - ---- - -## ━━━ ADDING A NEW TOOL ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# When you encounter a situation that required manual bash commands to resolve — -# write a tool. You'll face it again. The cost of writing the tool is 30 minutes. -# The cost of reconstructing the commands at 2am is much higher. -# -# Checklist for a new Tools script: -# -# ✓ Header explains the specific situation that requires this tool -# ✓ "When to Use" section — exactly the symptoms that trigger this -# ✓ Root check — most tools need root -# ✓ --dry-run support — always -# ✓ --status support — show current state before acting -# ✓ Confirmation for destructive operations (read -p "Type YES:") -# ✓ Notify on completion — success and failure -# ✓ Leave system in clean state on any exit — trap for cleanup -# ───────────────────────────────────────────────────────────────────────────── - -# Minimal skeleton: -#!/bin/bash -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../load_config.sh" -parse_args "$@" - -if [[ "$EUID" -ne 0 ]]; then error "Must be run as root"; exit 1; fi -validate_unraid_cmd "/usr/local/emhttp/plugins/dynamix/scripts/notify" "" "" "notify" -acquire_lock -detect_hosts - -# Show status if requested -if [[ "$SHOW_STATUS" == true ]]; then - echo "Current state: ..." - exit 0 -fi - -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" - -# Confirm before destructive operations -read -r -p "Type YES to proceed: " CONFIRM -[[ "$CONFIRM" != "YES" ]] && { warn "Aborted."; exit 0; } - -# Do the work -# ... - -notify "Tool completed on $(hostname) ($MY_ID)" "Tool Name" "normal" -``` - ---- - -## ━━━ PHILOSOPHY ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -``` -Write the tool when you solve the problem. -Store it here. -Find it at 2am when you need it again. - -Tools exist because not every problem has a scheduled solution. -Some things only need to happen once. -Some things only happen after something goes wrong. -Having a dedicated folder keeps the other folders clean — -everything in Orchestrators, Docker_Essentials, and Monitors -has a reason to run regularly. - -Everything here has a reason to exist and wait. -``` \ No newline at end of file diff --git a/Tools/bulk_permissions_repair.sh b/Tools/bulk_permissions_repair.sh index c78f65c..bc29783 100644 --- a/Tools/bulk_permissions_repair.sh +++ b/Tools/bulk_permissions_repair.sh @@ -2,47 +2,84 @@ # ============================================================================================== # ============================= Bulk Permissions Repair ======================================== # ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── # Applies correct ownership and permissions to one or more specific paths. -# Faster than running media_shares_permissions.sh which processes all configured shares. +# Targeted repair — faster than media_shares_permissions.sh, which processes +# every configured share. Use after failed transfers that left root:root ownership, +# containers writing as root before PUID/PGID was fixed, manual file copies, or +# new shares that need permissions applied before the next nightly run. # -# ── WHEN TO USE ─────────────────────────────────────────────────────────────────────────────── -# Use for targeted repair after: -# - A failed transfer that left files owned by wrong user (root:root from rsync) -# - A container writing as root instead of nobody:users — before PUID/PGID was fixed -# - Manual file copies that bypassed normal permission handling -# - A new share that needs permissions applied before the next nightly run -# - A large rsync that imported thousands of files before media_shares_permissions.sh ran +# Counts files with wrong ownership before fixing. A high count on a recently +# written share means a container has wrong PUID/PGID — add PUID=99 PGID=100 +# to its Docker template. Common culprits: SABnzbd, qBittorrent, slskd. # -# ── PERMISSIONS MODEL ───────────────────────────────────────────────────────────────────────── -# Directories: PERMISSIONS_DIR_MODE (default 755) -# Owner (nobody) — rwx enter, list, create files -# Group (users) — r-x enter and list -# Others — r-x Samba guests can browse +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== # -# Files: PERMISSIONS_FILE_MODE (default 664) -# Owner (nobody) — rw read + write -# Group (users) — rw arrs can import and rename -# Others — r Samba guests can read -# No execute bit — media files are never executable +# Permissions Model +# Directories (PERMISSIONS_DIR_MODE, default 755): +# Owner (nobody) — rwx enter, list, create files +# Group (users) — r-x enter and list +# Others — r-x Samba guests can browse +# Files (PERMISSIONS_FILE_MODE, default 664): +# Owner (nobody) — rw read + write +# Group (users) — rw arrs can import and rename +# Others — r Samba guests can read +# No execute bit — media files are never executable # -# ── DIAGNOSTIC — HIGH WRONG OWNER COUNT ─────────────────────────────────────────────────────── -# This script counts files with wrong ownership before applying the fix. -# A high count on a share that was recently written → a container has wrong PUID/PGID. -# Fix: add PUID=99 PGID=100 to the container's Docker template. -# Common culprits: SABnzbd, qBittorrent, slskd. +# Separate Passes +# Directories and files are chmod'd in separate find passes. A combined pass +# with mode 664 would wrongly strip the execute bit from directories, making +# them untraversable. # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# Root check — required for chown -# Path existence check — skips missing paths with error -# Separate passes — directories and files chmod'd separately for correctness -# validate_unraid_cmd — notify validated before use -# Silent on success — only failures produce visible output +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root Required +# chown requires root — exits immediately if not running as root. +# +# Path Existence Check +# Each path is verified before processing — missing paths log an error and +# are skipped rather than silently passing. +# +# Notification Validated +# validate_unraid_cmd confirms the notify script is present before use. +# +# Silent on Success +# Only failures and the wrong-owner diagnostic produce visible output. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# PERMISSIONS_OWNER +# Owner applied to all paths. (default: nobody:users) +# +# PERMISSIONS_DIR_MODE +# chmod mode for directories. (default: 755) +# +# PERMISSIONS_FILE_MODE +# chmod mode for files. (default: 664) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# bulk_permissions_repair.sh /path/to/share [/another/path ...] +# Apply ownership and permissions to each specified path. +# +# bulk_permissions_repair.sh /path/to/share --dry-run +# Show wrong-owner count per path. No chown or chmod applied. +# +# bulk_permissions_repair.sh /path/to/share --log +# Verbose output including per-path file counts and modes applied. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# bulk_permissions_repair.sh /mnt/user/Movies -# bulk_permissions_repair.sh /mnt/user/Movies /mnt/user/Tv_Shows -# bulk_permissions_repair.sh /mnt/user/Movies --dry-run -# bulk_permissions_repair.sh /mnt/user/Movies --log # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Tools/claude_startup.sh b/Tools/claude_startup.sh index 89f1c68..4eb6cb5 100755 --- a/Tools/claude_startup.sh +++ b/Tools/claude_startup.sh @@ -2,21 +2,34 @@ # ============================================================================================== # ================================= Claude Code Startup ======================================== # ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── # Restores Claude Code's persistent data after an Unraid reboot and launches Claude. +# Unraid's root filesystem lives in RAM — /root/.claude and /root/.local are wiped on +# every reboot. This script symlinks both directories back to persistent appdata storage +# before launching Claude, so memory, sessions, and settings survive across reboots. # -# Unraid's root filesystem lives in RAM — /root/.claude and /root/.local are wiped on every -# reboot. This script symlinks both directories back to persistent appdata storage before -# launching Claude, so memory, sessions, and settings survive across reboots. +# On first run with no existing persistent data, migrates from the current live locations: +# /root/.claude → PERSIST_DIR/.claude (memory, sessions, settings) +# /root/.local/share/claude → PERSIST_DIR/local/share/claude (installed binaries) +# Subsequent runs skip the migration and only create the symlinks. # -# ── FIRST RUN ───────────────────────────────────────────────────────────────────────────────── -# If persistent storage has no data yet, migrates from the current live locations: -# /root/.claude → PERSIST_DIR/.claude (memory, sessions, settings) -# /root/.local/share/claude → PERSIST_DIR/local/share/claude (installed binaries) -# Subsequent runs skip the migration and just create the symlinks. +# Standalone script — no common.sh dependency. Safe to run directly from terminal +# or from array_started.sh. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# claude_startup.sh +# Set up persistent symlinks and launch Claude. +# +# claude_startup.sh --setup +# Set up persistent symlinks only — do not launch Claude. +# Used by array_started.sh to prepare the environment on boot without +# immediately launching an interactive session. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# claude_startup.sh — set up persistent symlinks and launch Claude -# claude_startup.sh --setup — set up only, do not launch (for array_started.sh use) # ============================================================================================== PERSIST_DIR="/mnt/user/appdata/claude-code" diff --git a/Tools/container_data_export.sh b/Tools/container_data_export.sh index 192b3d0..eac59c8 100644 --- a/Tools/container_data_export.sh +++ b/Tools/container_data_export.sh @@ -2,46 +2,67 @@ # ============================================================================================== # ============================= Container Data Export ========================================== # ============================================================================================== -# Exports a container's appdata directory to a compressed tar archive. -# Stops the container before archiving and restarts it after — ensures clean consistent backup. -# Verifies the archive after creation — confirms backup is valid before restarting container. # -# ── WHEN TO USE ─────────────────────────────────────────────────────────────────────────────── -# - Before major container updates (roll back if update goes wrong) -# - Before pool migrations or disk replacements -# - When archiving a container being removed from the stack -# - Before destructive operations on appdata (database migrations etc.) -# - One-off backup of a specific container without running full backup +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Exports a container's appdata directory to a compressed tar archive. Stops +# the container before archiving and restarts it after — ensures a clean, +# consistent backup. Use before major updates, pool migrations, destructive +# appdata operations, or when archiving a container being removed from the stack. # -# ── OUTPUT FILE NAMING ──────────────────────────────────────────────────────────────────────── -# ContainerName_YYYY-MM-DD_HH-MM.tar.gz -# Timestamp in filename — run multiple times safely, no overwrite ✅ +# Output: ContainerName_YYYY-MM-DD_HH-MM.tar.gz — timestamped, no overwrite. # -# ── SPACE CHECK ─────────────────────────────────────────────────────────────────────────────── -# Estimates required space as appdata size × 1.1 (10% buffer). -# Compressed archive will typically be much smaller — this is a conservative floor. -# gzip compression ratio depends heavily on content — database files compress well, -# media files do not. If output is on a media share estimate may be pessimistic. +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== # -# ── ARCHIVE VERIFICATION ────────────────────────────────────────────────────────────────────── -# After creation the archive is tested with tar --test-file before restarting the container. -# If verification fails the container is still restarted (data unchanged) and an error logged. -# A corrupt archive is not a usable backup — do not assume the archive is good without this. +# Archive Verification Before Restart +# The archive is tested with tar --test-file before the container is restarted. +# A corrupt archive is not a usable backup — this catches tar failures, I/O +# errors, and truncated writes before declaring success. If verification fails, +# the container is still restarted (appdata is unchanged) and an error logged. # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# DOCKER_TIMEOUT — docker calls protected against hung daemon -# Container restart rule — was running → restart | was stopped → leave stopped ✅ -# Archive cleanup — partial archive removed on tar failure -# Archive verification — tar --test-file after creation -# Container restart on — any failure path still restarts container if it was running -# validate_unraid_cmd — notify validated before use -# Silent on success — only problems produce visible output +# Conservative Space Estimate +# Required space is estimated as appdata size × 1.1 (10% buffer). The actual +# compressed archive will typically be much smaller — database files compress +# well, media files do not. The estimate is a conservative floor, not a +# prediction. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Container Restart Rule +# Tracks whether the container was running before the export. Running containers +# are restarted after completion; already-stopped containers are left stopped. +# The restart happens on every exit path — a failed tar does not leave the +# container stuck stopped. +# +# Partial Archive Cleanup +# If tar fails, the incomplete archive is removed. A partial archive is worse +# than no archive — it can look valid but restore to an incomplete state. +# +# Docker Timeout +# DOCKER_TIMEOUT (default: 30s) caps all docker calls. Guards against a hung +# daemon blocking the script indefinitely. +# +# Notification Validated +# validate_unraid_cmd confirms the notify script is present before use. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# container_data_export.sh ContainerName /path/to/appdata /path/to/output/dir +# Stop container, create archive, verify, restart container. +# Example: container_data_export.sh Emby /mnt/media-servers/.../Emby /mnt/user/Backups/ +# +# container_data_export.sh ContainerName /path/to/appdata /path/to/output/dir --dry-run +# Show what would be archived and estimated size. No container stop, no tar. +# +# container_data_export.sh ContainerName /path/to/appdata /path/to/output/dir --log +# Verbose output: space check, tar progress, verification result. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# container_data_export.sh ContainerName /path/to/appdata /path/to/output/dir -# container_data_export.sh Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/ -# container_data_export.sh Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/ --dry-run -# container_data_export.sh Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/ --log # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Tools/emby_database_repair.sh b/Tools/emby_database_repair.sh index 7d7494d..ec4e687 100644 --- a/Tools/emby_database_repair.sh +++ b/Tools/emby_database_repair.sh @@ -2,43 +2,81 @@ # ============================================================================================== # ============================= Emby Database Repair =========================================== # ============================================================================================== -# Stops Emby, runs SQLite integrity checks on all Emby databases, and restarts. -# Use when Emby reports database corruption, unexpected crashes, or playback state issues. # -# ── CHECKS PERFORMED ────────────────────────────────────────────────────────────────────────── -# PRAGMA integrity_check — full SQLite integrity verification per database -# Skips missing databases gracefully — not all files exist on all setups +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Stops Emby, runs SQLite PRAGMA integrity_check on all Emby databases, and +# restarts. Use when Emby reports corruption, unexpected crashes, or playback +# state issues. # -# ── DATABASES CHECKED ───────────────────────────────────────────────────────────────────────── +# Reports which databases are corrupted. Does NOT automatically repair. +# Repair requires manual steps — guidance is printed in the summary output. +# Always take a backup before deleting any database file. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Databases Checked # library.db — media library metadata (largest, most critical) -# library.db-wal — write-ahead log (if exists — uncommitted transactions) +# library.db-wal — write-ahead log (if present — uncommitted transactions) # librarydb.db — legacy library database # users.db — user accounts and settings # authentication.db — API keys and sessions -# activity.db — activity log (least critical, safe to delete) +# activity.db — activity log (least critical, safe to delete if corrupt) # -# ── IF CORRUPTION FOUND ─────────────────────────────────────────────────────────────────────── -# Reports which databases are corrupted. Does NOT automatically repair. -# Corruption repair requires manual steps — see guidance in summary output. -# Always take a backup before deleting any database file. +# Missing databases are skipped gracefully — not all files exist on all setups. +# Emby's config path is detected from the Docker mount — no hardcoded paths. # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID and aliases HOST*_EMBY_CONTAINER → EMBY_CONTAINER. -# Each server checks its own Emby instance automatically. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# EXIT trap — Emby always restarted even if script crashes mid-check -# DOCKER_TIMEOUT — all docker calls protected against hung daemon -# jq validation — verifies jq available before config path detection -# validate_unraid_cmd — sqlite3 and notify validated before use -# Container verify — checks Emby stayed running after restart -# Silent healthy — only corruption produces visible output +# Guaranteed Restart +# EXIT trap ensures Emby is always restarted even if the script crashes +# mid-check — Emby is never left stopped due to a script error. +# +# Docker Timeout +# DOCKER_TIMEOUT (30s) protects all docker calls against a hung daemon. +# Emby can take time to stop cleanly — 30s is intentionally generous. +# +# Tool Validation +# validate_unraid_cmd confirms sqlite3 and the notify script are present +# before use. jq is checked separately — required for config path detection. +# +# Post-Restart Verify +# Checks that Emby is still running after restart — detects cases where +# Emby crashes immediately after start (which would indicate deeper trouble). +# +# Silent When Healthy +# Only corruption produces visible output and a notification. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# +# HOST*_EMBY_CONTAINER +# Name of the Emby Docker container on this host. +# Aliased by detect_hosts() → EMBY_CONTAINER. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# emby_database_repair.sh +# Stop Emby, check all databases with PRAGMA integrity_check, restart. +# +# emby_database_repair.sh --dry-run +# Show which databases would be checked and Emby container name. No stop. +# +# emby_database_repair.sh --log +# Verbose output with per-database check result. +# +# emby_database_repair.sh --status +# Show Emby container name and config path detected from Docker. Then exit. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# emby_database_repair.sh — stop Emby, check all databases, restart -# emby_database_repair.sh --dry-run — show what would be checked, no Emby stop -# emby_database_repair.sh --log — verbose output per database -# emby_database_repair.sh --status — show config and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Tools/fallback_state_reset.sh b/Tools/fallback_state_reset.sh index 19b0f59..b58ee11 100644 --- a/Tools/fallback_state_reset.sh +++ b/Tools/fallback_state_reset.sh @@ -2,46 +2,66 @@ # ============================================================================================== # ============================= Failover State Reset =========================================== # ============================================================================================== -# Resets the fallback state file to NORMAL and clears all tier flags. -# Use when the fallback state file is stuck in a non-NORMAL state after: -# - Failover testing that left state as FALLBACK -# - A failed handback that did not complete cleanly -# - Manual intervention that left state inconsistent -# - fallback.sh was killed mid-cycle and state is unknown # -# ── WHAT THIS DOES ──────────────────────────────────────────────────────────────────────────── +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Resets the fallback state file to NORMAL and clears all tier flags. Use when +# the state file is stuck after failover testing, a failed handback, manual +# intervention, or fallback.sh being killed mid-cycle. +# # Writes a fresh state file with: -# state=NORMAL -# fallback_start=0 -# handback_strikes=0 +# state=NORMAL / fallback_start=0 / handback_strikes=0 # tier2_started=false / tier3_started=false / tier4_started=false # -# Does NOT start or stop any containers — state file only. -# After reset, fallback.sh will resume from NORMAL on its next cycle. +# Does NOT start or stop containers — state file only. After reset, fallback.sh +# resumes from NORMAL on its next cycle. # -# ── ⚠️ ONLY RUN WHEN SAFE ──────────────────────────────────────────────────────────────────── -# Verify BEFORE resetting: -# ✓ Right containers running on the right server -# ✓ DDNS pointing at the correct server -# ✓ No active failover actually in progress -# ✓ Both servers can see each other +# WARNING: Only run when you have verified the stack is actually in a normal +# state — right containers on the right server, DDNS correct, no active failover +# in progress. Resetting state during a real failover causes fallback.sh to stop +# covering the remote server until the next detection cycle. # -# Resetting state while a real fallback is happening causes fallback.sh to stop -# covering the remote server — services go offline until next detection cycle. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# fallback.sh running check — warns if fallback.sh is active when reset is attempted -# acquire_lock — prevents concurrent resets -# flock on state write — prevents race with fallback.sh mid-cycle read -# Confirmation required — interactive: type YES | non-interactive: --force flag -# validate_unraid_cmd — notify validated before use +# Active Fallback Detection +# Checks whether fallback.sh is currently running and warns if so. A reset +# during an active cycle causes fallback.sh to lose its state on the next read. +# +# Single Instance Lock +# acquire_lock prevents concurrent resets. +# +# flock on State Write +# The state file write is protected with flock — prevents a race condition +# with fallback.sh reading the file mid-cycle. +# +# Confirmation Required +# Interactive mode prompts for YES before writing. Use --force to bypass in +# non-interactive contexts (cron, scripts). +# +# Notification Validated +# validate_unraid_cmd confirms the notify script is present before use. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# fallback_state_reset.sh +# Show current state and prompt for YES before resetting. +# +# fallback_state_reset.sh --dry-run +# Show current state and what the new state file would contain. No write. +# +# fallback_state_reset.sh --status +# Show current state file contents and exit. +# +# fallback_state_reset.sh --force +# Reset without interactive confirmation. Safe for scripted use. +# +# fallback_state_reset.sh --force --dry-run +# Dry run without the confirmation prompt. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# fallback_state_reset.sh — interactive reset (prompts for YES) -# fallback_state_reset.sh --dry-run — show current state, show what would be written -# fallback_state_reset.sh --status — show current state file contents and exit -# fallback_state_reset.sh --force — non-interactive reset (no prompt, use in scripts) -# fallback_state_reset.sh --force --dry-run — dry run without prompt # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Tools/recreate_shares.sh b/Tools/recreate_shares.sh index 8c1fd34..95fe1ab 100644 --- a/Tools/recreate_shares.sh +++ b/Tools/recreate_shares.sh @@ -2,52 +2,77 @@ # ============================================================================================== # ============================= Recreate Shares ================================================ # ============================================================================================== -# Creates share directories on the correct disks after a fresh unRAID install or disk rebuild. -# Reads all .cfg files from /boot/config/shares/ and creates the corresponding directories -# on each disk listed in the shareInclude setting. # -# ── WHEN TO USE ─────────────────────────────────────────────────────────────────────────────── -# Run directly on HOST2 after array is started following: -# - A full disk replacement or rebuild where share folders were lost -# - A fresh unRAID install where /boot/config/shares/*.cfg files were restored -# - Any situation where the share folder structure exists in config but not on disk +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Creates share directories on the correct disks after a fresh unRAID install +# or disk rebuild. Reads all .cfg files from /boot/config/shares/ and creates +# the corresponding directories on each disk listed in the shareInclude setting. +# The array must be started before running — /mnt/user must be mounted. # -# The array must be started before running this script — /mnt/user must be mounted. +# Typically run on HOST2 after a full disk replacement or fresh install where +# share folders were lost but /boot/config/shares/*.cfg files were restored. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── WHAT IT DOES ────────────────────────────────────────────────────────────────────────────── # For each share .cfg file: # 1. Reads shareInclude= to determine which disks own this share # 2. Creates /mnt/diskN/ShareName/ on each listed disk if it doesn't exist # 3. Places a .recovery marker file in /mnt/user/ShareName/ via the union filesystem # -# ── .RECOVERY MARKER FILE ───────────────────────────────────────────────────────────────────── -# The .recovery marker signals to rsync.sh that this is a fresh share with no existing data. -# rsync.sh checks for .recovery before running with --delete: -# .recovery present → rsync WITHOUT --delete (safe — new files only, nothing removed) -# .recovery absent → rsync WITH --delete (normal — mirror mode) +# .recovery Marker +# Signals to rsync.sh that this is a fresh share with no existing data. +# rsync.sh checks for .recovery before running with --delete: +# .recovery present → rsync WITHOUT --delete (new files only, nothing removed) +# .recovery absent → rsync WITH --delete (normal mirror mode) # -# The marker self-cleans: after the first successful rsync the source side has no .recovery -# file so the second nightly run will delete it from the mirror, restoring normal --delete -# behaviour automatically. No manual cleanup needed. ✅ +# Self-cleaning: after the first successful rsync the source side has no .recovery +# file, so the second nightly run deletes it from the mirror, restoring normal +# --delete behaviour automatically. No manual cleanup needed. # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# This script runs on the server that needs shares recreated — typically HOST2 during rebuild. -# detect_hosts() sets MY_ID for output clarity. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock — prevents duplicate runs placing duplicate markers -# Root check — mkdir on /mnt/diskN requires root -# Array mount check — exits cleanly if array not started -# Empty cfg guard — warns if no share cfg files found -# Per-disk guards — skips missing disks with warning, continues others -# validate_unraid_cmd — notify validated before use -# Silent on success — only failures produce visible output +# Single Instance Lock +# acquire_lock prevents duplicate runs placing duplicate .recovery markers. +# +# Root Required +# mkdir on /mnt/diskN requires root. +# +# Array Mount Check +# Exits cleanly if the array is not started — /mnt/user not mounted means +# all share operations would fail silently. +# +# Per-Disk Guards +# Missing disks are skipped with a warning and the rest continue — a single +# offline disk does not abort the full run. +# +# Empty Config Guard +# Warns if no share .cfg files are found — catches the case where +# /boot/config/shares/ was not restored. +# +# Notification Validated +# validate_unraid_cmd confirms the notify script is present before use. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# recreate_shares.sh +# Read all .cfg files, create share directories, place .recovery markers. +# +# recreate_shares.sh --dry-run +# Show what directories and markers would be created. No changes. +# +# recreate_shares.sh --log +# Verbose output per share and per disk. +# +# recreate_shares.sh --status +# Show which shares exist in config and which directories exist on disk. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# recreate_shares.sh — create all shares from .cfg files -# recreate_shares.sh --dry-run — preview what would be created, no changes -# recreate_shares.sh --log — verbose output per disk -# recreate_shares.sh --status — show current share state and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Tools/watchdog_skip_list_manager.sh b/Tools/watchdog_skip_list_manager.sh index 05011c3..1663527 100644 --- a/Tools/watchdog_skip_list_manager.sh +++ b/Tools/watchdog_skip_list_manager.sh @@ -2,51 +2,73 @@ # ============================================================================================== # =========================== Watchdog Skip List Manager ======================================= # ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── # View and manage the persistent container skip list used by docker_watchdog.sh. +# docker_watchdog.sh adds a container to the skip list when it exceeds +# WATCHDOG_CONTAINER_RESTART_LIMIT restarts within WATCHDOG_CONTAINER_RESTART_WINDOW +# hours — prevents infinite restart loops on containers that keep crashing. # -# ── WHAT THE SKIP LIST IS ───────────────────────────────────────────────────────────────────── -# docker_watchdog.sh adds a container to the skip list when it exceeds the restart loop -# limit (WATCHDOG_CONTAINER_RESTART_LIMIT in WATCHDOG_CONTAINER_RESTART_WINDOW hours). -# Once on the skip list the watchdog stops restarting it — prevents infinite restart loops. +# Skip list persists on /boot/config (survives reboots). Auto-clears when +# docker_watchdog.sh sees the container running on a later cycle. Use this +# script to clear manually after fixing the underlying problem. # -# Skip list persists on /boot/config — survives reboots. -# Auto-clears when docker_watchdog.sh sees the container running on a cycle. -# This script clears it manually when you have fixed the underlying problem. +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── ACTIONS ─────────────────────────────────────────────────────────────────────────────────── -# --status — show skip list, container states, restart history -# --clear ContainerName — clear a specific container from skip list + history -# --clear-all — clear all skip lists and restart history +# Skip List Lifecycle +# 1. Container crashes repeatedly → watchdog adds to skip list, notifies +# 2. Watchdog stops restarting the container on subsequent cycles +# 3a. If container recovers on its own (Docker restart policy), watchdog +# sees it running, removes from skip list automatically +# 3b. If stuck stopped → fix the root cause, clear via this script, then +# docker start ContainerName manually +# 4. Watchdog monitors normally on next cycle. If it crashes again → re-added. # -# ── AFTER CLEARING ──────────────────────────────────────────────────────────────────────────── -# 1. Fix whatever was causing the container to fail -# 2. Start it manually: docker start ContainerName -# 3. docker_watchdog.sh monitors it normally on the next cycle -# 4. If it crashes again → watchdog adds it back and notifies +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── SKIP LIST AUTO-CLEAR ────────────────────────────────────────────────────────────────────── -# docker_watchdog.sh auto-clears a container from the skip list when it sees it running. -# So if a container recovers on its own (Docker restart policy eventually works), -# the watchdog will see it running, remove it from the skip list, and resume monitoring. -# Manual clear only needed when container is stuck stopped and needs intervention. +# Single Instance Lock +# acquire_lock prevents concurrent access with docker_watchdog.sh writing +# the same files. # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock — prevents concurrent access with docker_watchdog.sh writing files -# docker_watchdog check — warns if watchdog is running during clear (could re-add instantly) -# DOCKER_TIMEOUT — docker inspect calls protected against daemon hangs -# Confirmation required — interactive: YES | non-interactive: --force flag -# validate_unraid_cmd — notify validated before use +# Active Watchdog Detection +# Warns if docker_watchdog.sh is currently running when a clear is attempted +# — the watchdog could re-add the container to the skip list within seconds. # -# ── FILES MANAGED ───────────────────────────────────────────────────────────────────────────── -# SYS_WATCHDOG_FAILED_FILE — persistent container skip list -# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection +# Docker Timeout +# DOCKER_TIMEOUT caps docker inspect calls against a hung daemon. +# +# Confirmation Required +# Interactive mode prompts for YES before clearing. Use --force for scripts. +# +# ============================================================================================== +# STATE FILES +# ============================================================================================== +# +# SYS_WATCHDOG_FAILED_FILE — persistent container skip list (on /boot/config) +# WATCHDOG_CONTAINER_RESTART_LOG — restart history used for loop detection +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# watchdog_skip_list_manager.sh [--status] +# Show skip list, container states, and recent restart history. +# +# watchdog_skip_list_manager.sh --clear ContainerName +# Remove a specific container from the skip list and clear its restart history. +# Prompts for YES unless --force is passed. +# +# watchdog_skip_list_manager.sh --clear-all +# Clear all skip lists and all restart history. +# Prompts for YES unless --force is passed. +# +# All actions support --dry-run (show what would change) and --force (skip prompt). # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# watchdog_skip_list_manager.sh — show status -# watchdog_skip_list_manager.sh --status — show status explicitly -# watchdog_skip_list_manager.sh --clear ContainerName — clear specific container -# watchdog_skip_list_manager.sh --clear-all — clear everything -# Any action supports --dry-run and --force # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Tools/zfs_pool_scrub.sh b/Tools/zfs_pool_scrub.sh index 6cece26..8febee2 100644 --- a/Tools/zfs_pool_scrub.sh +++ b/Tools/zfs_pool_scrub.sh @@ -2,47 +2,80 @@ # ============================================================================================== # ================================= ZFS Pool Scrub ============================================ # ============================================================================================== -# Triggers a ZFS scrub on all pools (or a specific pool) and waits for completion. -# Sends a notification when scrub completes with a summary of any errors found. # -# ── WHAT ZFS SCRUB DOES ─────────────────────────────────────────────────────────────────────── -# Reads every block on every pool and verifies checksums against the stored hash. -# Catches silent data corruption that would otherwise only surface when you read the -# corrupted data — by then it may be too late for redundancy to help. +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Triggers a ZFS scrub on all pools (or a specific pool), waits for completion, +# and sends a notification with any errors found. Reads every block on every pool +# and verifies checksums — catches silent corruption that would otherwise only +# surface when the corrupted data is read (possibly after redundancy can no +# longer help). Monthly recommended for all pools; quarterly minimum for large pools. # -# Scrub is safe to run while the pool is in use — it does not interrupt normal I/O. -# It does consume I/O bandwidth — run during off-peak hours or maintenance windows. -# Monthly is recommended for all pools. Quarterly minimum for large pools. +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── BEHAVIOUR ───────────────────────────────────────────────────────────────────────────────── -# Starts scrub on each pool then polls every 60 seconds until all complete. -# Progress shown via warn() every poll (visible) when scrub is running. -# Safe to leave running or interrupt — scrub continues even if script is stopped. -# On completion reports errors per pool and notifies if any found. +# Starts a scrub on each pool, then polls every 60 seconds until all complete. +# Progress is shown every poll — safe to leave running or interrupt. ZFS scrub +# continues in the kernel even if the script is stopped — it does not depend on +# this script remaining alive. # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS → ZFS_REPORT_IGNORE_POOLS. -# Pools in ZFS_REPORT_IGNORE_POOLS are skipped (single-disk VMs, temp pools etc.) -# unless specified explicitly as a positional argument. +# Scrub is safe to run while the pool is in use. It does consume I/O bandwidth — +# schedule during off-peak hours or maintenance windows. # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock — prevents concurrent scrub starts on same server -# detect_hosts() — correct pool ignore list per host -# validate_unraid_cmd — zpool and notify validated before use -# Scrub-in-progress check — skips pools already scrubbing rather than erroring -# SIGTERM trap — poll loop exits cleanly on signal -# Silent when clean — only errors produce visible output and notification +# Pools in HOST*_ZFS_REPORT_IGNORE_POOLS are skipped automatically (single-disk +# VM pools, temp pools, etc.). Specifying a pool by name bypasses the ignore list. # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── -# HOST*_ZFS_REPORT_IGNORE_POOLS — pools excluded from automatic scrub -# Aliased by detect_hosts() — script uses ZFS_REPORT_IGNORE_POOLS +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Single Instance Lock +# acquire_lock prevents concurrent scrub starts on the same server. +# +# Scrub-in-Progress Check +# Detects pools already scrubbing and skips them rather than erroring — safe +# to run when a scrub may have been started by another path. +# +# SIGTERM Trap +# The poll loop exits cleanly on signal. The ZFS scrub continues regardless. +# +# Tool Validation +# validate_unraid_cmd confirms zpool and the notify script are present before use. +# +# Silent When Clean +# Only errors produce visible output and a notification. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# +# HOST*_ZFS_REPORT_IGNORE_POOLS +# Pools to exclude from automatic scrub. Typically single-disk VM pools +# or temporary pools that do not need integrity checking. +# Aliased by detect_hosts() → ZFS_REPORT_IGNORE_POOLS. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# zfs_pool_scrub.sh +# Scrub all pools not in ZFS_REPORT_IGNORE_POOLS. Wait for completion. +# +# zfs_pool_scrub.sh poolname +# Scrub a specific pool by name. Bypasses the ignore list. +# +# zfs_pool_scrub.sh --status +# Show current scrub status for all pools and exit. +# +# zfs_pool_scrub.sh --dry-run +# Show which pools would be scrubbed. No scrub started. +# +# zfs_pool_scrub.sh --log +# Verbose progress output every poll cycle. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# zfs_pool_scrub.sh — scrub all non-ignored pools -# zfs_pool_scrub.sh poolname — scrub specific pool (bypasses ignore list) -# zfs_pool_scrub.sh --status — show scrub status for all pools -# zfs_pool_scrub.sh --dry-run — show what would be scrubbed -# zfs_pool_scrub.sh --log — verbose progress output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Transcodes/Manual-Transcoding.md b/Transcodes/Manual-Transcoding.md new file mode 100644 index 0000000..961feec --- /dev/null +++ b/Transcodes/Manual-Transcoding.md @@ -0,0 +1,460 @@ +# ━━━━━ TRANSCODES — Manual ━━━━━ + +Configuration reference, Docker mount setup, threshold sizing, and troubleshooting +for the ramdisk transcode system. Read the Docker mount section before anything else. + +--- + +## ━━━ CONTENTS ━━━ + +- [Docker Mount — Required Configuration](#docker-mount--required-configuration) +- [The Symlink Architecture](#the-symlink-architecture) +- [ramdisk_setup.sh](#ramdisk_setupsh) +- [transcode_manager.sh](#transcode_managersh) +- [transcode_cleanup.sh](#transcode_cleanupsh) +- [Full Configuration Reference](#full-configuration-reference) +- [Schedule](#schedule) +- [Troubleshooting](#troubleshooting) + +--- + +## Docker Mount — Required Configuration + +> **This is the most important configuration requirement in this folder.** +> Get this wrong and symlink flips silently stop working after the first flip. +> The system appears to work initially and fails subtly. + +### Required Mount + +In Emby's **Extra Parameters** in the unRAID Docker template: + +``` +--mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared +``` + +This **replaces** the transcode path in the standard template path mapping UI. Do NOT +add this via the path mapping UI — that UI does not support propagation. Extra Parameters only. + +In Emby's transcoding settings, set the transcode path to `/ext-ram-transcode`. + +### Why `shared` Is Required + +``` +rprivate (Docker's default): + Docker resolves the symlink target at first mount and locks that inode. + Flip: ramdisk → SSD → works (new target locked in) + Flip: SSD → ramdisk → Docker ignores it. Container still sees SSD binding. + All sessions continue to land on SSD forever until Emby restarts. + Symptom: symlink on host is correct, Emby still uses SSD. Confusing. + +shared: + Host mount changes propagate into the container in real time. + Every symlink flip is immediately visible inside the container. ✅ +``` + +### Verify the Mount + +```bash +# Check propagation — must show "shared": +docker inspect Emby | grep -A4 "ext-ram" +# Expected: "Propagation": "shared" + +# Check Emby's transcode path setting: +docker exec Emby cat /config/config/encoding.xml | grep TranscodingTempPath +# Expected: /ext-ram-transcode +``` + +### What NOT to Do + +Do NOT add a static SSD transcode path as a second path mapping in the template: +``` +/mnt/cache/Temp_Storage/Emby/Transcodes → /ssd-transcode ← do not do this +``` + +If the SSD path is mounted inside the container, Emby can see it as an accessible +transcode location. It will route sessions there independently of the symlink — +bypassing the management system entirely. Sessions land on SSD regardless of symlink +state. The whole system stops working. + +--- + +## The Symlink Architecture + +``` +Emby is configured to write transcodes to TRANSCODE_LINK (/mnt/ram-transcode). +TRANSCODE_LINK is a symlink — its target is managed at runtime. + +Normal operation (ramdisk has headroom): + /mnt/ram-transcode → /mnt/ramdisk_transcodes/ (ramdisk — fast, no wear) + +Heavy load (ramdisk filling up): + /mnt/ram-transcode → /mnt/cache/Temp_Storage/Emby/Transcodes/ (SSD) + +Emby doesn't know this symlink exists. It writes to /mnt/ram-transcode. +ffmpeg resolves the symlink ONCE when a session starts. +After that it holds a direct reference to the actual directory. + +Flipping the symlink has ZERO effect on sessions already in progress. +Only NEW sessions care about where the symlink currently points. +``` + +### What Lives Where + +``` +/mnt/ramdisk_transcodes/ ← tmpfs (HOST*_RAMDISK_SIZE ceiling) + transcoding-temp/ ← pre-created by ramdisk_setup.sh — always here + E0D8DC/ ← Emby session (Live TV HLS segments) + F1A9BB/ ← another session + +/mnt/ram-transcode ← symlink — managed by transcode_manager.sh + currently points at: /mnt/ramdisk_transcodes/ + +/mnt/cache/Temp_Storage/Emby/Transcodes/ ← SSD fallback + transcoding-temp/ ← also pre-created — Emby finds ramdisk version first + xyz789/ ← sessions that started when ramdisk was full +``` + +--- + +## ramdisk_setup.sh + +Creates the ramdisk, SSD fallback directory, symlink, and `transcoding-temp` on the +ramdisk. Run once at array start. Idempotent — already-mounted ramdisk exits cleanly. + +### What It Creates + +``` +1. RAMDISK_PATH (/mnt/ramdisk_transcodes) + mount -t tmpfs -o size=HOST*_RAMDISK_SIZE tmpfs /mnt/ramdisk_transcodes + tmpfs uses only as much RAM as actually needed — RAMDISK_SIZE is a ceiling. + An empty ramdisk uses essentially zero RAM. + +2. transcoding-temp/ inside the ramdisk + Pre-created so Emby always finds it here first. + Without this: Emby creates transcoding-temp at its first writable location, + which may be the SSD fallback even when the symlink points at the ramdisk. + chown nobody:users — correct ownership for Emby (PUID=99) + +3. TRANSCODE_SSD (/mnt/cache/Temp_Storage/Emby/Transcodes/) + mkdir -p — created if missing, silent if exists + Also pre-creates transcoding-temp/ inside SSD fallback for consistency + +4. TRANSCODE_LINK (/mnt/ram-transcode) + ln -sfn /mnt/ramdisk_transcodes /mnt/ram-transcode + Always reset to ramdisk at array start — clean state every boot +``` + +### Verify After Setup + +```bash +# All three must pass: +mountpoint /mnt/ramdisk_transcodes +# Expected: /mnt/ramdisk_transcodes is a mountpoint + +readlink /mnt/ram-transcode +# Expected: /mnt/ramdisk_transcodes + +ls /mnt/ramdisk_transcodes/ +# Expected: transcoding-temp/ +``` + +### Usage + +```bash +ramdisk_setup.sh # normal run (called by array_start.sh) +ramdisk_setup.sh --dry-run # show what would be created without creating +ramdisk_setup.sh --status # show current ramdisk, symlink, and SSD state +ramdisk_setup.sh --log # verbose — show each creation step +``` + +--- + +## transcode_manager.sh + +Monitors ramdisk usage and manages the symlink direction. Called second in every 3-minute +cycle by `transcode_management.sh`. + +### Three Modes + +```bash +# master.conf +TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd + +# smart (default — use in production): +# Ramdisk above RAMDISK_WARN_GB → flip symlink to SSD +# Ramdisk below RAMDISK_LOW_GB → flip symlink back to ramdisk +# Hysteresis gap prevents flip-flopping under moderate load + +# ramdisk: +# Always uses ramdisk. Warns if usage exceeds threshold. Never flips. +# Use for: light load server, guaranteed RAM performance, testing ramdisk behaviour. + +# ssd: +# Always uses SSD. Never uses ramdisk. +# Use for: post-flip drain (waiting for ramdisk sessions to end naturally), +# maintenance windows, ramdisk capacity testing. +``` + +### Threshold Sizing + +```bash +# master_host*.conf +HOST1_RAMDISK_SIZE="10G" +HOST1_RAMDISK_WARN_GB=8.8 # flip to SSD above this +HOST1_RAMDISK_LOW_GB=6.5 # flip back below this +``` + +Keep ~1.5–2.5GB hysteresis gap between WARN and LOW. Without the gap, usage hovering +near WARN causes constant flip-flopping. The gap requires multiple sessions to end +completely before flipping back — a genuine recovery, not a brief fluctuation. + +Production data from a 7-household Live TV system: +``` +Normal (2–3 streams) → ~1.5–2.0 GB +Busy evening (5–6 streams) → ~3.5–4.5 GB +Peak (8 streams, Live TV) → ~5.2 GB +Current setup: 10G ramdisk, 8.8 GB threshold → 1.2 GB safety headroom +``` + +Recommended thresholds by ramdisk size: + +| RAMDISK_SIZE | RAMDISK_WARN_GB | RAMDISK_LOW_GB | +|-------------|----------------|----------------| +| 6G | 4.8 | 3.5 | +| 8G | 6.8 | 5.5 | +| 10G | 8.8 | 6.5 | +| 12G | 10.5 | 8.5 | + +### Multi-Server Configuration + +```bash +# master.conf +TRANSCODE_SERVERS=( + "Emby|http://localhost:8096|your-api-key|emby" + + # Temporarily cover HOST2's Emby during maintenance: + # "Emby-Jayred365|http://100.x.x.x:8096|HOST2-api-key|emby" + + # Jellyfin instance (separate port): + # "Jellyfin|http://localhost:8097|jellyfin-api-key|jellyfin" +) +``` + +Type field: `emby` | `jellyfin` | `plex` — controls which API endpoint format is used. +Entries with placeholder API keys are skipped automatically. +Comment out unused entries rather than deleting — placeholders show what's available. + +**Tdarr does NOT belong here.** Tdarr encodes full video files — large working files +fill the ramdisk rapidly and cause constant flips. Tdarr belongs on SSD permanently. + +### Session Display + +``` +━━━ Active Emby Sessions ━━━ +Total: 7 | Live TV: 5 | Transcoding: 5 | Direct: 2 +Storage: ramdisk + +Sunny — ABC (WTAE) — Live TV — Transcode +Mama Bear — Cinemax — Live TV — Transcode +Gmer4Lfe — WAN Show — TV Show — Transcode +``` + +Split state during a flip (ramdisk sessions draining, new sessions on SSD): +``` +⚠️ Split state — 4 folder(s) on ramdisk / 2 on SSD +Storage: ramdisk (4) + SSD (2) +``` + +### Usage + +```bash +transcode_manager.sh # normal run +transcode_manager.sh --dry-run # preview flip decision without flipping +transcode_manager.sh --status # show current state, usage, sessions, flip history +transcode_manager.sh --log # verbose output per safety check +transcode_manager.sh --no-log # suppress daily log write +``` + +--- + +## transcode_cleanup.sh + +Removes stale transcode files from ramdisk and SSD fallback. Called first in every +3-minute cycle — cleanup before usage measurement is non-negotiable. + +### Deletion Rules + +A file is eligible for deletion only when ALL conditions are true: +1. **Older than TRANSCODE_MAX_AGE minutes** (mtime — last write time). Active segments + are written every few seconds. Not touched in 20 minutes = session ended. +2. **Not currently open by any process** (lsof pre-built map, O(1) lookup per file). + If ffmpeg has a file open, it is not deleted regardless of age. + +`transcoding-temp/` is **never deleted**, even when empty. Protected by name exclusion +in the find command — deleting it causes Emby to route all sessions to the SSD fallback. + +### Usage + +```bash +transcode_cleanup.sh # normal cleanup run +transcode_cleanup.sh --dry-run # show what would be deleted +transcode_cleanup.sh --status # show file counts, ages, open-file status per location +transcode_cleanup.sh --log # verbose per-file output +``` + +--- + +## Full Configuration Reference + +```bash +# master.conf + +# ── Paths ────────────────────────────────────────────────────────────────────── +TRANSCODE_LINK="/mnt/ram-transcode" # symlink Emby points at +TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" # SSD fallback + +# ── Per-Host (master_host*.conf) ─────────────────────────────────────────────── +HOST1_RAMDISK_PATH="/mnt/ramdisk_transcodes" # ramdisk mount point +HOST1_RAMDISK_SIZE="10G" # tmpfs ceiling (not a reservation) +HOST1_RAMDISK_WARN_GB=8.8 # flip to SSD above this +HOST1_RAMDISK_LOW_GB=6.5 # flip back below this + +# ── Thresholds ───────────────────────────────────────────────────────────────── +RAMDISK_SSD_MIN_GB=20 # minimum SSD free space before allowing SSD flip + # prevents accidentally filling the SSD cache pool +TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in one hour + # high flip count = ramdisk undersized for the load + +# ── Cleanup ──────────────────────────────────────────────────────────────────── +TRANSCODE_MAX_AGE=20 # minutes — files older than this are stale +TRANSCODE_ORPHAN_AGE=30 # minutes — orphaned session folders removed after this + +# ── Manager Mode ─────────────────────────────────────────────────────────────── +TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd +TRANSCODE_CHECK_EMBY=true # skip threshold checks when Emby not running + # prevents unnecessary flips overnight + +# ── Permissions ──────────────────────────────────────────────────────────────── +TRANSCODE_OWNER="nobody:users" # matches PUID=99 PGID=100 container env +TRANSCODE_CHMOD="755" + +# ── Daily Log ────────────────────────────────────────────────────────────────── +TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db" +TRANSCODE_LOG_RETENTION=90 # days — trimmed on every write +TRANSCODE_STATE_FILE="/tmp/transcode_state.db" # /tmp — resets on reboot correctly + +# ── Multi-Server ─────────────────────────────────────────────────────────────── +TRANSCODE_SERVERS=( + "ContainerName|http://host:port|api-key|emby" # type: emby | jellyfin | plex +) +``` + +--- + +## Schedule + +``` +At Startup of Array (via array_start.sh in unRAID_Essentials/): + ramdisk_setup.sh — creates ramdisk, symlink, transcoding-temp + Must run BEFORE Emby starts + +Every 3 minutes (via transcode_management.sh in Orchestrators/): + 1. transcode_cleanup.sh — remove stale files, check flip-back + 2. transcode_manager.sh — check usage, flip if needed, display sessions + +Do NOT schedule transcode_manager.sh or transcode_cleanup.sh directly. +``` + +--- + +## Troubleshooting + +### Sessions Landing on SSD Despite Symlink Pointing at Ramdisk + +```bash +# Check 1 — Docker mount propagation (most common cause): +docker inspect Emby | grep Propagation +# Expected: "Propagation": "shared" +# Wrong: "Propagation": "rprivate" +# +# Fix: Update Emby Extra Parameters, restart Emby: +# --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared +# Remove any standard path mapping for the transcode directory. + +# Check 2 — transcoding-temp exists on ramdisk: +ls /mnt/ramdisk_transcodes/ +# Expected: transcoding-temp/ +# +# Fix if missing: +mkdir -p /mnt/ramdisk_transcodes/transcoding-temp +chown nobody:users /mnt/ramdisk_transcodes/transcoding-temp + +# Check 3 — no duplicate SSD mount in Emby template: +docker inspect Emby | grep -A3 "Mounts" +# Should show only /mnt/ram-transcode → /ext-ram-transcode +# Should NOT show /mnt/cache/Temp_Storage/... as a second mount +``` + +### Flip Count High — 3+ Per Hour + +```bash +# Ramdisk filling up regularly — load exceeds the current ceiling. +# Check peak usage from the weekly health digest: Transcodes → "Week peak: X.XGB" +# +# If peak is close to RAMDISK_WARN_GB → increase ramdisk size: +# master_host1.conf +HOST1_RAMDISK_SIZE="12G" # increase by 2G +HOST1_RAMDISK_WARN_GB=10.5 # adjust thresholds accordingly +HOST1_RAMDISK_LOW_GB=8.5 + +# Remount at new size — run ramdisk_setup.sh manually: +ramdisk_setup.sh --log +# Ramdisk must be unmounted first if already mounted: +# umount /mnt/ramdisk_transcodes && ramdisk_setup.sh --log +``` + +### Ramdisk Not Mounting at Array Start + +```bash +# Check if tmpfs is mounted: +mountpoint /mnt/ramdisk_transcodes +# "not a mountpoint" → setup failed or not run yet + +# Run manually to see the error: +ramdisk_setup.sh --log + +# Common causes: +# /mnt/ramdisk_transcodes missing → mkdir -p /mnt/ramdisk_transcodes +# Insufficient RAM → check free RAM: free -h +# RAMDISK_SIZE too large → reduce HOST*_RAMDISK_SIZE +``` + +### Emergency Manual Flip + +```bash +# Flip to SSD immediately — all new sessions go to SSD: +ln -sfn /mnt/cache/Temp_Storage/Emby/Transcodes /mnt/ram-transcode + +# Flip back to ramdisk — all new sessions go to ramdisk: +ln -sfn /mnt/ramdisk_transcodes /mnt/ram-transcode + +# Check current symlink target: +readlink /mnt/ram-transcode + +# Existing sessions in progress are NEVER affected — only new sessions follow the flip. +``` + +### Increasing Ramdisk Size After Initial Setup + +```bash +# 1. Set new size and thresholds in master_host*.conf +# 2. Unmount the existing ramdisk (no sessions should be active): +umount /mnt/ramdisk_transcodes + +# 3. Re-run setup to mount at new size: +ramdisk_setup.sh --log + +# 4. Verify: +df -h /mnt/ramdisk_transcodes +# Should show new size as total +``` diff --git a/Transcodes/README-Transcoding.md b/Transcodes/README-Transcoding.md index 88bf41f..7808938 100644 --- a/Transcodes/README-Transcoding.md +++ b/Transcodes/README-Transcoding.md @@ -1,724 +1,137 @@ -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# 🎬 TRANSCODING -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# ━━━━━ TRANSCODES ━━━━━ -**Ramdisk-based transcode storage with automatic SSD fallback, session-safe symlink -flipping, and stale file cleanup.** Emby transcodes to RAM at full speed. If the ramdisk -fills up, new sessions automatically shift to SSD — without interrupting anything -already playing. When pressure drops, new sessions shift back. +**Ramdisk-based transcode storage with automatic SSD fallback.** Emby transcodes to RAM +at full speed. When the ramdisk fills, new sessions shift to SSD automatically — +without interrupting anything already playing. When pressure drops, new sessions shift +back to RAM. -> **This system has two subtle configuration requirements that are not obvious and -> both were discovered the hard way in production.** The Docker mount must use -> `bind-propagation=shared` or symlink flips are silently ignored after the first -> flip. The `transcoding-temp` directory must be pre-created on the ramdisk or Emby -> finds the SSD version and routes all sessions there until restarted. Both are -> documented and both will bite you if missed. +> **Two configuration requirements that are not obvious and were both discovered the +> hard way in production.** The Docker mount must use `bind-propagation=shared` or +> symlink flips are silently ignored after the first flip. The `transcoding-temp` +> directory must be pre-created on the ramdisk or Emby finds the SSD version and +> routes all sessions there until restarted. Both are documented in Manual-Transcoding.md. --- -## ━━━ THE PROBLEM THAT BUILT THIS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ━━━ THE PROBLEM THAT BUILT THIS ━━━ + +**Three Storage Options, None Perfect on Their Own** +Hard drives: seek times cause buffering on multi-stream transcoding. SSD: fast enough, +but constant small file writes at Emby volume accelerate wear over months. RAM: fastest, +no wear, files vanish instantly on session end — but limited by available memory. +Fix: RAM by default, SSD as a safety net. The system manages the transition automatically. + +**Changing Transcode Location Requires Restarting Emby** +Configuring Emby to switch between ramdisk and SSD requires a restart. Restarting +during active streams drops everyone. A 7-person household with 5 Live TV streams at +9pm is not a good moment to restart Emby. +Fix: symlink indirection. Emby points at a fixed path. The symlink target changes. +ffmpeg resolves the symlink once at session start — existing sessions are completely +unaffected by flips. Only new sessions follow the new target. + +**Docker Bind Mount Silently Ignored After First Flip** +Symlink flip from ramdisk → SSD worked. Flip back: nothing. All new sessions still land +on SSD. The symlink on the host is correct. Emby doesn't see it. +Cause: Docker's default `rprivate` propagation resolves the symlink target at mount time +and locks that inode. Subsequent flips are invisible to the container. +Fix: `bind-propagation=shared` in Extra Parameters. Host mount changes propagate into +the container in real time. Requires `--mount` syntax — the path mapping UI doesn't +support propagation. + +**Sessions Drifting to SSD After a Day of Operation** +System working correctly for hours, then sessions gradually drift to SSD despite the +ramdisk having plenty of space. +Cause: cleanup was removing the empty `transcoding-temp` directory from the ramdisk. +Emby then found the SSD fallback version and routed all sessions there. +Fix: `transcoding-temp` is excluded from cleanup by name. `ramdisk_setup.sh` pre-creates +it at mount time. Both protections together prevent this permanently. + +**lsof Per File on a Live TV System** +Early cleanup called `lsof filename` per file to check if anything had it open. On a busy +Live TV night with 5 simultaneous streams, the ramdisk contains thousands of HLS segment +files — thousands of subprocess calls every 3 minutes. +Fix: lsof called once per location to build a complete open-file map. All subsequent +checks are O(1) lookups against that map. --- -### 🔴 Three Storage Options, None Perfect on Their Own +## ━━━ WHAT THIS FOLDER DOES ━━━ -Emby transcodes generate hundreds of small HLS segment files written and read -continuously at high throughput. Where those files live matters a lot: +Three scripts, one goal: keep transcodes on RAM, fall back to SSD when needed. -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Hard drives: -# Seek times cause buffering on simultaneous multi-stream transcoding. -# 5 streams trying to seek on spinning disks = constant buffering for everyone. -# -# SSD (cache pool): -# Fast enough for any realistic load. -# But: constant small file writes at Emby volume accelerate SSD wear. -# A busy Live TV night writes and deletes thousands of segment files. -# Over months, this adds up. -# -# RAM (tmpfs): -# Fastest possible — no disk I/O at all. -# No wear — RAM doesn't have write cycles. -# Files disappear instantly on session end — no cleanup needed for normal exits. -# One risk: running out of RAM during heavy load. -# ───────────────────────────────────────────────────────────────────────────── -``` +`ramdisk_setup.sh` runs at array start — creates the tmpfs, SSD fallback directory, +symlink, and pre-creates `transcoding-temp`. Everything that must exist before Emby starts. -The correct answer is RAM by default, SSD as a safety net. The ramdisk handles normal -operation. The SSD absorbs unexpected load spikes. The system manages the transition -automatically. +`transcode_cleanup.sh` runs first in every 3-minute cycle — removes stale files from both +ramdisk and SSD. Cleans up before usage is measured, so the manager sees real load. + +`transcode_manager.sh` runs second — measures ramdisk usage, flips the symlink if +thresholds are crossed, runs safety checks, displays active sessions, writes the daily log. + +The symlink is the mechanism that makes this seamless. Emby writes to a fixed path. That +path is a symlink whose target is managed at runtime. Sessions in progress never notice. --- -### 🔴 Changing Transcode Location Requires Restarting Emby - -The obvious approach — configure Emby to use the ramdisk, configure SSD as fallback in -Emby's settings — requires restarting Emby to switch between them. Restarting Emby -during active streams drops everyone. A 7-person household with 5 Live TV streams -running at 9pm is not a good moment to restart Emby. - -The fix: symlink indirection. Emby points at a fixed path that never changes. -The symlink target changes. ffmpeg resolves the symlink once at session start and -holds the resolved path — existing sessions are completely unaffected by symlink -changes. Only new sessions care about where the symlink currently points. - -The transition is seamless. Sessions in progress when the flip happens continue -writing to wherever they started. Only new sessions after the flip go to the new -location. No restart. No interruption. - ---- - -### 🔴 Docker Bind Mount Silently Ignored After First Flip - -Got the symlink system working. First flip from ramdisk to SSD: works. Flip back to -ramdisk: nothing. All new sessions still land on SSD. The symlink on the host clearly -points at the ramdisk — `readlink /mnt/ram-transcode` shows the correct path — but -Emby keeps writing to SSD. - -The cause: Docker's default bind mount uses `rprivate` propagation. With `rprivate`, -Docker resolves the symlink target at first mount and locks that inode. When the -symlink flips, Docker ignores it — it already has a private SSD binding locked in. -The container sees the path but the mount behind it doesn't update. - -The fix: `bind-propagation=shared` in Extra Parameters. With shared propagation, host -mount changes propagate into the container in real time. Symlink flips on the host are -immediately visible inside the container. This requires using `--mount` syntax instead -of a standard template path mapping — that syntax supports propagation, the path -mapping UI does not. - ---- - -### 🔴 Sessions Drifting to SSD After a Day of Operation - -System working correctly for hours. Then gradually sessions start landing on SSD even -though the ramdisk has plenty of space and the symlink points at the ramdisk. Next day -all sessions are on SSD. - -The cause: cleanup was removing the empty `transcoding-temp` directory from the -ramdisk. When `transcoding-temp` doesn't exist on the ramdisk, Emby searches its -accessible paths for an existing one. It finds the SSD fallback version. All subsequent -sessions route there until Emby is restarted. - -The fix: `transcoding-temp` is protected from cleanup — excluded by name from the -`find` command. And `ramdisk_setup.sh` pre-creates it at mount time so Emby always -finds it on the ramdisk first. Both fixes together prevent this permanently. - ---- - -### 🔴 lsof Per File on a Live TV System - -Early cleanup implementation called `lsof filename` per file to check if anything had -it open. On a busy Live TV night with 5 simultaneous streams, the ramdisk contains -thousands of HLS segment files. Calling `lsof` once per file was creating thousands -of subprocess calls every 3 minutes. The cleanup script was spending more time on -lsof calls than on actual cleanup. - -The fix: `lsof` is called once per location to build a complete open-file map. All -subsequent file checks are O(1) lookups against that pre-built map. Thousands of files, -one `lsof` call, no performance penalty. - ---- - -## ━━━ THE DESIGN ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -### ── The Symlink Architecture ───────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Emby is configured to write transcodes to TRANSCODE_LINK. -# TRANSCODE_LINK is a symlink — its target is managed at runtime. -# -# Normal operation (ramdisk has headroom): -# /mnt/ram-transcode → /mnt/ramdisk_transcodes/ (ramdisk, fast, no wear) -# -# Heavy load (ramdisk filling up): -# /mnt/ram-transcode → /mnt/cache/Temp_Storage/Emby/Transcodes/ (SSD) -# -# Emby doesn't know this symlink exists. It writes to /mnt/ram-transcode. -# ffmpeg resolves the symlink ONCE when a session starts. -# After that it holds a direct reference to the actual directory. -# Flipping the symlink has ZERO effect on sessions already in progress. -# Only NEW sessions care about where the symlink currently points. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── The Threshold Logic ────────────────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# Two thresholds with a hysteresis gap to prevent flip-flopping. -# - RAMDISK_WARN_GB=8.8 # flip to SSD when ramdisk usage exceeds this - RAMDISK_LOW_GB=6.5 # flip back to ramdisk when usage drops below this -# -# The 2.3GB hysteresis gap: -# Without this gap: usage hovers at 8.7GB → flip to SSD → sessions drain -# → usage drops to 8.5GB → flip back → new sessions fill → flip again -# The symlink would oscillate every few minutes under moderate load. -# -# With the gap: usage must drop all the way to 6.5GB before flipping back. -# That requires multiple sessions to end completely — a genuine recovery, -# not a brief fluctuation. Stable, predictable behaviour. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── What Lives Where ────────────────────────────────────────────────────────── +## ━━━ RELATIONSHIP TO OTHER FOLDERS ━━━ ``` -/mnt/ramdisk_transcodes/ ← tmpfs (HOST*_RAMDISK_SIZE ceiling) - transcoding-temp/ ← pre-created by ramdisk_setup.sh — always here - E0D8DC/ ← Emby session (Live TV HLS segments) - F1A9BB/ ← another session +unRAID_Essentials/ + array_start.sh ──────────────────────────────► ramdisk_setup.sh (at array start) -/mnt/ram-transcode ← symlink — managed at runtime by transcode_manager.sh - currently points at: /mnt/ramdisk_transcodes/ +Orchestrators/ + transcode_management.sh ──── cleanup first ──► transcode_cleanup.sh + ──── then manager ──► transcode_manager.sh + (every 3 minutes — order non-negotiable) -/mnt/cache/Temp_Storage/Emby/Transcodes/ ← SSD fallback - transcoding-temp/ ← also pre-created — Emby finds ramdisk version first - xyz789/ ← sessions that started when ramdisk was full +Monitors/ + weekly_health_digest.sh ◄─── reads ──────────── TRANSCODE_DAILY_LOG ``` ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## ⚙️ DOCKER MOUNT — READ THIS BEFORE ANYTHING ELSE -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -> **This is the most important configuration requirement in the entire folder.** -> Get this wrong and symlink flips silently stop working after the first flip. -> The system appears to work initially and fails subtly. +Do not schedule `transcode_cleanup.sh` or `transcode_manager.sh` directly. +Both are called by `transcode_management.sh` in the correct order. --- -### ── Required Mount Configuration ─────────────────────────────────────────── +## ━━━ SCRIPTS IN THIS FOLDER ━━━ -```bash -# In Emby's Extra Parameters in the unRAID Docker template: -# ───────────────────────────────────────────────────────────────────────────── ---mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared -# ───────────────────────────────────────────────────────────────────────────── -# This REPLACES the transcode path in the standard template path mapping UI. -# Do NOT add this via the path mapping UI — that UI does not support propagation. -# Use Extra Parameters only. -``` +| Script | Role | When It Runs | +|--------|------|-------------| +| `ramdisk_setup.sh` | Create tmpfs, SSD fallback dir, symlink, transcoding-temp | At array start (via array_start.sh) | +| `transcode_cleanup.sh` | Remove stale files, check for flip-back opportunity | Every 3 min via transcode_management.sh — runs first | +| `transcode_manager.sh` | Check usage, flip symlink, safety checks, session display, daily log | Every 3 min via transcode_management.sh — runs second | --- -### ── Why `shared` Is Required ──────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# rprivate (Docker's default): -# Docker resolves the symlink target at first mount and locks that inode. -# Flip: ramdisk → SSD → flip works. -# Flip: SSD → ramdisk → Docker ignores it. Container still sees SSD binding. -# All sessions continue to land on SSD forever until Emby restarts. -# Symptom: symlink on host is correct, Emby still uses SSD. Confusing. -# -# shared: -# Host mount changes propagate into the container in real time. -# Flip: ramdisk → SSD → visible in container immediately ✅ -# Flip: SSD → ramdisk → visible in container immediately ✅ -# System works as designed. Every flip is immediate and correct. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Verify the Mount ───────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Check the propagation — must show "shared" not "rprivate": -docker inspect Emby | grep -A4 "ext-ram" -# Expected output includes: "Propagation": "shared" -# Wrong output: "Propagation": "rprivate" - -# Check Emby's transcode path setting (inside container): -docker exec Emby cat /config/config/encoding.xml | grep TranscodingTempPath -# Expected: /ext-ram-transcode -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── What NOT to Do ────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# DO NOT add a static SSD transcode path as a second path mapping in the template: -# /mnt/cache/Temp_Storage/Emby/Transcodes → /ssd-transcode -# -# If the SSD path is mounted inside the container, Emby can see it as an -# accessible transcode location. It will route sessions there independently of -# the symlink — completely bypassing the management system. Sessions land on SSD -# regardless of symlink state. The whole system stops working. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🚀 ramdisk_setup.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Creates the ramdisk, SSD fallback directory, symlink, and `transcoding-temp` on the -ramdisk. Run once at array start. Idempotent — if the ramdisk is already mounted it -reports status and exits cleanly. - -```bash -# Scheduled: At Startup of Array (via array_start.sh) -# This runs BEFORE Emby starts — order matters in ARRAY_START_SCRIPTS -``` - ---- - -### ── What It Creates ────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# 1. RAMDISK_PATH (/mnt/ramdisk_transcodes) -# mount -t tmpfs -o size=HOST*_RAMDISK_SIZE tmpfs /mnt/ramdisk_transcodes -# In-memory tmpfs — uses only as much RAM as actually needed. -# RAMDISK_SIZE is a ceiling, not a reservation — an empty ramdisk uses ~0 RAM. -# -# 2. transcoding-temp inside the ramdisk -# mkdir -p /mnt/ramdisk_transcodes/transcoding-temp -# Pre-created so Emby always finds it here first. -# Without this: Emby creates transcoding-temp at its first writable location, -# which may be the SSD fallback even when the symlink points at the ramdisk. -# chown nobody:users — correct ownership for Emby to write as PUID=99 -# -# 3. TRANSCODE_SSD (/mnt/cache/Temp_Storage/Emby/Transcodes/) -# mkdir -p — creates if missing, silent if exists -# Also pre-creates transcoding-temp/ inside SSD fallback for consistency -# -# 4. TRANSCODE_LINK (/mnt/ram-transcode) -# ln -sfn /mnt/ramdisk_transcodes /mnt/ram-transcode -# Always reset to ramdisk at array start — clean state every boot -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Verify After Setup ─────────────────────────────────────────────────────── - -```bash -# Run these after first setup — all three must pass: - -mountpoint /mnt/ramdisk_transcodes -# Expected: /mnt/ramdisk_transcodes is a mountpoint - -readlink /mnt/ram-transcode -# Expected: /mnt/ramdisk_transcodes - -ls /mnt/ramdisk_transcodes/ -# Expected: transcoding-temp/ -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -ramdisk_setup.sh # normal run (at array start via array_start.sh) -ramdisk_setup.sh --dry-run # show what would be created without creating -ramdisk_setup.sh --status # show current ramdisk, symlink, and SSD state -ramdisk_setup.sh --log # verbose — show each creation step -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🔄 transcode_manager.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Monitors ramdisk usage and manages the symlink direction. Called by -`transcode_management.sh` — not scheduled directly. Every 3 minutes it checks usage, -makes a flip decision if needed, runs safety checks, and shows active sessions. - -```bash -# Called by: transcode_management.sh (every 3 minutes) -# Not scheduled directly — use transcode_management.sh -``` - ---- - -### ── Three Modes ────────────────────────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── - TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd -# -# smart: -# Auto-flips between ramdisk and SSD based on usage thresholds. -# Normal operation — use this in production. -# Ramdisk above RAMDISK_WARN_GB → flip to SSD. -# Ramdisk below RAMDISK_LOW_GB → flip back to ramdisk. -# -# ramdisk: -# Always uses ramdisk. Never flips to SSD. -# Use: light load server, guaranteed RAM performance, testing ramdisk behaviour. -# Warning logged if usage exceeds threshold — no automatic action. -# -# ssd: -# Always uses SSD. Never uses ramdisk. -# Use: post-flip drain (waiting for ramdisk sessions to end naturally), -# maintenance windows, ramdisk capacity testing. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Safety Checks — Every Run Regardless of Mode ─────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# These run on EVERY cycle — they protect against state drift: -# -# Symlink missing or broken -# → Recreate pointing at ramdisk, notify -# → Can happen after manual intervention or filesystem issue -# -# Ramdisk disappeared (unmounted) -# → Auto-flip to SSD immediately, notify warning -# → Can happen if tmpfs was manually unmounted or system ran out of memory -# -# SSD path missing -# → Disable SSD fallback (mode=ssd: error) -# → Can happen if SSD pool is not mounted -# -# transcoding-temp missing from ramdisk -# → Recreate immediately, no notification -# → Prevents sessions silently routing to SSD version -# -# Permissions drift -# → Fix silently every run -# → nobody:users ownership, TRANSCODE_CHMOD mode -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Session Display ────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Each run shows active sessions from all configured TRANSCODE_SERVERS: -# -━━━ 🎬 Active Emby Sessions ━━━ -🎬 Total: 7 | 💨 Live TV: 5 | 🔄 Transcoding: 5 | 🏁 Direct: 2 -🔗 Storage: 💨 ramdisk - -🎬 Sunny — ABC (WTAE) — Live TV — Transcode -🎬 Mama Bear — Cinemax — Live TV — Transcode -🎬 Gmer4Lfe — WAN Show — TV Show — Transcode - -# Split state — sessions on both ramdisk and SSD simultaneously: -# Normal during a flip — ramdisk sessions draining, new sessions on SSD -# -⚠️ Split state — 4 folder(s) on ramdisk / 2 on SSD -🔗 Storage: 💨 ramdisk (4) + 💾 SSD (2) -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Multi-Server Configuration ───────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# Any number of media servers can share the same ramdisk scratch space. -# They never touch each other's files — each writes to its own session subfolder. -# Format: "ContainerName|URL|APIKey|Type" -# -TRANSCODE_SERVERS=( - "Emby|http://localhost:8096|0c27448d93a7431f9ac63569f7655829|emby" - - # Temporarily cover HOST2's Emby during maintenance: - # "Emby-Jayred365|http://100.x.x.x:8096|HOST2-api-key|emby" - - # Jellyfin instance (separate port): - # "Jellyfin|http://localhost:8097|jellyfin-api-key|jellyfin" -) -# -# Type field controls which API endpoint format is used: -# emby → /Sessions endpoint -# jellyfin → /Sessions endpoint (same format, same code path) -# plex → /status/sessions (different format) -# -# ⚠️ Tdarr does NOT belong here. -# Tdarr encodes full video files — large working files would fill the ramdisk -# rapidly and cause constant flips. Tdarr belongs on SSD permanently. -# dedicated tdarr_cleanup.sh handles Tdarr orphan management separately. -# -# Entries with placeholder API keys are skipped automatically. -# Comment out unused entries rather than deleting — placeholders show what's available. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Sizing the Ramdisk ─────────────────────────────────────────────────────── - -```bash -# master_host1.conf -# ───────────────────────────────────────────────────────────────────────────── -# tmpfs uses only as much RAM as actually needed — RAMDISK_SIZE is a ceiling. -# An empty ramdisk uses essentially zero RAM. -# - HOST1_RAMDISK_SIZE="10G" # 10GB ceiling — verified against production usage below - -# Production data from this setup (7-household Live TV system): -# Normal (2-3 streams) → ~1.5-2.0GB -# Busy evening (5-6 streams) → ~3.5-4.5GB -# Peak (8 streams, Live TV) → ~5.2GB -# Current ramdisk → 10G with 8.8GB threshold → 1.2GB safety headroom -# ───────────────────────────────────────────────────────────────────────────── - -# Threshold sizing — keep ~1.5-2.5GB hysteresis gap: -# -# ┌──────────────┬─────────────────┬────────────────┐ -# │ RAMDISK_SIZE │ RAMDISK_WARN_GB │ RAMDISK_LOW_GB │ -# ├──────────────┼─────────────────┼────────────────┤ -# │ 6G │ 4.8 │ 3.5 │ -# │ 8G │ 6.8 │ 5.5 │ -# │ 10G │ 8.8 │ 6.5 │ -# │ 12G │ 10.5 │ 8.5 │ -# └──────────────┴─────────────────┴────────────────┘ -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🧹 transcode_cleanup.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Removes stale transcode files from both ramdisk and SSD fallback locations. Called by -`transcode_management.sh` before `transcode_manager.sh` — order is critical. - -```bash -# Called by: transcode_management.sh (cleanup runs BEFORE manager) -# Not scheduled directly — use transcode_management.sh -``` - ---- - -### ── Deletion Rules ──────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# A file is eligible for deletion only when ALL conditions are true: -# -# 1. Older than TRANSCODE_MAX_AGE minutes (mtime — last write time) -# Active segments are being written every few seconds. -# A file not touched in 20 minutes is from a session that ended. -# -# 2. Not currently open by any process -# lsof map built once per location — O(1) lookup per file. -# If ffmpeg has a file open, it is not deleted regardless of age. -# "Session ended in API but ffmpeg still writing" → safe, not deleted. -# -# transcoding-temp directory: -# NEVER deleted, even when empty. -# Protected by name exclusion in find command. -# Deleting it causes Emby to find the SSD version and route there permanently. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Why Not Session-Aware Cleanup ──────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# ffmpeg generates its own folder names inside transcoding-temp, independently -# of the media server API session IDs. There is no reliable mapping between -# API session IDs and the actual folder names on disk. -# -# Attempting to correlate them: session E0D8DC → folder E0D8DC → safe assumption? -# No. The folder name is an internal ffmpeg identifier. It may match, may not. -# Using this correlation would falsely treat active sessions as ended. -# -# lsof is the correct check: -# If ffmpeg has a file open, the file is active regardless of session state. -# If no process has the file open, the file is safely deletable. -# No correlation needed. No race condition. Always correct. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Flip-Back After Cleanup ────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# transcode_cleanup.sh checks ramdisk usage AFTER removing stale files. -# If usage dropped below RAMDISK_LOW_GB → triggers flip-back to ramdisk. -# This handles the recovery direction so transcode_manager.sh doesn't have to. -# -# Without cleanup running first, the manager would see inflated usage from stale -# files and potentially flip to SSD unnecessarily. -# With cleanup running first, the manager always sees real active session usage. -# This is why the order in transcode_management.sh is non-negotiable. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -## ━━━ FULL CONFIGURATION REFERENCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── - -# ── Paths ────────────────────────────────────────────────────────────────── - TRANSCODE_LINK="/mnt/ram-transcode" # the symlink Emby points at - TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" # SSD fallback - -# ── Per-Host (master_host*.conf) ────────────────────────────────────────── - HOST1_RAMDISK_PATH="/mnt/ramdisk_transcodes" # ramdisk mount point - HOST1_RAMDISK_SIZE="10G" # tmpfs ceiling (not a reservation) - HOST1_RAMDISK_WARN_GB=8.8 # flip to SSD above this - HOST1_RAMDISK_LOW_GB=6.5 # flip back below this - -# ── Thresholds ───────────────────────────────────────────────────────────── - RAMDISK_SSD_MIN_GB=20 # minimum SSD free space before allowing SSD flip - # prevents filling the SSD cache pool accidentally - TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in an hour - # high flip count = ramdisk undersized - -# ── Cleanup ──────────────────────────────────────────────────────────────── - TRANSCODE_MAX_AGE=20 # minutes — files older than this are stale - TRANSCODE_ORPHAN_AGE=30 # minutes — orphaned session folders removed after this - -# ── Manager Mode ─────────────────────────────────────────────────────────── - TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd - TRANSCODE_CHECK_EMBY=true # skip threshold checks when Emby not running - # prevents unnecessary flips at night - -# ── Permissions ──────────────────────────────────────────────────────────── - TRANSCODE_OWNER="nobody:users" # matches PUID=99 PGID=100 container env - TRANSCODE_CHMOD="755" - -# ── Daily Log ────────────────────────────────────────────────────────────── - TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db" - TRANSCODE_LOG_RETENTION=90 # days — bounded, trimmed on every write - TRANSCODE_STATE_FILE="/tmp/transcode_state.db" # /tmp — resets on reboot - -# ── Multi-Server ─────────────────────────────────────────────────────────── - TRANSCODE_SERVERS=( - "ContainerName|http://host:port|api-key|type" # emby|jellyfin|plex - ) -``` - ---- - -## ━━━ SCHEDULE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ━━━ HOW THE SCRIPTS RELATE ━━━ ``` -At Startup of Array: - ramdisk_setup.sh ← via array_start.sh — creates ramdisk + symlink + transcoding-temp +Array starts + │ + ▼ +ramdisk_setup.sh + Creates: /mnt/ramdisk_transcodes (tmpfs) + /mnt/ramdisk_transcodes/transcoding-temp/ + /mnt/cache/Temp_Storage/Emby/Transcodes/ (SSD fallback) + /mnt/ram-transcode → /mnt/ramdisk_transcodes (symlink) + │ + ▼ +Emby starts, reads transcode path from config + Sees: /ext-ram-transcode (bind-mounted from /mnt/ram-transcode) + All new sessions write to: /mnt/ram-transcode → /mnt/ramdisk_transcodes/ -Every 3 minutes: - transcode_management.sh ← cleanup first, then manager — order non-negotiable - 1. transcode_cleanup.sh ← remove stale files, check if flip-back possible - 2. transcode_manager.sh ← check usage, flip if needed, show sessions -Do NOT schedule transcode_manager.sh or transcode_cleanup.sh directly. +Every 3 minutes (transcode_management.sh): + │ + ├─ transcode_cleanup.sh + │ Remove files older than TRANSCODE_MAX_AGE, not open by any process + │ transcoding-temp: never deleted + │ If ramdisk recovered below RAMDISK_LOW_GB → trigger flip-back + │ + └─ transcode_manager.sh + Safety checks (symlink, ramdisk mount, transcoding-temp, permissions) + smart mode: ramdisk > RAMDISK_WARN_GB → flip symlink to SSD + ramdisk < RAMDISK_LOW_GB → flip symlink back to ramdisk + Session display (all TRANSCODE_SERVERS) + Append to TRANSCODE_DAILY_LOG ``` - ---- - -## ━━━ TROUBLESHOOTING ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - ---- - -### 🔴 Sessions Landing on SSD Despite Symlink Pointing at Ramdisk - -```bash -# Check 1 — Docker mount propagation (most common cause): -docker inspect Emby | grep Propagation -# Expected: "Propagation": "shared" -# Wrong: "Propagation": "rprivate" -# -# Fix: Add to Emby Extra Parameters and restart Emby: -# --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared -# Remove any standard path mapping for the transcode directory. - -# Check 2 — transcoding-temp exists on ramdisk: -ls /mnt/ramdisk_transcodes/ -# Expected: transcoding-temp/ -# -# Fix if missing: -mkdir -p /mnt/ramdisk_transcodes/transcoding-temp -chown nobody:users /mnt/ramdisk_transcodes/transcoding-temp - -# Check 3 — no duplicate SSD mount in Emby template: -docker inspect Emby | grep -A3 "Mounts" -# Should show: only /mnt/ram-transcode → /ext-ram-transcode -# Should NOT show: /mnt/cache/Temp_Storage/... as a second mount -``` - ---- - -### 🔴 Flip Count High — 3+ Per Hour - -```bash -# Ramdisk filling up regularly — sessions draining before more arrive. -# Check peak usage from the weekly coffee report: -# Transcodes section → "Week peak: X.XGB" -# -# If peak is close to RAMDISK_WARN_GB → increase ramdisk: -# master_host1.conf -HOST1_RAMDISK_SIZE="12G" # increase by 2G -HOST1_RAMDISK_WARN_GB=10.5 # adjust thresholds accordingly -HOST1_RAMDISK_LOW_GB=8.5 -# -# Then re-run ramdisk_setup.sh to remount at new size: -bash /mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh -``` - ---- - -### 🔴 Ramdisk Not Mounting at Array Start - -```bash -# Check if tmpfs mounted: -mountpoint /mnt/ramdisk_transcodes -# "not a mountpoint" → setup failed or not run yet - -# Run manually to see the error: -bash /mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh --log - -# Common causes: -# /mnt/ramdisk_transcodes directory missing → mkdir -p /mnt/ramdisk_transcodes -# Insufficient RAM → check free RAM: free -h -# RAMDISK_SIZE too large for available RAM → reduce HOST*_RAMDISK_SIZE -``` - ---- - -### 🔴 Emergency Manual Flip - -```bash -# Flip to SSD immediately — all new sessions go to SSD: -ln -sfn /mnt/cache/Temp_Storage/Emby/Transcodes /mnt/ram-transcode - -# Flip back to ramdisk — all new sessions go to ramdisk: -ln -sfn /mnt/ramdisk_transcodes /mnt/ram-transcode - -# Check current symlink target: -readlink /mnt/ram-transcode - -# Existing sessions in progress are NEVER affected by these changes. -# Only new sessions follow the new symlink target. -``` \ No newline at end of file diff --git a/Transcodes/ramdisk_setup.sh b/Transcodes/ramdisk_setup.sh index fcecbae..e7e1a8b 100755 --- a/Transcodes/ramdisk_setup.sh +++ b/Transcodes/ramdisk_setup.sh @@ -2,61 +2,105 @@ # ============================================================================================== # ================================= Ramdisk Setup ============================================== # ============================================================================================== -# Creates a tmpfs ramdisk for Emby transcodes and points the transcode symlink at it. -# Run once at array start via User Scripts plugin — scheduled as "At Startup of Array". -# If ramdisk is already mounted reports status and exits cleanly without remounting. # -# ── WHAT IT CREATES ─────────────────────────────────────────────────────────────────────────── -# RAMDISK_PATH — tmpfs mount point (in-memory transcode location) -# Size: HOST*_RAMDISK_SIZE (e.g. 8G) — must fit in available RAM -# TRANSCODE_SSD — SSD fallback directory (created if missing) -# transcode_manager.sh flips symlink here if ramdisk fills up -# TRANSCODE_LINK — symlink pointing at RAMDISK_PATH by default -# transcoding-temp/ — pre-created inside ramdisk so Emby always finds it there -# Without this Emby creates it at its own first-writable location -# which may be SSD even when symlink points at ramdisk +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Creates the tmpfs ramdisk, SSD fallback directory, transcode symlink, and +# pre-creates transcoding-temp on the ramdisk. Run once at array start via +# array_start.sh (unRAID_Essentials/). Idempotent — already-mounted ramdisk +# reports status and exits cleanly. Always resets the symlink to the ramdisk +# on boot, ensuring a clean state regardless of what state it was in before +# shutdown. # -# ── TRANSCODE_LINK SYMLINK ──────────────────────────────────────────────────────────────────── -# Emby's transcode path is set to TRANSCODE_LINK in Emby config. -# transcode_manager.sh flips the symlink between RAMDISK_PATH and TRANSCODE_SSD at runtime -# based on ramdisk usage — Emby sessions automatically follow without restart. -# This script always sets the link to RAMDISK_PATH at array start (clean state). +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── STATE FILE ──────────────────────────────────────────────────────────────────────────────── -# Initialises /tmp/transcode_state.db with current target and flip tracking counters. -# /tmp resets on reboot — correct, transcode state is ephemeral. +# Creates four things in order: +# 1. RAMDISK_PATH — tmpfs mount (size: HOST*_RAMDISK_SIZE ceiling, not a reservation) +# 2. TRANSCODE_SSD — SSD fallback directory and transcoding-temp inside it +# 3. TRANSCODE_LINK — symlink reset to RAMDISK_PATH (clean state at every boot) +# 4. transcoding-temp/ inside RAMDISK_PATH — pre-created before Emby starts # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID and aliases HOST*_RAMDISK_SIZE → RAMDISK_SIZE. -# HOST*_RAMDISK_SIZE, HOST*_RAMDISK_WARN_GB, HOST*_RAMDISK_LOW_GB must all be -# configured per host — different servers have different amounts of RAM available. +# The transcoding-temp pre-creation is critical: if it doesn't exist on the ramdisk +# when Emby starts, Emby searches all accessible paths for an existing one and finds +# the SSD fallback version — routing all sessions there until Emby restarts. # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# Root check — mount and symlink require root -# acquire_lock — prevents duplicate runs at array start -# detect_hosts() — correct RAMDISK_SIZE per host -# Already mounted — exits cleanly without remounting (idempotent) -# validate_unraid_cmd — notify validated before use -# Silent on success — startup script runs every boot — no noise when healthy +# Initialises /tmp/transcode_state.db with current target and flip counters. +# /tmp resets on reboot — correct, transcode state should not persist across boots. # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── -# HOST*_RAMDISK_SIZE — tmpfs size (e.g. 8G) — must change together with WARN_GB/LOW_GB -# HOST*_RAMDISK_WARN_GB — warn threshold in GB -# HOST*_RAMDISK_LOW_GB — flip to SSD threshold in GB -# HOST*_TRANSCODE_SSD — SSD fallback path -# HOST*_TRANSCODE_SERVERS — which servers run transcoding -# Aliased by detect_hosts() +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# TRANSCODE_LINK — symlink path Emby uses as transcode directory -# TRANSCODE_CHMOD — permissions applied to ramdisk and fallback -# TRANSCODE_OWNER — owner applied (default nobody:users) +# Root Required +# mount and symlink creation require root. +# +# Single Instance Lock +# acquire_lock prevents duplicate runs at array start. +# +# Idempotent Mount Check +# If RAMDISK_PATH is already a mountpoint, reports status and exits cleanly +# without attempting to remount or changing anything. +# +# Notification Validated +# validate_unraid_cmd confirms the notify script is present before use. +# +# Silent on Success +# Startup script runs on every boot — no output when healthy. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# +# HOST*_RAMDISK_SIZE +# tmpfs ceiling (e.g. 10G). Must change together with WARN_GB and LOW_GB. +# Aliased by detect_hosts() → RAMDISK_SIZE. +# +# HOST*_RAMDISK_WARN_GB +# Usage level at which transcode_manager.sh flips symlink to SSD. +# +# HOST*_RAMDISK_LOW_GB +# Usage level at which transcode_manager.sh flips back to ramdisk. +# +# HOST*_TRANSCODE_SSD +# SSD fallback directory path. +# Aliased by detect_hosts() → TRANSCODE_SSD. +# +# master.conf +# +# TRANSCODE_LINK +# Symlink path Emby uses as its transcode directory. Must match the path +# configured in Emby's transcoding settings. +# +# TRANSCODE_CHMOD / TRANSCODE_OWNER +# Permissions applied to both ramdisk and SSD directories. (default: 755 / nobody:users) +# +# ============================================================================================== +# STATE FILES +# ============================================================================================== +# +# /tmp/transcode_state.db — current symlink target + flip count tracking +# Lives in /tmp (ephemeral — resets on reboot correctly) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# ramdisk_setup.sh +# Normal setup run. Called by array_start.sh at boot. +# +# ramdisk_setup.sh --dry-run +# Show what would be created without creating anything. +# +# ramdisk_setup.sh --status +# Show current ramdisk mount state, symlink target, and SSD directory state. +# +# ramdisk_setup.sh --log +# Verbose output showing each creation step. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# ramdisk_setup.sh — normal setup (runs at array start) -# ramdisk_setup.sh --dry-run — preview without making changes -# ramdisk_setup.sh --status — show current ramdisk and symlink state -# ramdisk_setup.sh --log — verbose output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Transcodes/transcode_cleanup.sh b/Transcodes/transcode_cleanup.sh index 8a985e3..3747bd2 100644 --- a/Transcodes/transcode_cleanup.sh +++ b/Transcodes/transcode_cleanup.sh @@ -2,66 +2,98 @@ # ============================================================================================== # ================================= Transcode Cleanup ========================================== # ============================================================================================== -# Removes old inactive transcode files from both ramdisk and SSD fallback locations. -# Called every 5 minutes by transcode_manager.sh — must be fast and non-blocking. -# Never deletes files that are currently open by any process. # -# ── SAFETY RULES ────────────────────────────────────────────────────────────────────────────── +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Removes stale transcode files from both ramdisk and SSD fallback locations. +# Called by transcode_management.sh (Orchestrators/) before transcode_manager.sh — +# cleanup must run first so the manager sees real active-session usage, not +# inflated usage from stale files. Must be fast and non-blocking. +# # A file is eligible for deletion only if ALL conditions are true: -# 1. Older than TRANSCODE_MAX_AGE minutes (mtime — last modified time) +# 1. Older than TRANSCODE_MAX_AGE minutes (mtime — last write time) # 2. Not currently open by any process (checked via lsof pre-built map) # -# ── WHY NOT SESSION-AWARE CLEANUP ───────────────────────────────────────────────────────────── -# ffmpeg generates folder names independently of the media server API session IDs. -# There is no reliable correlation between API session IDs and transcoding-temp subfolder -# names — matching them would falsely treat active sessions as ended. -# lsof is the correct and reliable active file check — if ffmpeg has a file open, -# lsof sees it regardless of folder naming or session state. +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== # -# ── TRANSCODING-TEMP PROTECTION ─────────────────────────────────────────────────────────────── -# The transcoding-temp directory is excluded from deletion even when empty. -# If cleanup removes the empty transcoding-temp folder from the ramdisk, Emby finds -# the SSD version instead and all new sessions land on SSD until Emby restarts. -# ! -name "transcoding-temp" exclusion in find prevents this permanently. +# lsof Called Once, Not Per File +# On a busy Live TV system the ramdisk contains thousands of HLS segment files. +# Calling lsof once per file creates thousands of subprocess calls every 3 minutes. +# lsof is called once per location to build a complete open-file map. All subsequent +# checks are O(1) lookups against that map — thousands of files, one lsof call. # -# ── PERFORMANCE ─────────────────────────────────────────────────────────────────────────────── -# lsof is called ONCE per location — never once per file. -# Per-file lsof stalls on busy systems with live TV buffering hundreds of segments. +# No Session-Aware Cleanup +# ffmpeg generates folder names independently of the media server API session IDs. +# There is no reliable correlation between API session IDs and transcoding-temp +# subfolder names. Attempting to correlate them would falsely treat active sessions +# as ended. lsof is the correct check — if ffmpeg has a file open, it is active +# regardless of folder naming or session state. # -# Open file check uses in-memory associative array (OPEN_FILES_MAP): -# Was: echo "$OPEN_FILES" | grep -qF "$file" — O(n) per file → O(n²) total -# Now: [[ -n "${OPEN_FILES_MAP[$file]:-}" ]] — O(1) per file → O(n) total -# Same lesson as TRACKED_MAP in arr cleanup scripts. +# transcoding-temp Is Never Deleted +# If cleanup removes the empty transcoding-temp folder from the ramdisk, Emby +# searches all accessible paths for an existing one, finds the SSD fallback version, +# and routes all new sessions there until Emby restarts. The directory is excluded +# from find by name — protected even when completely empty. # -# ── POST-CLEANUP SYMLINK FLIP ───────────────────────────────────────────────────────────────── -# After cleanup, if ramdisk has recovered below RAMDISK_LOW_GB and symlink currently -# points at SSD → triggers transcode_manager.sh to flip back to ramdisk. +# Post-Cleanup Flip-Back +# After removing stale files, checks whether ramdisk usage dropped below +# RAMDISK_LOW_GB. If so — and symlink currently points at SSD — triggers a +# flip back to ramdisk. This is the recovery path; the manager handles +# the fill-up path. # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID and aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_LOW_GB. -# Each server cleans its own transcode locations at the correct thresholds. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock "wait" — wait if previous cleanup still running -# detect_hosts() — correct paths and thresholds per host -# lsof timeout — lsof call capped at 15 seconds per location -# OPEN_FILES_MAP — in-memory O(1) active file lookup -# transcoding-temp guard — never deletes this directory -# Silent by default — runs every 5 minutes, must not produce noise when healthy +# Wait Lock +# acquire_lock "wait" — waits if a previous cleanup run is still active rather +# than exiting. The caller's 3-minute interval can overlap on a slow system. +# +# lsof Timeout +# lsof call capped at 15 seconds per location — prevents blocking indefinitely +# on a system with many open files. +# +# transcoding-temp Guard +# `! -name "transcoding-temp"` in the find command — protected unconditionally. +# +# Silent by Default +# Runs every 3 minutes — must not produce noise when healthy. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── # HOST*_RAMDISK_PATH / HOST*_TRANSCODE_SSD / HOST*_RAMDISK_LOW_GB -# Aliased by detect_hosts() +# Aliased by detect_hosts() → RAMDISK_PATH / TRANSCODE_SSD / RAMDISK_LOW_GB. # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# TRANSCODE_MAX_AGE — minutes before an inactive transcode file is eligible -# TRANSCODE_ORPHAN_AGE — minutes for orphan detection (informational — future use) +# master.conf +# +# TRANSCODE_MAX_AGE +# Minutes before an inactive transcode file is eligible for deletion. (default: 20) +# +# TRANSCODE_ORPHAN_AGE +# Minutes for orphan folder detection. (default: 30) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# transcode_cleanup.sh +# Remove stale files from ramdisk and SSD. Check for flip-back opportunity. +# +# transcode_cleanup.sh --dry-run +# Show which files would be deleted. No deletions, no flip. +# +# transcode_cleanup.sh --status +# Show current file counts, ages, and open-file status per location. +# +# transcode_cleanup.sh --log +# Verbose per-file output including age, open status, and deletion result. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# transcode_cleanup.sh — normal cleanup run -# transcode_cleanup.sh --dry-run — show what would be deleted -# transcode_cleanup.sh --status — show current state -# transcode_cleanup.sh --log — verbose per-file output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Transcodes/transcode_manager.sh b/Transcodes/transcode_manager.sh index 5f638b5..5dad2f6 100644 --- a/Transcodes/transcode_manager.sh +++ b/Transcodes/transcode_manager.sh @@ -2,56 +2,125 @@ # ============================================================================================== # ================================= Transcode Manager ========================================== # ============================================================================================== -# Manages Emby transcode storage using filesystem symlink indirection. -# Called every 5 minutes by User Scripts — must be fast, non-blocking, and silent when healthy. # -# ── HOW IT WORKS ────────────────────────────────────────────────────────────────────────────── -# Emby's transcode path is set to TRANSCODE_LINK (a symlink). -# ffmpeg resolves the symlink ONCE at session start — existing sessions are never affected. -# Only NEW sessions care about where the symlink currently points. -# Flipping the symlink mid-stream is safe — in-progress transcodes continue uninterrupted. +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Monitors ramdisk usage and manages the transcode symlink direction. Called by +# transcode_management.sh (Orchestrators/) every 3 minutes — always after +# transcode_cleanup.sh runs first. Must be fast, non-blocking, and silent when +# nothing has changed. # -# ── THREE MODES ─────────────────────────────────────────────────────────────────────────────── +# Emby's transcode path points at TRANSCODE_LINK (a symlink). ffmpeg resolves +# the symlink once at session start and holds a direct reference — existing +# sessions are completely unaffected by symlink flips. Only new sessions care +# where the symlink currently points. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Three Modes (TRANSCODE_MANAGER_MODE) # smart — auto-flips between ramdisk and SSD based on usage thresholds (default) # ramdisk above RAMDISK_WARN_GB → flip to SSD # ramdisk below RAMDISK_LOW_GB → flip back to ramdisk # ramdisk — always uses ramdisk, warns if above threshold, never flips -# ssd — always uses SSD, never uses ramdisk +# ssd — always uses SSD, never uses ramdisk (use during drain or maintenance) # -# ── SAFETY CHECKS — EVERY RUN ───────────────────────────────────────────────────────────────── -# Symlink missing/broken → auto-recreate pointing at ramdisk, notify -# Ramdisk disappeared → auto-flip to SSD, notify warning -# SSD path missing → disable SSD fallback / error if mode=ssd -# transcoding-temp missing → recreate on ramdisk immediately -# Permissions drift → fix silently +# Safety Checks — Every Run Regardless of Mode +# Symlink missing/broken → recreate pointing at ramdisk, notify +# Ramdisk disappeared → flip to SSD immediately, notify warning +# SSD path missing → disable SSD fallback (error if mode=ssd) +# transcoding-temp missing → recreate on ramdisk silently +# Permissions drift → fix silently every run # Emby not running → skip threshold checks, verify symlink only # -# ── SESSION DISPLAY ─────────────────────────────────────────────────────────────────────────── -# Shows active Emby/Jellyfin/Plex streams with user, title, type, and play method. -# Split state shown when sessions exist on both ramdisk and SSD simultaneously — -# this happens naturally when symlink flips mid-session. +# Session Display +# Shows active streams from all configured TRANSCODE_SERVERS with user, title, +# type (Live TV / TV Show / Movie), and play method (Transcode / Direct). +# Split state shown when sessions exist on both ramdisk and SSD — normal during +# a flip while ramdisk sessions drain. # -# ── DAILY LOG ───────────────────────────────────────────────────────────────────────────────── -# Appends to TRANSCODE_DAILY_LOG after each run — read by weekly_health_digest.sh. -# Format: DATE|RAMDISK_USED_GB|FLIP_COUNT|RAM_SESSION_COUNT|SSD_SESSION_COUNT|FILES_CLEANED +# Daily Log +# Appends one entry per run to TRANSCODE_DAILY_LOG, read by weekly_health_digest.sh. +# Format: DATE|RAMDISK_USED_GB|FLIP_COUNT|RAM_SESSION_COUNT|SSD_SESSION_COUNT|FILES_CLEANED # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID and aliases HOST*_TRANSCODE_SERVERS, HOST*_RAMDISK_PATH, -# HOST*_TRANSCODE_SSD, HOST*_RAMDISK_WARN_GB, HOST*_RAMDISK_LOW_GB, HOST*_RAMDISK_SIZE. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock "wait" — wait if previous run still active -# detect_hosts() — correct paths and thresholds per host -# DOCKER_TIMEOUT — all docker calls protected against daemon hangs -# validate_unraid_cmd — notify validated before use -# Silent by default — runs every 5 minutes, only speaks when something changes +# Wait Lock +# acquire_lock "wait" — waits if the previous run is still active. The 3-minute +# interval can overlap on a system under heavy load. +# +# Docker Timeout +# DOCKER_TIMEOUT caps all docker calls against a hung daemon. +# +# Notification Validated +# validate_unraid_cmd confirms the notify script is present before use. +# +# Silent by Default +# Runs every 3 minutes — only speaks when something changes or needs attention. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# +# HOST*_RAMDISK_PATH / HOST*_TRANSCODE_SSD +# Ramdisk mount point and SSD fallback path. +# Aliased by detect_hosts(). +# +# HOST*_RAMDISK_WARN_GB / HOST*_RAMDISK_LOW_GB / HOST*_RAMDISK_SIZE +# Thresholds and ceiling. Change all three together. +# Aliased by detect_hosts(). +# +# HOST*_TRANSCODE_SERVERS +# Array of media server definitions: "ContainerName|URL|APIKey|Type" +# Type: emby | jellyfin | plex +# Aliased by detect_hosts() → TRANSCODE_SERVERS. +# +# master.conf +# +# TRANSCODE_MANAGER_MODE +# smart | ramdisk | ssd. (default: smart) +# +# TRANSCODE_CHECK_EMBY +# Skip threshold checks when Emby not running — prevents unnecessary flips +# overnight when no sessions are active. (default: true) +# +# TRANSCODE_FLIP_WARN +# Notify if symlink flips this many times in one hour — indicates ramdisk +# is undersized for the load. (default: 3) +# +# ============================================================================================== +# STATE FILES +# ============================================================================================== +# +# /tmp/transcode_state.db — current symlink target, flip count, last flip time +# Lives in /tmp (ephemeral — resets correctly on reboot) +# TRANSCODE_DAILY_LOG — per-run append, read by weekly_health_digest.sh +# Trimmed to TRANSCODE_LOG_RETENTION days on each write +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# transcode_manager.sh +# Check usage, flip if needed, run safety checks, display active sessions. +# +# transcode_manager.sh --dry-run +# Show current usage and what flip decision would be made. No changes. +# +# transcode_manager.sh --status +# Show current symlink target, ramdisk usage, session counts, and flip history. +# +# transcode_manager.sh --log +# Verbose output including per-check results and session detail. +# +# transcode_manager.sh --no-log +# Suppress daily log write. Used internally when called by transcode_cleanup.sh. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# transcode_manager.sh — normal run -# transcode_manager.sh --dry-run — preview without making changes -# transcode_manager.sh --status — show current state and exit -# transcode_manager.sh --log — verbose output -# transcode_manager.sh --no-log — suppress daily log write (called by cleanup) # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/common.sh b/common.sh index 1dc1eab..faa056e 100644 --- a/common.sh +++ b/common.sh @@ -572,6 +572,10 @@ detect_hosts() { _alias_array "PARTNERSHIP_AUTH_WEBUIS" _alias_array "PARTNERSHIP_MIRROR_BACKUPS" _alias_array "PARTNERSHIP_OWN_CONTAINERS" + _alias_array "PARTNERSHIP_AUTH_STACK" + _alias_array "PARTNERSHIP_REPLACE_CONTAINERS" + _alias_array "PARTNERSHIP_ARR_STACK" + _alias_array "PARTNERSHIP_ARR_REPLACE_CONTAINERS" _alias_array "RW_PAUSE_CONTAINERS" _alias_array "RW_STOP_CONTAINERS" diff --git a/master_host1.conf b/master_host1.conf index 81ab920..f911926 100644 --- a/master_host1.conf +++ b/master_host1.conf @@ -99,6 +99,32 @@ "Authelia-Secondary|9092" ) +# XML templates (from this server's templates-user/) pushed to mirror during onboard. +# These become the mirror's active auth stack, backed by the rsync-synced appdata. +# Update filename if Lldap is renamed to drop the host suffix. + HOST1_PARTNERSHIP_AUTH_STACK=( + # Dependencies first — Mariadb/Redis must be healthy before Authelia starts + "my-Mariadb-Authelia.xml" + "my-Mariadb-Authelia-Secondary.xml" + "my-Redis-Authelia.xml" + "my-Redis-Authelia-Secondary.xml" + # Auth apps — deployed after their deps are confirmed healthy + "my-Authelia.xml" + "my-Authelia-Secondary.xml" + "my-NginxProxyManager.xml" + "my-Lldap-Gmer4Lfe.xml" + ) + +# XML templates pushed to mirror for the arr stack during onboard. +# Deps (e.g. databases) first if any — same ordering rule as auth stack. + HOST1_PARTNERSHIP_ARR_STACK=( + # "my-Sonarr.xml" + # "my-Radarr.xml" + # "my-Lidarr.xml" + # "my-Prowlarr.xml" + # "my-Bazarr.xml" + ) + # Paths HOST2 should collect during the grace window after offboard. # Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS. HOST1_PARTNERSHIP_MIRROR_BACKUPS=( diff --git a/master_host2.conf b/master_host2.conf index 6203435..620381d 100644 --- a/master_host2.conf +++ b/master_host2.conf @@ -99,6 +99,28 @@ # "NginxProxyManager|81" ) +# Containers to stop on this server before the owner deploys the auth stack during onboard. +# List whatever auth/proxy containers are currently running here. + HOST2_PARTNERSHIP_REPLACE_CONTAINERS=( + "NginxProxyManager" + "Authelia" + "Authelia-Secondary" + "Mariadb-Authelia" + "Mariadb-Authelia-Secondary" + "Redis-Authelia" + "Redis-Authelia-Secondary" + "Lldap" + ) + +# Arr containers to stop on this server before the owner deploys the arr stack during onboard. + HOST2_PARTNERSHIP_ARR_REPLACE_CONTAINERS=( + # "Sonarr" + # "Radarr" + # "Lidarr" + # "Prowlarr" + # "Bazarr" + ) + # Paths HOST1 should collect during the grace window after offboard. # Notified on offboard — no auto-deletion, HOST1 must collect manually within PARTNERSHIP_GRACE_HOURS. HOST2_PARTNERSHIP_MIRROR_BACKUPS=( diff --git a/unRAID_Essentials/Manual-Unraid_Essentials.md b/unRAID_Essentials/Manual-Unraid_Essentials.md new file mode 100644 index 0000000..03a0dc3 --- /dev/null +++ b/unRAID_Essentials/Manual-Unraid_Essentials.md @@ -0,0 +1,768 @@ +# ━━━━━ UNRAID ESSENTIALS — Manual ━━━━━ + +Configuration reference, operational procedures, and troubleshooting for +system-level scripts. Read the ARRAY_START_SCRIPTS order section before +adding or reordering scripts at array start. + +--- + +## ━━━ CONTENTS ━━━ + +- [ARRAY_START_SCRIPTS Order](#array_start_scripts-order) +- [system_watchdog.sh](#system_watchdogsh) +- [resource_watchdog.sh](#resource_watchdogsh) +- [webgui_restart.sh](#webgui_restartsh) +- [inotify_tuning.sh](#inotify_tuningsh) +- [php_fpm_max_children.sh](#php_fpm_max_childrensh) +- [docker_syslog_filter.sh](#docker_syslog_filtersh) +- [clear_logs.sh](#clear_logssh) +- [mover_stop.sh](#mover_stopsh) +- [rsync_stop.sh](#rsync_stopsh) +- [user_scripts_stop.sh](#user_scripts_stopsh) +- [server_reboot.sh](#server_rebootsh) +- [Full Configuration Reference](#full-configuration-reference) +- [Troubleshooting](#troubleshooting) + +--- + +## ARRAY_START_SCRIPTS Order + +> **The order of scripts in ARRAY_START_SCRIPTS matters for three of these +> scripts.** Getting it wrong causes subtle failures that don't show up +> immediately. + +```bash +# master.conf +ARRAY_START_SCRIPTS=( + "inotify_tuning.sh" # 1 — FIRST: kernel limits must be set before + # any container starts. Containers inherit + # inotify limits at launch, not dynamically. + "docker_syslog_filter.sh" # 2 — SECOND: before any veth interfaces are + # created. If a container starts first, its + # veth creation is already in syslog. + "php_fpm_max_children.sh" # 3 — before WebGUI is under load + "ramdisk_setup.sh" # (from Transcodes/) before Emby starts + ... + "system_watchdog.sh" # LAST or near-last — starts background loop +) +``` + +Why inotify FIRST: If Code-Server starts before limits are raised, it inherits +the old low limits. The limits are kernel-wide — a restart of Code-Server picks +up the new values, but it's a manual step. Avoid by running inotify_tuning.sh first. + +Why docker_syslog_filter SECOND: The filter must be in place before any container +starts creating veth interfaces. The first container start after array start +generates veth messages — these will appear in syslog if the filter isn't active. + +--- + +## system_watchdog.sh + +### Three-Tier Response + +The watchdog categorizes every failure into one of three tiers: + +**Tier 1 — CRITICAL (immediate reboot, no strikes)** +| Condition | Threshold | Why immediate | +|-----------|-----------|---------------| +| Docker daemon unresponsive | N/A | Nothing can be healed; every docker command hangs | +| rootfs usage | SYS_WATCHDOG_ROOTFS_CRITICAL_PCT (99%) | SSH stops; state files fail silently | +| Kernel oops/BUG in dmesg | delta > 0 | Kernel running with corrupted state | +| File descriptor exhaustion | SYS_WATCHDOG_FD_CRITICAL_PCT (95%) | New connections silently failing | +| /boot read-only unexpectedly | write test fails | Config writes silently failing | + +**Tier 2 — URGENT (bypass strikes with OOM confirmation)** + +RAM below MEM_GB AND OOM kills this cycle >= SYS_WATCHDOG_OOM_LIMIT. +Both conditions required — RAM alone without OOM uses the standard strike system. +OOM confirms the system is dying faster than watchdogs can heal. + +**Tier 3 — STANDARD (SYS_WATCHDOG_STRIKES consecutive failures → reboot)** +| Check | Threshold | +|-------|-----------| +| Free RAM | MEM_WARN_GB → MEM_SHUTDOWN_GB → MEM_GB | +| Load average | SYS_WATCHDOG_LOAD_MULTIPLIER × cpu_count | +| CPU temperature | SYS_WATCHDOG_CPU_TEMP | +| Zombie processes | SYS_WATCHDOG_ZOMBIES | +| /var/log usage | SYS_WATCHDOG_VAR_LOG_PCT | +| /tmp usage | SYS_WATCHDOG_TMP_PCT | +| Array disk errors | mdstat error delta > 0 | +| NIC state | interface operstate != "up" | +| Required containers | containers in SYS_WATCHDOG_REQUIRED_CONTAINERS | + +### RAM Tiers + +``` +MEM_WARN_GB (10GB) → warn + notify, no action +MEM_SHUTDOWN_GB (6GB) → stop non-essential containers, wait for recovery +MEM_GB (4GB) → strike → reboot (URGENT bypass with OOM) +MEM_RECOVER_GB (30GB) → RAM must reach this before stopped containers restart +``` + +At MEM_SHUTDOWN_GB, all containers NOT listed in +`SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED` are stopped. Excluded containers by default: +NginxProxyManager, Authelia, Mariadb, Redis, Emby, Dispatcharr. Adjust in +master.conf for your critical services. + +### Abort Conditions + +These conditions prevent a reboot — running them would cause data loss: + +```bash +SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # ZFS pool degraded/faulted +SYS_WATCHDOG_ABORT_ON_PARITY=true # Parity check/rebuild running +SYS_WATCHDOG_ABORT_ON_MOVER=true # Mover running +``` + +CRITICAL tier bypasses all abort conditions — an imminent crash outweighs +data safety concerns. + +### Strike System + +The strike count tracks consecutive failures. A single recovery resets strikes +to 0. The reboot only fires after SYS_WATCHDOG_STRIKES consecutive failures on +the same check — transient spikes (a brief load burst, a momentary RAM dip) don't +trigger reboots. + +### Reboot Rate Limit + +```bash +SYS_WATCHDOG_REBOOT_WINDOW_HRS=2 # window in hours +SYS_WATCHDOG_MAX_REBOOTS=3 # max reboots within the window +``` + +If the server has rebooted SYS_WATCHDOG_MAX_REBOOTS times within the window, +system_watchdog stops rebooting and notifies instead. This prevents a boot loop +where the watchdog reboots → something crashes again immediately → reboot again. + +### Usage + +```bash +system_watchdog.sh # start continuous monitoring loop +system_watchdog.sh --dry-run # run detection logic without rebooting +system_watchdog.sh --status # show thresholds, current state, strike counts +system_watchdog.sh --log # verbose per-cycle output +``` + +### Verify Running + +```bash +pgrep -a -f system_watchdog.sh +# Expected: shows PID and path +``` + +--- + +## resource_watchdog.sh + +### Pressure Levels + +```bash +# master.conf +RW_RAM_SOFT_GB=20 # Level 1 trigger — throttle downloaders +RW_RAM_MEDIUM_GB=15 # Level 2 trigger — throttle + pause containers +RW_RAM_HARD_GB=10 # Level 3 trigger — stop containers +RW_RAM_RECOVER_GB=25 # recover to this before un-stopping at level 3 + +RW_LOAD_SOFT_MULTIPLIER=2.0 # load > 2× cpu count = level 1 +RW_LOAD_MEDIUM_MULTIPLIER=3.0 # load > 3× cpu count = level 2 + +RW_RECOVER_CYCLES=3 # consecutive under-threshold runs before de-escalating +``` + +### Per-Host Container Lists + +```bash +# master_host1.conf +HOST1_RW_PAUSE_CONTAINERS=("Tdarr" "HandBrake") # paused at medium pressure +HOST1_RW_STOP_CONTAINERS=("LocalAI" "Satisfactory") # stopped at hard pressure +``` + +Containers in `RW_CRITICAL_CONTAINERS` are never paused or stopped regardless of +pressure level. Default includes: Emby, NginxProxyManager, Authelia, Mariadb, Redis. + +### docker_watchdog Coordination + +At level 3, resource_watchdog writes `mem_shutdown_active=true` to `RW_STATE_FILE`. +docker_watchdog.sh reads this flag each cycle and skips all container restart logic +while it is set. Without this coordination, docker_watchdog would immediately +restart containers that resource_watchdog just stopped to free RAM. + +The flag is cleared when level 3 pressure resolves and containers are restarted. + +### Usage + +```bash +resource_watchdog.sh # single pass (called by watchdog_orchestrator.sh) +resource_watchdog.sh --dry-run # show what would be throttled/paused/stopped +resource_watchdog.sh --status # current level, active actions, recovery cycle count +resource_watchdog.sh --log # verbose per-check output +``` + +--- + +## webgui_restart.sh + +### Escalation Logic + +``` +curl $WEBGUI_URL → 200 OK → exit 0 (silent) + +Not responding: + 1. /etc/rc.d/rc.nginx restart + wait WEBGUI_NGINX_WAIT (15s) → recheck + → recovered: notify, exit 0 + + 2. /etc/rc.d/rc.php-fpm restart + wait WEBGUI_PHP_WAIT (10s) → recheck + → recovered: notify, exit 0 + + 3. /usr/local/sbin/emhttp stop && start + wait WEBGUI_EMHTTP_WAIT (30s) → recheck + → recovered: notify, exit 0 + + All three failed → notify warning, exit 1 +``` + +### Configuration + +```bash +WEBGUI_URL="http://localhost" # URL to check +WEBGUI_TIMEOUT=5 # curl timeout in seconds +WEBGUI_NGINX_WAIT=15 # seconds after nginx restart before recheck +WEBGUI_PHP_WAIT=10 # seconds after php-fpm restart before recheck +WEBGUI_EMHTTP_WAIT=30 # seconds after emhttp restart before recheck +``` + +### WebGUI Frozen — Manual Recovery + +```bash +# Check which services are running: +webgui_restart.sh --status + +# Try manual restart sequence (same as the script): +/etc/rc.d/rc.nginx restart +# wait 15s, then: +curl -sf --max-time 5 http://localhost >/dev/null && echo "OK" || echo "still down" + +# If nginx didn't fix it, php-fpm: +/etc/rc.d/rc.php-fpm restart + +# If still down, emhttp: +/usr/local/sbin/emhttp stop && /usr/local/sbin/emhttp start + +# If all three failed: +server_reboot.sh --status # check for active sessions first +``` + +--- + +## inotify_tuning.sh + +### What It Sets + +```bash +INOTIFY_MAX_INSTANCES=1024 # max inotify fd objects per user (default: 128) +INOTIFY_MAX_WATCHES=1048576 # max watches shared across all users (default: 8192) +INOTIFY_MAX_QUEUED_EVENTS=32768 # max buffered events (default: 16384) +``` + +Verify current values: + +```bash +inotify_tuning.sh --status +# Shows current vs target for each limit, active instance count, top consumers +``` + +### If Code-Server Shows "Unable to Watch for File Changes" + +```bash +# 1. Verify limits are set: +sysctl fs.inotify.max_user_watches +# Expected: 1048576 + +# 2. Check total usage across all containers: +find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l + +# 3. If limits are set but Code-Server still shows the error: +docker restart Code-Server +# Running containers inherit limits at launch. Restart picks up the new values. + +# 4. If limits are NOT set (inotify_tuning.sh hasn't run yet): +inotify_tuning.sh --log +``` + +--- + +## php_fpm_max_children.sh + +### What It Sets + +```bash +PHP_MAX_CHILDREN=250 # target pm.max_children (default: 4-8 on unRAID) +PHP_CONF="/etc/php83/php-fpm.d/www.conf" +``` + +250 workers × ~2MB per worker = ~500MB total. On 128GB this is trivially small. +The default of 4–8 saturates immediately under load on a busy server. + +### Verify + +```bash +php_fpm_max_children.sh --status +# Shows current value vs target, PHP-FPM worker count + +# Manual verify: +grep "^pm.max_children" /etc/php83/php-fpm.d/www.conf +# Expected: pm.max_children = 250 +``` + +### If WebGUI Is Slow Despite the Setting + +```bash +# Check PHP-FPM worker utilization (requires system_tuning_monitor.sh in Monitors/): +# Look at the webgui_restart.sh escalation — step 2 (php-fpm restart) is specifically +# for worker exhaustion. If webgui_restart.sh is regularly hitting step 2, the +# pm.max_children value may still be too low, or there's a PHP worker leak. + +# Check running worker count: +pgrep -fc php-fpm +# Compare to pm.max_children — if equal, workers are saturated + +# Increase if needed: +# master.conf: PHP_MAX_CHILDREN=350 +# Then: php_fpm_max_children.sh --log (will update and restart php-fpm) +``` + +--- + +## docker_syslog_filter.sh + +### What It Creates + +``` +/etc/rsyslog.d/ignore-docker-veth.conf: + if ($msg contains "veth" or $msg contains "docker0") then { + stop + } +``` + +This drops any syslog message containing "veth" or "docker0" before it reaches +any output target, including the log file. The drop rule is applied at rsyslog +level — not at the log viewer level. + +### Verify + +```bash +docker_syslog_filter.sh --status +# Shows filter file content and rsyslog process state + +# Manual verify: +cat /etc/rsyslog.d/ignore-docker-veth.conf +pgrep -x rsyslogd && echo "rsyslog running" || echo "rsyslog NOT running" + +# Test the filter is active (should produce no syslog output): +logger "test veth message" +grep "test veth" /var/log/syslog 2>/dev/null || echo "filtered correctly" +``` + +### If Syslog Still Has Veth Noise + +```bash +# 1. Verify filter file exists with correct content: +docker_syslog_filter.sh --status + +# 2. If content differs — re-apply: +docker_syslog_filter.sh --log + +# 3. Verify rsyslog is using the conf.d directory: +grep -r "IncludeConfig" /etc/rsyslog.conf +# Expected: IncludeConfig /etc/rsyslog.d/*.conf (or similar) +``` + +--- + +## clear_logs.sh + +### Thresholds + +```bash +LOG_MIN_SIZE_MB=10 # skip system log if under this — keep recent history +LOG_DOCKER_MAX_MB=100 # clear Docker container log only if over this + +LOG_FILES=( + "/var/log/syslog" + "/var/log/messages" + "/var/log/dmesg" +) +``` + +### Why Truncation Not Deletion + +unRAID writes logs to tmpfs (`/var/log`). Truncation (`: > file`) keeps the file +descriptor open and valid while emptying content — syslogd continues writing to +the same fd without interruption. Deleting the file would orphan the file +descriptor and syslog would stop writing until restarted. + +### Identifying Large Docker Logs + +```bash +clear_logs.sh --status +# Shows top 10 Docker logs by size, current size vs threshold + +# Find the biggest log manually: +du -sh /var/lib/docker/containers/*/*.log 2>/dev/null | sort -rh | head -5 +``` + +--- + +## mover_stop.sh + +### Stop Sequence + +``` +1. Check if mover is running (pgrep "emhttp.*Mover") → exit cleanly if not +2. Wall message to all logged-in terminal users +3. Wait MOVER_STOP_TIMEOUT seconds (default: 30) +4. SIGTERM — mover finishes its current file operation, then stops +5. Wait 5 seconds → verify stopped +6. SIGKILL if still running — forced stop, partial files possible +7. Final verify — error if still running after SIGKILL +``` + +SIGTERM first because the mover can finish the file it is currently moving, +leaving no partial copies split across cache and array. SIGKILL is a last resort. + +### Configuration + +```bash +MOVER_STOP_TIMEOUT=30 # seconds between wall warning and SIGTERM +``` + +### Usage + +```bash +mover_stop.sh # check and stop if running +mover_stop.sh --status # show current mover state and PID +mover_stop.sh --dry-run # show what would happen without stopping +``` + +--- + +## rsync_stop.sh + +### Auto-Detection Logic + +rsync_stop.sh detects whether an orchestrator script (daily/weekly/critical sync) +is the parent of the running rsync process by scanning lock files in `$LOCK_DIR`. + +**Default behavior (orchestrator detected):** +Kills only the rsync subprocess. The orchestrator sees rsync died, moves to the +next share or exits cleanly. The orchestrator is NOT killed — it can still clean up. + +**Default behavior (no orchestrator):** +Kills rsync directly (standalone rsync.sh run). + +**--full-stop:** +Kills the orchestrator first, then kills rsync. Nothing continues after this. +Use when everything needs to stop immediately. + +### Container Recovery + +After killing rsync, the script checks all containers in `PROFILE_CRITICAL_CONTAINER_NAMES` +for any that were stopped by the interrupted rsync session and restarts them. +Remote containers are left for docker_watchdog.sh to recover. + +Skip container recovery with `--rsync-only` — used when called by other scripts +that handle recovery themselves. + +### Usage + +```bash +rsync_stop.sh # smart stop (auto-detect orchestrator) +rsync_stop.sh --full-stop # kill orchestrator + rsync +rsync_stop.sh --rsync-only # kill rsync, skip container recovery +rsync_stop.sh --status # show local and remote rsync state +rsync_stop.sh --dry-run # preview without changes +rsync_stop.sh --full-stop --dry-run # preview full stop +``` + +--- + +## user_scripts_stop.sh + +### Process Identification + +Scans `/proc/*/cmdline` for any process whose command line contains +`/tmp/user.scripts`. The unRAID User Scripts plugin stages all scripts in +`/tmp/user.scripts/` before execution — this signature is reliable regardless of +what the script is named or how it was launched. + +Script names are extracted from the path for display: you see which scripts are +being stopped, not just PIDs. + +### Self-Exclusion + +If this script is run via the User Scripts plugin, it would find its own PID in +the scan. It excludes both `$$` (its own PID) and `$PPID` (its parent process) +from the kill list. + +### Usage + +```bash +user_scripts_stop.sh # stop all User Script processes +user_scripts_stop.sh --status # show running scripts with names and elapsed time +user_scripts_stop.sh --dry-run # show what would be stopped +``` + +--- + +## server_reboot.sh + +### Full Shutdown Sequence + +``` +1. Pre-flight checks (warn only — do not block): + - rsync running? → warn, suggest rsync_stop.sh first + - mover running? → warn, suggest mover_stop.sh first + - Emby sessions? → warn (active streams will be interrupted) + +2. Wall message to all logged-in terminal users + +3. unRAID dashboard notification + +4. Wait REBOOT_SLEEP seconds (default: 30) + +5. Graceful VM shutdown: + virsh shutdown (ACPI signal — clean shutdown) + Wait REBOOT_VM_WAIT seconds (default: 30) for VMs to respond + +6. /etc/rc.d/rc.libvirt stop (VM Manager) + +7. /etc/rc.d/rc.docker stop (all containers stop) + +8. sync (flush filesystem buffers to disk) + +9. /sbin/reboot +``` + +### Configuration + +```bash +REBOOT_SLEEP=30 # seconds between warning and shutdown sequence +REBOOT_VM_WAIT=30 # seconds to wait for VMs to shut down gracefully +``` + +### Recommended Pre-Reboot Sequence + +For a clean reboot when services are active: + +```bash +rsync_stop.sh # stop any active rsync (smart mode) +mover_stop.sh # stop mover gracefully +server_reboot.sh --status # check what's still running +server_reboot.sh --reason="planned maintenance" +``` + +### Usage + +```bash +server_reboot.sh # reboot with 30s warning +server_reboot.sh --dry-run # walk through without rebooting +server_reboot.sh --status # show what would be affected +server_reboot.sh --reason="disk work" # include reason in notification +``` + +--- + +## Full Configuration Reference + +```bash +# master.conf + +# ── System Watchdog ──────────────────────────────────────────────────────────── +SYS_WATCHDOG_INTERVAL=300 # seconds between check cycles +SYS_WATCHDOG_STRIKES=3 # consecutive failures before reboot +SYS_WATCHDOG_REBOOT_WINDOW_HRS=2 # rate limit window +SYS_WATCHDOG_MAX_REBOOTS=3 # max reboots in window +SYS_WATCHDOG_OOM_LIMIT=3 # OOM kills/cycle for URGENT bypass + +MEM_WARN_GB=10 # warn + notify +MEM_SHUTDOWN_GB=6 # stop non-essential containers +MEM_GB=4 # strike → reboot +MEM_RECOVER_GB=30 # recovery threshold + +SYS_WATCHDOG_ROOTFS_CRITICAL_PCT=99 # Tier 1 trigger +SYS_WATCHDOG_FD_CRITICAL_PCT=95 # Tier 1 trigger +SYS_WATCHDOG_LOAD_MULTIPLIER=4 # Tier 3 — ×cpu_count +SYS_WATCHDOG_CPU_TEMP=85 # Tier 3 — Celsius +SYS_WATCHDOG_ZOMBIES=20 # Tier 3 — process count +SYS_WATCHDOG_VAR_LOG_PCT=80 # Tier 3 — percent full +SYS_WATCHDOG_TMP_PCT=85 # Tier 3 — percent full + +SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true +SYS_WATCHDOG_ABORT_ON_PARITY=true +SYS_WATCHDOG_ABORT_ON_MOVER=true + +SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED=( + "NginxProxyManager" "Authelia" "Mariadb" "Redis" "Emby" "Dispatcharr" +) +SYS_WATCHDOG_REQUIRED_CONTAINERS=() # containers that must be running + +# State files: +SYS_WATCHDOG_STATE_FILE="/tmp/sys_watchdog_state.db" +SYS_WATCHDOG_REBOOT_LOG="/tmp/sys_watchdog_reboots.db" +SYS_WATCHDOG_FAILED_FILE="/tmp/sys_watchdog_failed.db" +SYS_WATCHDOG_OOM_FILE="/tmp/sys_watchdog_oom.db" + +# ── Resource Watchdog ────────────────────────────────────────────────────────── +RW_ENABLED=true +RW_STATE_FILE="/tmp/resource_watchdog_state.db" + +RW_RAM_SOFT_GB=20 +RW_RAM_MEDIUM_GB=15 +RW_RAM_HARD_GB=10 +RW_RAM_RECOVER_GB=25 + +RW_LOAD_SOFT_MULTIPLIER=2.0 +RW_LOAD_MEDIUM_MULTIPLIER=3.0 +RW_RECOVER_CYCLES=3 + +RW_SABNZBD_ENABLED=true +RW_SABNZBD_SPEED_SOFT="50M" +RW_SABNZBD_SPEED_MEDIUM="10M" +RW_QBIT_ENABLED=true +RW_QBIT_DL_SOFT=51200 # KB/s +RW_QBIT_DL_MEDIUM=10240 + +RW_CRITICAL_CONTAINERS=("Emby" "NginxProxyManager" "Authelia" "Mariadb" "Redis") + +# ── Per-Host (master_host*.conf) ─────────────────────────────────────────────── +HOST1_RW_PAUSE_CONTAINERS=("Tdarr" "HandBrake") +HOST1_RW_STOP_CONTAINERS=("LocalAI" "Satisfactory") +HOST1_SABNZBD_URL="http://localhost:8080" +HOST1_SABNZBD_API_KEY="your-api-key" +HOST1_QBIT_URL="http://localhost:8090" +HOST1_QBIT_USERNAME="admin" +HOST1_QBIT_PASSWORD="your-password" + +# ── WebGUI Watchdog ──────────────────────────────────────────────────────────── +WEBGUI_URL="http://localhost" +WEBGUI_TIMEOUT=5 +WEBGUI_NGINX_WAIT=15 +WEBGUI_PHP_WAIT=10 +WEBGUI_EMHTTP_WAIT=30 + +# ── inotify Tuning ───────────────────────────────────────────────────────────── +INOTIFY_MAX_INSTANCES=1024 +INOTIFY_MAX_WATCHES=1048576 +INOTIFY_MAX_QUEUED_EVENTS=32768 + +# ── PHP-FPM ──────────────────────────────────────────────────────────────────── +PHP_MAX_CHILDREN=250 +PHP_CONF="/etc/php83/php-fpm.d/www.conf" + +# ── Syslog Filter ────────────────────────────────────────────────────────────── +FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" + +# ── Log Cleaner ──────────────────────────────────────────────────────────────── +LOG_FILES=("/var/log/syslog" "/var/log/messages" "/var/log/dmesg") +LOG_MIN_SIZE_MB=10 +LOG_DOCKER_MAX_MB=100 + +# ── Mover Stop ───────────────────────────────────────────────────────────────── +MOVER_STOP_TIMEOUT=30 + +# ── Server Reboot ────────────────────────────────────────────────────────────── +REBOOT_SLEEP=30 +REBOOT_VM_WAIT=30 +``` + +--- + +## Troubleshooting + +### system_watchdog.sh Not Starting + +```bash +# Check if already running (acquire_lock prevents second instance): +pgrep -a -f system_watchdog.sh + +# Check state file permissions: +ls -la /tmp/sys_watchdog_*.db + +# Run with --log to see startup: +system_watchdog.sh --log +``` + +### system_watchdog Rebooted Unexpectedly + +```bash +# Check reboot log: +cat /tmp/sys_watchdog_reboots.db +# Shows timestamp and reason for each watchdog-triggered reboot + +# Check what condition triggered it: +# Look in /var/log/syslog for "system_watchdog" near the reboot time +grep "system_watchdog" /var/log/syslog | tail -20 +``` + +### resource_watchdog Paused Containers It Shouldn't Have + +```bash +# Check current state: +resource_watchdog.sh --status + +# Add the container to RW_CRITICAL_CONTAINERS in master.conf: +RW_CRITICAL_CONTAINERS=("Emby" "NginxProxyManager" "Authelia" "Mariadb" "Redis" "MyContainer") + +# Un-pause manually if needed: +docker unpause MyContainer +``` + +### rsync_stop Killed the Wrong Thing + +If --full-stop killed an orchestrator you didn't intend to kill: + +```bash +# Next time use default mode (no --full-stop) to kill only rsync subprocess. +# To verify what would be killed before running: +rsync_stop.sh --status # shows running rsync and detected orchestrators +rsync_stop.sh --dry-run # shows smart mode decision +rsync_stop.sh --full-stop --dry-run # shows full-stop decision +``` + +### WebGUI Recovery After All Three Steps Failed + +```bash +# Check if processes are running: +pgrep -x nginx && echo "nginx: yes" || echo "nginx: no" +pgrep emhttpd && echo "emhttp: yes" || echo "emhttp: no" +pgrep -f php-fpm && echo "php-fpm: yes" || echo "php-fpm: no" + +# Check recent nginx errors: +cat /var/log/nginx/error.log | tail -20 + +# Check emhttp log: +tail -20 /var/log/syslog | grep emhttp + +# Last resort — reboot: +server_reboot.sh --reason="WebGUI unrecoverable" +``` + +### PHP-FPM Config Not Found After unRAID Update + +unRAID updates occasionally change the PHP version. If `php_fpm_max_children.sh` +errors with "config file not found": + +```bash +# Find the new config path: +find /etc -name "www.conf" 2>/dev/null + +# Update PHP_CONF in master.conf: +PHP_CONF="/etc/php84/php-fpm.d/www.conf" # example for php84 + +# Verify with: +php_fpm_max_children.sh --status +``` diff --git a/unRAID_Essentials/README-Unraid_Essentials.md b/unRAID_Essentials/README-Unraid_Essentials.md index 7e94d1e..f74dfbe 100644 --- a/unRAID_Essentials/README-Unraid_Essentials.md +++ b/unRAID_Essentials/README-Unraid_Essentials.md @@ -1,807 +1,176 @@ -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# 🖥️ UNRAID ESSENTIALS -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# ━━━━━ UNRAID ESSENTIALS ━━━━━ -**System-level scripts that act on the unRAID server itself — not containers, not -media, not monitoring.** Keeping the server stable under load, recovering a frozen -WebGUI, tuning kernel limits, suppressing log noise, and handling graceful shutdowns -and reboots with proper warning sequences. +**System-level scripts that act on the unRAID server itself — not containers, +not media, not monitoring.** Keeping the server stable under load, recovering a +frozen WebGUI, tuning kernel limits, suppressing log noise, and handling graceful +shutdowns with proper warning sequences. + +--- + +## ━━━ THE PROBLEM THAT BUILT THIS ━━━ + +**Server Getting Into Unstable States With No Recovery Path** +A container has a memory leak. RAM drops to 2GB. The system starts swapping. Docker +watchdog tries to restart the container — but Docker itself is barely responding. +The restart hangs. The server needs a reboot, but nothing in the ecosystem is +authorized to call one. Or: rootfs fills to 99%. SSH stops working. Docker can't +write log files. The server is functionally dead but still technically running. + +Fix: `system_watchdog.sh` — three-tier response: immediate reboot on critical +failures, OOM-confirmed bypass for RAM crises, strike system for sustained +threshold breaches. Last line of defense before a hard crash. + +**WebGUI Freezing and Nobody Noticing** +The WebGUI becomes unresponsive. Nginx gets into a bad state, or PHP-FPM workers +are saturated, or emhttp has frozen. From a user perspective: dashboard doesn't +load, settings don't save, containers can't be started or stopped via the UI. No +container-level alert fires because this isn't a container problem — it's a web +server problem. By the time someone notices it may have been broken for hours. + +Fix: `webgui_restart.sh` — checks every 10 minutes, escalates through nginx → +php-fpm → emhttp. Lightest fix first. Silent when healthy. + +**50+ Containers Starting and Filling Syslog With Veth Noise** +Array starts. 50+ containers come up simultaneously. Docker creates a virtual +network interface for each one. Each interface generates multiple syslog entries. +In the first minute after array start, syslog is buried under 200–400 lines of +`veth renamed from eth0` and `docker0: port entered forwarding state`. Real events +— a failed mount, a permission error, a service that didn't start — are invisible. + +Fix: `docker_syslog_filter.sh` — creates an rsyslog drop rule before any container +starts. Applied at array start. Idempotent — silent when already correct. + +**WebGUI Queuing Requests Under Load Without Explanation** +The WebGUI feels slow. Clicking a button takes 5 seconds. Nothing in the logs +explains it. The cause: PHP-FPM's `pm.max_children` defaults to 4–8 workers. With +multiple users, active plugins, and 50+ containers potentially hitting the WebGUI, +those workers saturate immediately. New requests queue behind active ones. + +Fix: `php_fpm_max_children.sh` — sets `pm.max_children=250` at array start. +250 workers × ~2MB = ~500MB total. On 128GB this is trivially small. + +**inotify Exhaustion Producing Unexplained Failures** +When inotify limits are exhausted, containers silently stop receiving filesystem +events. Arrs don't detect completed downloads. VSCode shows "unable to watch for +file changes." Code-Server with node_modules alone can consume 100K–200K watches, +and all containers share the same pool. + +Fix: `inotify_tuning.sh` — raises all three inotify limits at array start. Must +run FIRST in ARRAY_START_SCRIPTS before any containers start. + +**Mover Getting Killed Mid-Transfer Leaving Files Inconsistent** +The mover is running — moving a large batch of files from cache to array. A reboot +is triggered. The mover stops mid-file. The file exists partially on both cache and +array simultaneously. unRAID's deduplication layer is confused. + +Fix: `mover_stop.sh` — warns users via wall message, waits the configured timeout, +SIGTERM (graceful — finishes current file), SIGKILL only if needed. + +--- + +## ━━━ WHAT THIS FOLDER DOES ━━━ ``` -unRAID_Essentials/ ← acts on the server itself (this folder) -Docker_Essentials/ ← acts on containers -Media/ ← acts on the library -Monitors/ ← observes and reports -``` - -> **The escalation chain matters here.** Docker_Essentials handles container-level -> problems. unRAID_Essentials handles server-level problems. The watchdogs are -> designed to work together — docker_watchdog.sh heals containers first, -> system_watchdog.sh reboots only when healing has failed. - ---- - -## ━━━ THE PROBLEM THAT BUILT THIS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - ---- - -### 🔴 Server Getting Into Unstable States With No Recovery Path - -A container has a severe memory leak. RAM drops to 2GB. The system starts swapping. -Everything slows down. Docker watchdog tries to restart the container — but Docker -itself is barely responding. The restart hangs. The watchdog is stuck. Nothing is -getting better. The server needs a reboot, but nothing in the ecosystem is authorised -to call one. - -Or: rootfs fills to 99%. SSH stops working. Docker can't write log files. The WebGUI -shows nothing useful. The server is functionally dead but still technically running. -Again — needs a reboot, nothing calls one. - -The fix: `system_watchdog.sh` — the last line of defense. Three-tier response: -immediate reboot on critical failures, OOM-confirmed bypass for RAM crises, and a -strike system for sustained threshold breaches. When everything else has failed, -system_watchdog reboots cleanly before a hard crash happens. - ---- - -### 🔴 WebGUI Freezing and Nobody Noticing - -The WebGUI becomes unresponsive. Nginx gets into a bad state. Or PHP-FPM workers are -saturated and new requests are queueing indefinitely. Or emhttp itself has frozen. -From a user perspective: dashboard doesn't load, settings don't save, containers -can't be started or stopped via the UI. - -Nothing in the container stack alerts on a frozen WebGUI — it's not a container -problem, it's a web server problem. The only way to know is if someone tries to use -the UI and notices. By which point it may have been broken for hours. - -The fix: `webgui_restart.sh` — checks every 10 minutes, escalates through nginx → -php-fpm → emhttp. Lightest fix first. Notifies on any restart so you know it happened. -Silent when healthy — 144 runs per day with no output is the correct behaviour. - ---- - -### 🔴 50+ Containers Starting and Filling Syslog With Veth Noise - -Array starts. 50+ containers come up simultaneously. Docker creates a virtual network -interface for each one. Each creation generates multiple syslog entries. In the first -few minutes after array start the syslog is buried under hundreds of lines of: - -``` -kernel: veth2a3b4c5: renamed from eth0 -kernel: docker0: port 1(veth2a3b4c5) entered blocking state -kernel: docker0: port 1(veth2a3b4c5) entered forwarding state -``` - -Real events — a failed mount, a permission error, a service that didn't start — -are invisible in this noise. And on a busy server that restarts containers regularly, -this noise continues throughout the day. - -The fix: `docker_syslog_filter.sh` — creates an rsyslog drop rule before any -container starts. Applied at array start, idempotent, silent when already correct. - ---- - -### 🔴 WebGUI Queueing Requests Under Load Without Explanation - -The WebGUI feels slow. Clicking a button takes 5 seconds. Saving settings seems to -hang. Nothing in the logs explains it. Container starts from the UI timeout. The -server itself is not under load — CPU is fine, RAM is fine. - -The cause: PHP-FPM's `pm.max_children` defaults to 4-8 workers. On a server with -multiple users, active plugins, automated tools polling the API, and 50+ containers -all potentially hitting the WebGUI simultaneously, those 4-8 workers saturate -immediately. New requests queue behind active ones. Everything feels slow. - -The fix: `php_fpm_max_children.sh` — sets `pm.max_children=250` at array start. -250 workers × ~40MB = ~10GB worst case. On 128GB this is trivially small. The -WebGUI becomes responsive immediately. Idempotent — silent when already correct. - ---- - -### 🔴 inotify Exhaustion Producing Unexplained Failures - -Already documented in README-Monitors.md (system_tuning_monitor.sh section). Short -version: when inotify limits are exhausted, containers silently stop receiving file -system events. Downloads complete but arrs don't detect them. The kernel hits the -limit and new watches fail silently. VSCode shows "unable to watch for file changes" -and misses edits. - -The fix: `inotify_tuning.sh` — raises all three inotify limits at array start. -1M watches (raised from 512K — Code-Server with node_modules needs this), 1024 -instances, 32768 queued events. The startup race note: if Code-Server starts before -this runs, it inherits old limits. Restart Code-Server if the VSCode error appears -after limits are applied. - ---- - -### 🔴 Mover Getting Killed Mid-Transfer Leaving Files Inconsistent - -The mover is running — moving a large batch of files from cache to array. Someone -clicks reboot from the UI. Or a script kills the mover process directly. The mover -stops mid-file. The file exists partially on both cache and array simultaneously. -unRAID's deduplication layer is confused. The file is inaccessible. - -The fix: `mover_stop.sh` — warns logged-in users via wall message, waits the -configured timeout, then sends SIGTERM (graceful) and verifies. The mover gets to -finish its current file operation before stopping. SIGKILL is a last resort with a -warning that partial files may exist. - ---- - -## ━━━ WHAT THIS FOLDER DOES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Four distinct server-level roles: - -``` -🛡️ Last-resort stability system_watchdog.sh — reboots before crash -🌐 WebGUI availability webgui_restart.sh — recovers frozen UI -⚙️ Kernel tuning inotify_tuning.sh — file watch limits - php_fpm_max_children.sh — PHP worker count -🔇 Log hygiene docker_syslog_filter.sh — suppress veth noise - clear_logs.sh — weekly log trimming -🔄 Graceful operations mover_stop.sh — clean mover stop - server_reboot.sh — clean reboot with warning - user_scripts_stop.sh — stop running scripts +Last-resort stability system_watchdog.sh — reboots before crash +Pressure reduction resource_watchdog.sh — throttle/pause/stop under load +WebGUI availability webgui_restart.sh — nginx → php-fpm → emhttp escalation +Kernel tuning inotify_tuning.sh — file watch limits + php_fpm_max_children.sh — PHP worker count +Log hygiene docker_syslog_filter.sh — suppress veth noise at start + clear_logs.sh — weekly log trimming +Graceful operations mover_stop.sh — clean mover stop + rsync_stop.sh — smart rsync stop (orchestrator-aware) + user_scripts_stop.sh — stop running User Scripts + server_reboot.sh — clean reboot with pre-flight warnings ``` --- -## ━━━ SCRIPTS AT A GLANCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ━━━ RELATIONSHIP TO OTHER FOLDERS ━━━ -| Script | Purpose | When | -|--------|---------|------| -| `system_watchdog.sh` | Three-tier last-resort stability watchdog | Continuous background loop | -| `webgui_restart.sh` | WebGUI availability — nginx → php-fpm → emhttp escalation | Every 10 minutes | -| `inotify_tuning.sh` | Raise inotify kernel limits | At array start | -| `php_fpm_max_children.sh` | Set PHP-FPM worker count | At array start | -| `docker_syslog_filter.sh` | Suppress Docker veth log noise | At array start | -| `clear_logs.sh` | Size-threshold weekly log cleanup | Weekly via maintenance window | -| `mover_stop.sh` | Clean mover stop with SIGTERM → SIGKILL | Manual | -| `server_reboot.sh` | Graceful reboot with pre-flight warnings | Manual or called by system_watchdog | -| `user_scripts_stop.sh` | Stop all running User Script processes | Manual or called by server_reboot | +``` +Orchestrators/ + array_start.sh ─────────────────────────► inotify_tuning.sh (first in sequence) + ─────────────────────────► docker_syslog_filter.sh (second) + ─────────────────────────► php_fpm_max_children.sh + ─────────────────────────► system_watchdog.sh (background loop) ---- + watchdog_orchestrator.sh ───────────────► resource_watchdog.sh (every minute) + weekly_maintenance.sh ──────────────────► clear_logs.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🛡️ system_watchdog.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + server_reboot.sh ────────────────────────► user_scripts_stop.sh (called internally) -The last line of defense. Reboots the system cleanly before it crashes uncleanly. -Three-tier response system — critical failures bypass everything and reboot immediately, -OOM-confirmed crises bypass the strike system, sustained threshold breaches use strikes. -Runs continuously as a background process started by `array_start.sh`. - -> Full architecture documentation in `README-Docker_Essentials.md` under -> "Relationship to System Watchdog". This section covers the system_watchdog itself. - -```bash -# Started by: array_start.sh (continuous background process) -# Interval: SYSTEM_WATCHDOG_INTERVAL=300 (5 minutes) +Docker_Essentials/ + docker_watchdog.sh ◄─── reads ──────────── resource_watchdog.sh state + (mem_shutdown_active flag) ``` +`system_watchdog.sh` and `docker_watchdog.sh` (in Docker_Essentials/) are +designed to work together — docker_watchdog heals containers first, +system_watchdog reboots only when healing has failed. `resource_watchdog.sh` +coordinates with docker_watchdog via the `mem_shutdown_active` state flag to +prevent docker_watchdog from restarting containers that resource_watchdog just +stopped to free RAM. + --- -### ── Three-Tier Response System ────────────────────────────────────────────── +## ━━━ SCRIPTS IN THIS FOLDER ━━━ -```bash -# ───────────────────────────────────────────────────────────────────────────── -# TIER 1 — CRITICAL (bypass ALL strikes, reboot immediately) -# These failures are acute — the system is not recoverable by waiting. -# Single detection = immediate reboot. No confirmation window. -# -# Docker daemon unresponsive: -# Attempt /etc/rc.d/rc.docker restart first. -# Wait 15 seconds. Verify daemon responding. -# If still hung → CRITICAL reboot. -# A hung daemon cannot be healed — every subsequent docker command hangs. -# -# rootfs at ROOTFS_CRITICAL_PCT (99%+): -# Writes are failing. SSH may stop. Logs can't be written. -# Nothing can be fixed from this state without a reboot. -# -# Kernel BUG/Oops in dmesg: -# Kernel running with corrupted state. -# Delta-based: new oops since last cycle → reboot. -# -# File descriptor exhaustion (FD_CRITICAL_PCT=95%): -# New connections failing. Docker can't spawn processes. SSH drops. -# -# /boot read-only: -# State files and config writes silently failing. -# Write test on /boot every cycle. +| Script | Role | When It Runs | +|--------|------|-------------| +| `system_watchdog.sh` | Three-tier last-resort stability watchdog | Continuous background loop via array_start.sh | +| `resource_watchdog.sh` | Pressure reduction — throttle/pause/stop under load | Every minute via watchdog_orchestrator.sh | +| `webgui_restart.sh` | WebGUI availability — nginx → php-fpm → emhttp | Every 10 min via User Scripts | +| `inotify_tuning.sh` | Raise inotify kernel limits | At array start — FIRST | +| `php_fpm_max_children.sh` | Set PHP-FPM max worker count | At array start | +| `docker_syslog_filter.sh` | Suppress Docker veth syslog noise | At array start — before containers | +| `clear_logs.sh` | Size-threshold log cleanup | Weekly via weekly_maintenance.sh | +| `mover_stop.sh` | Clean mover stop with SIGTERM → SIGKILL | Manual / before reboot | +| `rsync_stop.sh` | Orchestrator-aware rsync stop | Manual | +| `user_scripts_stop.sh` | Stop all running User Script processes | Manual / called by server_reboot.sh | +| `server_reboot.sh` | Graceful reboot with pre-flight warnings | Manual | -# TIER 2 — URGENT (bypass strikes when OOM confirms crisis) -# RAM < MEM_GB (4GB) AND OOM kills >= OOM_LIMIT (3) in this cycle -# Without OOM confirmation → standard strike system applies. -# Rationale: 1-2 OOM kills = docker_watchdog.sh handles it. -# 3+ kills while RAM critical = system dying faster than watchdogs heal. -# OOM victims from dmesg included in reboot message (diagnostic context). +--- + +## ━━━ HOW THE SCRIPTS RELATE ━━━ -# TIER 3 — STANDARD (strike system — N consecutive failures → reboot) -# Everything else: RAM tiers, load, CPU temp, zombies, /var/log, -# /tmp, containers, NIC, mdstat, sshd -# ───────────────────────────────────────────────────────────────────────────── ``` +Array starts + │ + ├─ inotify_tuning.sh ← FIRST — kernel limits inherited at container launch + ├─ docker_syslog_filter.sh ← SECOND — before any veth interfaces are created + ├─ php_fpm_max_children.sh ← before WebGUI is under load + └─ system_watchdog.sh ← starts background loop ---- -### ── RAM Tiers ──────────────────────────────────────────────────────────────── +Every minute (watchdog_orchestrator.sh): + └─ resource_watchdog.sh + Level 1 (soft): throttle SABnzbd + qBit download speeds + Level 2 (medium): further throttle + docker pause non-essential containers + Level 3 (hard): docker stop optional services + set mem_shutdown_active=true + ↓ + docker_watchdog.sh reads mem_shutdown_active — defers restarts -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# Three-level graduated response — not a single threshold. -# - SYS_WATCHDOG_MEM_WARN_GB=10 # warn + notify once — informational only - SYS_WATCHDOG_MEM_SHUTDOWN_GB=6 # stop non-essential containers, wait for recovery - SYS_WATCHDOG_MEM_GB=4 # strike system → reboot (or bypass if OOM confirms) - SYS_WATCHDOG_MEM_RECOVER_GB=30 # RAM must reach this before restarting containers -# -# Containers excluded from RAM emergency shutdown — these stay running: - SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED=( - "NginxProxyManager" # external access — stop this and users lose everything - "Authelia" # auth — without this nothing is accessible - "Mariadb" # Authelia dependency - "Redis" # Authelia dependency - "Emby" # media server — Live TV buffering - "Dispatcharr" # Live TV scheduler — loses state if stopped - ) -# -# Coordination with docker_watchdog.sh: -# system_watchdog writes mem_shutdown_active=true to SYS_WATCHDOG_STATE_FILE. -# docker_watchdog reads this flag and defers ALL container restart logic. -# Without this: both watchdogs fight — system_watchdog stops containers, -# docker_watchdog restarts them, RAM never recovers. -# With this: docker_watchdog stands down until mem_shutdown_active clears. -# ───────────────────────────────────────────────────────────────────────────── + +Every 10 minutes (User Scripts): + └─ webgui_restart.sh + WebGUI OK → silent exit + Not responding: + Step 1: restart nginx → recheck + Step 2: restart php-fpm → recheck + Step 3: restart emhttp → recheck + All failed → notify, exit 1 + + +Weekly (weekly_maintenance.sh): + └─ clear_logs.sh + System logs: clear if > LOG_MIN_SIZE_MB + Docker logs: clear per-container if > LOG_DOCKER_MAX_MB + + +Manual operations: + mover_stop.sh → wall → SIGTERM → SIGKILL → verify stopped + rsync_stop.sh → detect orchestrator → kill rsync (or orchestrator+rsync) + user_scripts_stop.sh → scan /proc → SIGTERM → SIGKILL per process + server_reboot.sh → pre-flight → wall → wait → VMs → Docker → sync → reboot ``` - ---- - -### ── Per-Host Check Toggles ─────────────────────────────────────────────────── - -```bash -# master_host1.conf (all checks in master_host*.conf — not master.conf) -# ───────────────────────────────────────────────────────────────────────────── -# Each check is independently toggleable per server. -# HOST1 and HOST2 may have different hardware and different workloads. -# detect_hosts() aliases HOST*_SYS_WATCHDOG_CHECK_* → SYS_WATCHDOG_CHECK_* -# -# Tier 1 — Critical (bypass strikes): - HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true - HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true - HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true - HOST1_SYS_WATCHDOG_CHECK_FD=true - HOST1_SYS_WATCHDOG_CHECK_BOOT=true - -# Tier 2 — Urgent (OOM bypass): - HOST1_SYS_WATCHDOG_CHECK_OOM=true - HOST1_SYS_WATCHDOG_CHECK_RAM=true - -# Tier 3 — Standard (strike system): - HOST1_SYS_WATCHDOG_CHECK_LOG=true - HOST1_SYS_WATCHDOG_CHECK_ARC=true # HOST1 runs ZFS — enable - HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true - HOST1_SYS_WATCHDOG_CHECK_LOAD=false # disabled — transcoding causes normal spikes - HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true - HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true # monitors docker_watchdog persistent skip list - HOST1_SYS_WATCHDOG_CHECK_TMP=true - HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true - HOST1_SYS_WATCHDOG_CHECK_NETWORK=true - HOST1_SYS_WATCHDOG_CHECK_SSHD=true - HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false # disabled — may false positive during encoding - HOST1_SYS_WATCHDOG_NIC="eth0" # verify: ip link show | grep "^[0-9]" -``` - ---- - -### ── Abort Conditions ───────────────────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# Conditions that prevent a reboot even when a threshold is hit. -# CRITICAL tier bypasses these — truly critical conditions reboot regardless. -# - SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # rebooting with bad pool risks data loss - SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity is better than crashing - SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting mover is better than crashing -# -# Philosophy: a graceful reboot before a crash is always better than a hard crash. -# The abort conditions protect against the cases where a reboot itself causes harm -# (data loss from bad ZFS pool). Parity and mover can be restarted after reboot. -``` - ---- - -### ── Reboot Loop Protection ─────────────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# If the server keeps rebooting, something is wrong that rebooting isn't fixing. -# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS → shutdown instead. -# Shutdown prevents: hardware damage, filesystem corruption from repeated reboots, -# infinite loop that never lets you investigate. -# State file: /boot/config/system_watchdog_reboots.db — survives reboots. -# - SYS_WATCHDOG_REBOOT_LIMIT=3 # reboots before shutdown instead - SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 # rolling window in hours -``` - ---- - -### ── State File Heartbeat ───────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# system_watchdog writes watchdog_cycle=N to SYS_WATCHDOG_STATE_FILE every cycle. -# This keeps the file's modification time current. -# -# docker_watchdog.sh uses the state file mtime as a stale guard — if the file -# is more than 2 hours old while mem_shutdown_active=true is set, system_watchdog -# may have stopped running. docker_watchdog resumes normal operation rather than -# being silenced indefinitely by a stale flag. -# -# Without this heartbeat: if all standard checks pass and no state writes happen -# (e.g. CHECK_KERNEL_OOPS=false AND CHECK_MDSTAT=false), the file mtime could go -# stale even with the watchdog running. -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -system_watchdog.sh # normal (continuous — started by array_start.sh) -system_watchdog.sh --dry-run # trigger detection without rebooting -system_watchdog.sh --status # show all tiers, thresholds, active strikes, RAM state -system_watchdog.sh --log # verbose per-cycle output -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🌐 webgui_restart.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Monitors the unRAID WebGUI availability and recovers it automatically when unresponsive. -Three-step escalating strategy — lightest fix first, heaviest last. Silent when healthy. - -```bash -# Scheduled: */10 * * * * (every 10 minutes) -``` - ---- - -### ── Three-Step Escalation ──────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Step 1 — nginx restart -# Lightest fix — handles most WebGUI failures. -# nginx is the web server layer. Crash, worker stuck, connection timeout. -# Wait WEBGUI_NGINX_WAIT seconds → curl recheck. -# -# Step 2 — php-fpm restart -# Added because WebGUI can appear frozen due to PHP worker exhaustion. -# pm.max_children workers all occupied → new requests queue → dashboard hangs. -# php-fpm restart far less disruptive than emhttp. -# Wait WEBGUI_PHP_WAIT seconds → curl recheck. -# Note: system_tuning_monitor.sh tracks worker saturation over time. -# -# Step 3 — emhttp restart -# Heaviest fix. emhttp is the core unRAID management daemon. -# Array, Docker, shares continue running — only WebGUI management restarts. -# Takes longer to recover — WEBGUI_EMHTTP_WAIT gives it time. -# Wait WEBGUI_EMHTTP_WAIT seconds → curl recheck. -# -# If all three fail → notify warning — manual intervention needed. -# Guidance: pgrep -x nginx emhttp | check journalctl | consider server_reboot.sh -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Configuration ──────────────────────────────────────────────────────────── - -```bash -# master.conf - WEBGUI_URL="http://localhost" # adjust if non-standard port - WEBGUI_TIMEOUT=5 # seconds before curl times out - WEBGUI_NGINX_WAIT=15 # seconds after nginx restart before recheck - WEBGUI_PHP_WAIT=10 # seconds after php-fpm restart before recheck - WEBGUI_EMHTTP_WAIT=30 # seconds after emhttp restart — takes longer -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -webgui_restart.sh # check once — silent if healthy, escalates if not -webgui_restart.sh --dry-run # walk through escalation without restarting anything -webgui_restart.sh --status # show WebGUI state + nginx/php-fpm/emhttp process state -webgui_restart.sh --log # verbose — show each escalation step -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 📡 inotify_tuning.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Raises Linux inotify kernel limits at array start. Idempotent — completely silent -when values are already correct. - -```bash -# Scheduled: At Startup of Array (via array_start.sh — FIRST in ARRAY_START_SCRIPTS) -# Must run before containers start — containers inherit limits at startup -``` - ---- - -### ── Three Limits ───────────────────────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── - INOTIFY_MAX_INSTANCES=1024 # default: 128 — max inotify instances per user - # 1024 handles ~20-30 containers watching files - - INOTIFY_MAX_WATCHES=1048576 # default: 8192 — SHARED budget across ALL users/containers - # Was 512K — raised to 1M (1048576) - # VSCode/Code-Server alone needs ~50K-200K for large workspaces - # with node_modules. All arr containers + Emby + VSCode share - # this budget. 1M safe on 128GB RAM (~128MB kernel memory) - # If VSCode shows "unable to watch for file changes" → too low - - INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — events buffered before dropping -``` - ---- - -### ── Startup Race Condition ─────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# inotify limits are kernel-wide — they take effect immediately on sysctl write. -# But containers that have ALREADY started inherit the OLD limits at startup. -# Those containers keep their inherited (low) limits until restarted. -# -# This is why inotify_tuning.sh must be FIRST in ARRAY_START_SCRIPTS — before -# any container starts. If Code-Server starts before limits are raised: -# → Code-Server inherits old 8192 watch limit -# → VSCode shows "unable to watch for file changes" -# → Fix: docker restart Code-Server (picks up current kernel limits on start) -# -# The script warns if it changed any values: -# "If Code-Server is running: docker restart Code-Server" -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -inotify_tuning.sh # normal run (idempotent — silent when correct) -inotify_tuning.sh --dry-run # show what would change without changing -inotify_tuning.sh --status # current values vs targets + top inotify consumers -inotify_tuning.sh --log # verbose — show each sysctl write -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## ⚙️ php_fpm_max_children.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Raises PHP-FPM `pm.max_children` at array start to prevent WebGUI queueing under -load. Idempotent — completely silent when already correct. No PHP-FPM restart unless -value actually changed. - -```bash -# Scheduled: At Startup of Array (via array_start.sh) -``` - ---- - -### ── Why 250 Workers ────────────────────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── - PHP_MAX_CHILDREN=250 # default: 4-8 — far too low for a busy server - # Each worker: ~2MB resident memory at idle - # 250 workers × 2MB = ~500MB — trivial on 128GB - # Worst case (all active): ~250 × 40MB = ~10GB - # In practice: rarely all active simultaneously - # On 64GB (HOST2): still appropriate — 250 × 40MB - # = 10GB worst case = 15% of RAM, acceptable -# -# Without this fix: -# 5 users hit the WebGUI simultaneously → 8 workers exhausted → 5 more queue -# Each queued request waits for a worker to free → 5-10 second response times -# Looks like a slow server — it's just a queue -# - PHP_CONF="/etc/php83/php-fpm.d/www.conf" # path may change with PHP version -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -php_fpm_max_children.sh # normal run (idempotent) -php_fpm_max_children.sh --dry-run # show what would change -php_fpm_max_children.sh --status # show current value vs target + worker count -php_fpm_max_children.sh --log # verbose — show config write + restart -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🔇 docker_syslog_filter.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Creates an rsyslog drop rule for Docker veth/docker0 interface noise. Idempotent — -silent when filter already correct, only writes + restarts rsyslog when something changed. - -```bash -# Scheduled: At Startup of Array (via array_start.sh — before containers start) -``` - ---- - -### ── What Gets Suppressed ───────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# These kernel messages are generated on every container start and stop: -# -# kernel: veth2a3b4c5: renamed from eth0 -# kernel: docker0: port 1(veth2a3b4c5) entered blocking state -# kernel: docker0: port 1(veth2a3b4c5) entered forwarding state -# kernel: docker0: port 1(veth2a3b4c5) entered disabled state -# -# 50+ containers at array start = 200-400 lines of this in the first minute. -# Containers restart throughout the day = continuous noise. -# Real events buried and invisible in syslog. -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Idempotent Design ──────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# Runs at every array start but only changes something when needed: -# Filter file exists and content is correct → exit 0 silently -# Filter file missing or content changed → write + restart rsyslog -# -# Expected content compared exactly — single source of truth: -EXPECTED_FILTER='if ($msg contains "veth" or $msg contains "docker0") then { - stop -}' -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -docker_syslog_filter.sh # normal run (idempotent — silent when correct) -docker_syslog_filter.sh --dry-run # show what would be written without writing -docker_syslog_filter.sh --status # show current filter file + rsyslog state -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🗑️ clear_logs.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Clears system logs and Docker container logs using size thresholds. Only clears logs -large enough to be worth clearing — preserves recent diagnostic context on small logs. - -```bash -# Called by: weekly_sync_maintenance.sh via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am) -``` - ---- - -### ── Size Threshold Approach ────────────────────────────────────────────────── - -```bash -# master.conf -# ───────────────────────────────────────────────────────────────────────────── -# Why thresholds instead of clearing everything: -# A 2MB syslog contains useful recent history — not worth clearing. -# A 500MB Docker log is consuming rootfs and contains mostly noise — clear it. -# Blind truncation destroys diagnostic context for no benefit. -# - LOG_MIN_SIZE_MB=10 # skip system log if under this size — keep history - LOG_DOCKER_MAX_MB=100 # clear Docker container log only if over this size - # Active containers (Emby, SABnzbd) grow fastest - # 100MB × 30 containers = 3GB before any clearing kicks in - - LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) -# -# Truncation not logrotate: -# unRAID writes to tmpfs (/var/log) — logrotate's compress + archive approach -# would consume more tmpfs space, not less. -# : > file keeps the file descriptor valid while emptying content. -# Safe for running services (syslogd continues writing to the same fd). -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -clear_logs.sh # normal run — silent if all logs under threshold -clear_logs.sh --dry-run # show what would be cleared and sizes -clear_logs.sh --status # show current log sizes vs thresholds -clear_logs.sh --log # verbose — show each file evaluated -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## ⏹️ mover_stop.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Stops the unRAID mover cleanly — wall warning, configurable timeout, SIGTERM → verify -→ SIGKILL sequence. Safe to run when mover is not running — exits cleanly with a log. - ---- - -### ── Stop Sequence ──────────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# 1. Check if mover is running — exit cleanly if not -# 2. Wall message: "HOST1 (unRAID-Gmer4Lfe) — mover stopping in 30s" -# MY_ID included — on shared terminal it's clear which server -# 3. Wait MOVER_STOP_TIMEOUT seconds -# 4. SIGTERM — allows mover to finish its current file before stopping -# No partial files — the mover completes what it's working on -# 5. Wait 5 seconds — verify if stopped -# 6. If still running → SIGKILL (forced) -# Warning: partial files possible — same as a hard crash -# 7. Final verify — error if still running -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -mover_stop.sh # stop mover with configured timeout -mover_stop.sh --dry-run # show what would happen -mover_stop.sh --status # show mover state (running, PID, start time) -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🔁 server_reboot.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Graceful reboot with pre-flight warnings, wall message, unRAID notification, VM -graceful shutdown, then Docker and services stop, sync, reboot. - -> Full documentation in `README-Tools.md` — `server_reboot.sh` section. This is a -> quick reference. - ---- - -### ── Shutdown Sequence ──────────────────────────────────────────────────────── - -```bash -# ───────────────────────────────────────────────────────────────────────────── -# 1. Pre-flight warnings (warn not block): -# rsync running, mover running, active Emby sessions -# Warnings show in summary — you chose to reboot, these are for context -# 2. Wall message + unRAID notification — MY_ID included -# 3. Wait REBOOT_SLEEP seconds (default 30) -# 4. virsh shutdown each VM → wait REBOOT_VM_WAIT seconds for graceful exit -# 5. Stop libvirt (VM Manager) -# 6. Stop Docker service -# 7. sync — flush filesystem buffers -# 8. /sbin/reboot -# ───────────────────────────────────────────────────────────────────────────── -``` - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -server_reboot.sh # reboot with 30s warning -server_reboot.sh --dry-run # full sequence walkthrough without rebooting -server_reboot.sh --status # show running processes that would be affected -server_reboot.sh --reason="maintenance" # include reason in wall + notification -server_reboot.sh --log # verbose per-step output -``` - ---- - -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🛑 user_scripts_stop.sh -## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Stops all running User Script processes. Identifies by `/tmp/user.scripts` path -signature. Shows script names not PIDs. SIGTERM → verify → SIGKILL with self-exclusion. - -> Full documentation in `README-Tools.md` — `user_scripts_stop.sh` section. - ---- - -### ── Usage ─────────────────────────────────────────────────────────────────── - -```bash -user_scripts_stop.sh # stop all — SIGTERM → SIGKILL if needed -user_scripts_stop.sh --dry-run # show which scripts would be stopped, by name -user_scripts_stop.sh --status # show running scripts with PID and runtime -user_scripts_stop.sh --log # verbose per-process output -``` - ---- - -## ━━━ STARTUP SEQUENCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -```bash -# master.conf — ARRAY_START_SCRIPTS (order matters) -# ───────────────────────────────────────────────────────────────────────────── -ARRAY_START_SCRIPTS=( - # ── One-shot — run and exit ─────────────────────────────────────────────── - "unRAID_Essentials/inotify_tuning.sh" # FIRST — raise limits before - # containers inherit old values - "unRAID_Essentials/docker_syslog_filter.sh" # SECOND — before containers - # create veth interfaces - "unRAID_Essentials/php_fpm_max_children.sh" # before WebGUI serves requests - "Transcodes/ramdisk_setup.sh" # before Emby starts transcoding - "Docker_Essentials/docker_network_connect.sh" # before watchdogs check states - - # ── Continuous — run until array stops ─────────────────────────────────── - "unRAID_Essentials/system_watchdog.sh" # before docker_watchdog — - # writes state file docker_watchdog reads - "Docker_Essentials/docker_watchdog.sh" # before failover — containers - # must be healthy for failover decisions - "Failover/failover.sh" # last — needs everything stable -) -# ───────────────────────────────────────────────────────────────────────────── -# array_start.sh is the ONLY "At Startup of Array" entry in User Scripts. -# It launches everything above in order. -``` - ---- - -## ━━━ FULL SCHEDULE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -```bash -# At Startup of Array — via array_start.sh: -# inotify_tuning.sh -# docker_syslog_filter.sh -# php_fpm_max_children.sh -# (ramdisk_setup.sh — in Transcodes/) -# system_watchdog.sh (continuous) - -# Every 10 minutes: -*/10 * * * * webgui_restart.sh # silent when healthy — escalates when not - -# Weekly — via weekly_sync_maintenance.sh: -# clear_logs.sh # Sunday 2:30am via WEEKLY_MAINTENANCE_SCRIPTS - -# Manual: -# mover_stop.sh — before array ops that need mover stopped -# server_reboot.sh — planned maintenance reboots -# user_scripts_stop.sh — emergency script stop or pre-reboot cleanup -``` \ No newline at end of file diff --git a/unRAID_Essentials/clear_logs.sh b/unRAID_Essentials/clear_logs.sh index 65dc1c8..a8c21d7 100644 --- a/unRAID_Essentials/clear_logs.sh +++ b/unRAID_Essentials/clear_logs.sh @@ -2,51 +2,81 @@ # ============================================================================================== # ================================= Clear Logs ================================================= # ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── # Clears system and Docker container logs to prevent rootfs fill over time. -# Runs weekly via WEEKLY_MAINTENANCE_SCRIPTS — Sunday 2:30am. -# Uses size thresholds — only clears logs that have grown large enough to matter. +# Called weekly via WEEKLY_MAINTENANCE_SCRIPTS. Uses size thresholds — only +# clears logs large enough to be worth clearing. Small logs are left intact, +# preserving recent diagnostic context. # -# ── WHAT IT CLEARS ──────────────────────────────────────────────────────────────────────────── -# System logs — LOG_FILES from master.conf (/var/log/syslog, messages, dmesg) -# Cleared if size exceeds LOG_MIN_SIZE_MB -# These grow continuously — weekly clearing keeps rootfs healthy +# System logs (LOG_FILES): cleared if size exceeds LOG_MIN_SIZE_MB. +# Docker logs (/var/lib/docker/containers/**/*-json.log): cleared only if the +# individual container log exceeds LOG_DOCKER_MAX_MB. Active containers (Emby, +# SABnzbd) grow fastest — inactive containers typically remain small. # -# Docker logs — /var/lib/docker/containers/**/*-json.log -# Cleared only if individual container log exceeds LOG_DOCKER_MAX_MB -# Active containers (Emby, SABnzbd) grow fastest — 100MB+ easily -# Inactive containers not cleared — their logs are typically small +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== # -# ── SIZE THRESHOLD APPROACH ─────────────────────────────────────────────────────────────────── -# Truncating everything blindly destroys useful diagnostic context. -# A 2MB log is not worth clearing — it contains useful recent history. -# A 500MB log is consuming rootfs and contains mostly noise — clear it. +# Size Thresholds, Not Blind Truncation +# A 2MB syslog contains useful recent diagnostic history — not worth clearing. +# A 500MB Docker log is consuming rootfs and contains mostly noise — clear it. +# Blind truncation destroys diagnostic context for no benefit. # -# LOG_MIN_SIZE_MB — system logs under this size are left alone -# LOG_DOCKER_MAX_MB — Docker logs under this size are left alone +# Truncation, Not Logrotate +# unRAID writes logs to tmpfs (/var/log). Logrotate's compress + archive approach +# would consume more tmpfs space, not less. Truncation (`: > file`) keeps the +# file descriptor open and valid while emptying content — syslogd continues +# writing to the same fd without interruption. # -# ── WHY NOT LOGROTATE ───────────────────────────────────────────────────────────────────────── -# unRAID writes to tmpfs (/var/log) — logrotate's compress + archive approach -# would consume even more tmpfs space. Truncation (: > file) keeps the file -# descriptor open and valid while emptying content — safe for running services. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock — prevents concurrent runs corrupting logs -# Root check — truncating system logs requires root -# Size thresholds — only clears logs that have grown large enough -# Byte tracking — reports MB freed for weekly digest -# validate_unraid — notify validated before use -# Silent on clean — small logs = nothing to clear = no output ✅ +# Single Instance Lock +# acquire_lock prevents concurrent runs from corrupting logs. # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# LOG_FILES — system log paths to check and clear -# LOG_MIN_SIZE_MB — minimum system log size before clearing (default 10MB) -# LOG_DOCKER_MAX_MB — clear Docker log only if above this size (default 100MB) +# Root Required +# Truncating system logs requires root. +# +# Size Thresholds +# Each file checked against its threshold before clearing. +# +# Silent When Clean +# All logs below threshold = no visible output. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# LOG_FILES +# System log paths to check and clear. (default: /var/log/syslog /var/log/messages /var/log/dmesg) +# +# LOG_MIN_SIZE_MB +# Skip system log if under this size — keep recent history. (default: 10) +# +# LOG_DOCKER_MAX_MB +# Clear Docker container log only if over this size. (default: 100) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# clear_logs.sh +# Check all configured logs. Clear those above threshold. Silent when all are small. +# +# clear_logs.sh --dry-run +# Show which logs would be cleared and their current sizes. No clearing. +# +# clear_logs.sh --status +# Show current log sizes vs thresholds. +# +# clear_logs.sh --log +# Verbose output — show each file evaluated, its size, and action taken. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# clear_logs.sh — normal run (threshold-based clearing) -# clear_logs.sh --dry-run — show what would be cleared and sizes -# clear_logs.sh --status — show current log sizes -# clear_logs.sh --log — verbose output per file # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/unRAID_Essentials/docker_syslog_filter.sh b/unRAID_Essentials/docker_syslog_filter.sh index 2b04f0f..e4d96b8 100755 --- a/unRAID_Essentials/docker_syslog_filter.sh +++ b/unRAID_Essentials/docker_syslog_filter.sh @@ -2,49 +2,76 @@ # ============================================================================================== # ============================= Docker Syslog Filter =========================================== # ============================================================================================== -# Suppresses noisy Docker veth/docker0 interface messages from syslog. -# Run once at array start via ARRAY_START_SCRIPTS in master.conf. -# Idempotent — completely silent when filter is already correct. # -# ── WHY THIS EXISTS ─────────────────────────────────────────────────────────────────────────── -# Every time Docker creates or destroys a container network interface it logs messages like: -# kernel: veth2a3b4c5: renamed from eth0 -# kernel: docker0: port 1(veth2a3b4c5) entered blocking state -# kernel: docker0: port 1(veth2a3b4c5) entered forwarding state +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Suppresses Docker veth/docker0 network interface messages from syslog. Run +# once at array start via ARRAY_START_SCRIPTS before any containers start. +# Idempotent — completely silent when the filter is already in place. # -# On a busy server creating and restarting many containers these fill syslog rapidly — -# hundreds of entries per minute on container restarts, completely masking real events. -# The filter tells rsyslog to drop these before they reach the log file. +# Every container start/stop generates kernel messages like: +# "veth2a3b4c5: renamed from eth0" +# "docker0: port 1(veth...) entered forwarding state" +# 50+ containers at array start = 200–400 lines of noise in the first minute, +# completely masking real events. # -# ── HOW IT WORKS ────────────────────────────────────────────────────────────────────────────── -# Creates /etc/rsyslog.d/ignore-docker-veth.conf (FILTER_FILE in master.conf). -# rsyslog processes .conf files in /etc/rsyslog.d/ automatically on startup. -# Filter uses rsyslog's RainerScript to match messages containing "veth" or "docker0" -# and calls stop — the message is dropped before reaching any output target. +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== # -# ── IDEMPOTENT DESIGN ───────────────────────────────────────────────────────────────────────── -# On every array start: checks if filter file already exists with correct content. -# If already correct → completely silent — no rsyslog restart, no output. -# Only writes + restarts rsyslog if filter is missing or content has changed. -# This prevents unnecessary rsyslog restarts on every boot. +# Idempotent Content Check +# Checks whether the filter file already exists with exactly the expected content. +# If already correct → silent exit, no rsyslog restart. Only writes and restarts +# rsyslog when the filter is missing or content has changed. Running at every boot +# without this would cause an unnecessary rsyslog restart on every array start. # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# Root check — writing to /etc/rsyslog.d/ requires root -# acquire_lock — prevents concurrent runs at array start -# Idempotent check — only restarts rsyslog when filter actually changed -# Directory creation — mkdir -p /etc/rsyslog.d/ before writing -# rsyslog verify — checks rsyslog running after restart -# validate_unraid — notify validated before use -# Silent on success — runs every boot, no noise when already correct +# rsyslog Drop Rule +# Creates FILTER_FILE (/etc/rsyslog.d/ignore-docker-veth.conf). +# Uses RainerScript `stop` action — messages matching "veth" or "docker0" are +# dropped before reaching any output target, including the log file. # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# FILTER_FILE — path for rsyslog drop filter (default /etc/rsyslog.d/ignore-docker-veth.conf) +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root Required +# Writing to /etc/rsyslog.d/ requires root. +# +# Single Instance Lock +# acquire_lock prevents concurrent runs at array start. +# +# rsyslog Process Verify +# Confirms rsyslog is running after restart. +# +# Silent on Success +# Runs every boot — no noise when filter is already correct. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# FILTER_FILE +# Path for the rsyslog drop filter config file. +# (default: /etc/rsyslog.d/ignore-docker-veth.conf) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# docker_syslog_filter.sh +# Check filter file. Write and restart rsyslog only if changed. Silent if correct. +# +# docker_syslog_filter.sh --dry-run +# Show what would be written without writing or restarting rsyslog. +# +# docker_syslog_filter.sh --status +# Show current filter file content and rsyslog process state. +# +# docker_syslog_filter.sh --log +# Verbose output showing the content comparison and any changes applied. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# docker_syslog_filter.sh — normal run (idempotent) -# docker_syslog_filter.sh --dry-run — show what would change -# docker_syslog_filter.sh --status — show filter file state and rsyslog status -# docker_syslog_filter.sh --log — verbose output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/unRAID_Essentials/inotify_tuning.sh b/unRAID_Essentials/inotify_tuning.sh index 5945d61..ce70f5f 100755 --- a/unRAID_Essentials/inotify_tuning.sh +++ b/unRAID_Essentials/inotify_tuning.sh @@ -2,65 +2,87 @@ # ============================================================================================== # ================================= inotify Tuning ============================================ # ============================================================================================== -# Raises Linux inotify limits at array start to prevent exhaustion across the container stack. -# Run once at array start via ARRAY_START_SCRIPTS in master.conf. -# Settings are lost on reboot — this script reapplies them on every array start. # -# ── THREE INOTIFY LIMITS ────────────────────────────────────────────────────────────────────── -# max_user_instances — max number of independent inotify file descriptor objects per user -# Each container that calls inotify_init() consumes one instance -# Default 128 — exhausted quickly with 20+ active containers +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Raises Linux inotify limits at array start to prevent exhaustion across the +# container stack. Run once at array start via ARRAY_START_SCRIPTS. Settings +# are lost on reboot — this script reapplies them on every array start. # -# max_user_watches — SHARED budget across ALL users and containers on the system -# Each watched file or directory costs one watch from this pool -# Default 8192 — VSCode alone can need 50K-200K for large workspaces +# Must run FIRST in ARRAY_START_SCRIPTS before any containers start — containers +# inherit inotify limits at launch, not dynamically. Running this after Code-Server +# starts requires a docker restart to pick up the new values. # -# max_queued_events — max events buffered before kernel starts dropping them -# Low value = events silently lost during high-activity periods -# Default 16384 — sufficient for most setups +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== # -# ── WHY VSCODE THROWS "UNABLE TO WATCH FOR FILE CHANGES" ───────────────────────────────────── -# VSCode (and Code-Server in Docker) opens one inotify watch per file in the workspace. -# A typical project with node_modules can easily have 100K-200K files. -# All containers on the host share max_user_watches — the combined usage of: -# Sonarr, Radarr, Lidarr, Emby, Nextcloud, Code-Server, AdGuard, all other arrs -# easily exceeds 524288 (512K) watches on a busy server. -# Raising to 1048576 (1M) gives sufficient headroom — safe on 128GB RAM (~128MB kernel use). +# Three inotify Limits +# max_user_instances — max independent inotify fd objects per user; each container +# calling inotify_init() consumes one. Default 128 — exhausted +# quickly with 20+ active containers. # -# ── STARTUP ORDER MATTERS ───────────────────────────────────────────────────────────────────── -# inotify_tuning.sh must run BEFORE containers that watch files start. -# In ARRAY_START_SCRIPTS order: inotify_tuning.sh first, then container-starting scripts. -# If Code-Server starts before limits are raised it inherits the old (low) limits. -# Code-Server restart fixes this: limits are kernel-wide, not process-bound at start. -# So if Code-Server is already running: docker restart Code-Server after this script runs. +# max_user_watches — SHARED budget across ALL users and containers on the system. +# Each watched file or directory costs one watch. Default 8192 — +# VSCode alone needs 50K–200K for large workspaces. Combined +# usage of Sonarr, Radarr, Lidarr, Emby, Nextcloud, Code-Server +# easily exceeds 512K on a busy server. # -# ── CONSUMERS ON THIS STACK ─────────────────────────────────────────────────────────────────── -# Emby — watches all media library paths (1 watch per folder) -# Sonarr — watches TV_Shows folder tree -# Radarr — watches Movies folder tree -# Lidarr — watches Music folder tree -# Nextcloud — watches data directory for changes -# Code-Server — watches entire workspace (can be 50K-200K with node_modules) -# AdGuard Home — watches config directory -# + all other containers using inotify internally +# max_queued_events — max events buffered before the kernel drops them. Low value = +# events silently lost during high-activity bursts. Default 16384. # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock — prevents duplicate runs at array start -# Root check — sysctl writes require root -# validate_unraid — notify validated before use -# Silent on success — runs every boot, no noise when already correct -# Only warns on changes or failures +# Why 1M Watches +# Raising max_user_watches to 1048576 (1M) gives sufficient headroom for all +# containers combined — safe on 128GB RAM (~128MB kernel use for the pool). # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# INOTIFY_MAX_INSTANCES — default 1024 -# INOTIFY_MAX_WATCHES — default 1048576 (1M) -# INOTIFY_MAX_QUEUED_EVENTS — default 32768 +# Idempotent Per-Setting +# Each sysctl value is read before writing. Only changed if different from target — +# no-op on boots where limits are already correct. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root Required +# sysctl writes require root. +# +# Single Instance Lock +# acquire_lock prevents duplicate runs at array start. +# +# Silent on Success +# Runs every boot — no noise when already correct. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# INOTIFY_MAX_INSTANCES +# Max inotify fd objects per user. (default: 1024) +# +# INOTIFY_MAX_WATCHES +# Max watched files/dirs shared across all users and containers. (default: 1048576) +# +# INOTIFY_MAX_QUEUED_EVENTS +# Max events buffered before kernel drops them. (default: 32768) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# inotify_tuning.sh +# Apply inotify limits. No-op per setting if already at target. +# +# inotify_tuning.sh --dry-run +# Show current vs target for each limit. No sysctl writes. +# +# inotify_tuning.sh --status +# Show current vs target, active instance count, and top consumers by PID. +# +# inotify_tuning.sh --log +# Verbose output — show each sysctl check and result. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# inotify_tuning.sh — normal run (apply settings) -# inotify_tuning.sh --dry-run — show what would change -# inotify_tuning.sh --status — show current vs target values and top consumers -# inotify_tuning.sh --log — verbose output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/unRAID_Essentials/mover_stop.sh b/unRAID_Essentials/mover_stop.sh index c24feb2..d97ed4f 100755 --- a/unRAID_Essentials/mover_stop.sh +++ b/unRAID_Essentials/mover_stop.sh @@ -2,44 +2,74 @@ # ============================================================================================== # ================================= Mover Stop ================================================= # ============================================================================================== -# Safely stops the unRAID mover process with a warning before halting. -# Warns all logged-in users via wall message, waits the configured timeout, then stops. # -# ── WHEN TO USE ─────────────────────────────────────────────────────────────────────────────── -# - Before a planned reboot when mover is running mid-cycle -# - Before disk replacement or array operations that need mover stopped -# - Before rsync — mover and rsync simultaneously moving the same files causes corruption -# - Called automatically by maintenance scripts that need the mover stopped first +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Safely stops the unRAID mover with a wall warning, configurable timeout, and +# SIGTERM → SIGKILL sequence. Use before planned reboots, disk operations, or +# any operation where mover and rsync running simultaneously could corrupt files. +# Exits cleanly if mover is not running. # -# ── STOP SEQUENCE ───────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Stop Sequence # 1. Check if mover is running — exit cleanly if not -# 2. Broadcast wall warning to all logged-in users -# 3. Wait MOVER_STOP_TIMEOUT seconds (default 30) — gives active sessions a chance to note it -# 4. Send SIGTERM — mover can complete its current file operation before exiting -# 5. Wait 5 seconds for graceful exit -# 6. Verify stopped — if still running send SIGKILL (force) +# 2. Wall message to all logged-in terminal users +# 3. Wait MOVER_STOP_TIMEOUT seconds +# 4. SIGTERM — allows mover to finish its current file before stopping +# (no partial files — the mover completes what it is working on) +# 5. Wait 5 seconds → verify stopped +# 6. SIGKILL if still running — forced stop, partial files possible # 7. Final verify — error if still running after SIGKILL # -# ── SIGTERM vs SIGKILL ──────────────────────────────────────────────────────────────────────── -# SIGTERM first — allows mover to finish the file it is currently moving (no partial files). -# SIGKILL only as fallback — forces immediate stop (may leave partial files on cache or array). +# SIGTERM first because the mover has an opportunity to finish the file it is +# currently moving, leaving no partial copies on cache or array. SIGKILL is only +# used as a last resort and may leave a file split across cache and array. # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock — prevents concurrent stop attempts racing each other -# Root check — pkill on emhttp processes requires root -# validate_unraid — notify validated before use -# SIGTERM → verify → SIGKILL sequence — graceful then forced -# Final verify — confirms mover actually stopped -# Silent on clean — mover not running = log() only, no output ✅ +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# MOVER_STOP_TIMEOUT — seconds to warn users before stopping (default 30) +# Single Instance Lock +# acquire_lock prevents concurrent stop attempts racing each other. +# +# Root Required +# pkill on emhttp processes requires root. +# +# Final Verify +# Confirms mover is actually stopped after the kill sequence — errors if it +# is still running after SIGKILL. +# +# Silent When Clean +# Mover not running = log() only, no visible output. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# MOVER_STOP_TIMEOUT +# Seconds between wall warning and SIGTERM. (default: 30) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# mover_stop.sh +# Check if mover is running. If so, warn users and stop it. +# +# mover_stop.sh --dry-run +# Show mover state and what would happen. No signals sent. +# +# mover_stop.sh --status +# Show mover state (running, PID, start time). Then exit. +# +# mover_stop.sh --log +# Verbose output showing each step of the stop sequence. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# mover_stop.sh — stop mover with configured timeout -# mover_stop.sh --dry-run — show what would happen -# mover_stop.sh --status — show mover state -# mover_stop.sh --log — verbose output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/unRAID_Essentials/php_fpm_max_children.sh b/unRAID_Essentials/php_fpm_max_children.sh index 227b857..f8a4350 100755 --- a/unRAID_Essentials/php_fpm_max_children.sh +++ b/unRAID_Essentials/php_fpm_max_children.sh @@ -2,53 +2,91 @@ # ============================================================================================== # ============================= PHP-FPM Max Children =========================================== # ============================================================================================== -# Persistently sets PHP-FPM pm.max_children on unRAID to prevent WebGUI slowdowns. -# Run once at array start via ARRAY_START_SCRIPTS in master.conf. -# Idempotent — completely silent when value is already correct. # -# ── WHY THIS EXISTS ─────────────────────────────────────────────────────────────────────────── -# unRAID's WebGUI runs through PHP-FPM. The default pm.max_children is very low (4-8). -# Under load — multiple users, Docker operations, heavy dashboard usage — all PHP workers -# saturate and new requests queue. The WebGUI becomes slow or unresponsive. +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Raises PHP-FPM pm.max_children to prevent WebGUI slowdowns under load. Run +# once at array start via ARRAY_START_SCRIPTS. Idempotent — silent when the +# value is already correct, no restart on clean boot. # -# pm.max_children controls how many PHP worker processes can run simultaneously. -# Raising it allows the WebGUI to handle more concurrent requests without queuing. -# Too high: wastes RAM. Too low: WebGUI slowdowns. -# PHP_MAX_CHILDREN=250 is appropriate for 128GB — ~2MB per worker = ~500MB total. +# unRAID's WebGUI runs through PHP-FPM. The default pm.max_children is very +# low (4–8). Under load — multiple users, Docker operations, heavy dashboard +# usage — all PHP workers saturate and new requests queue. The WebGUI becomes +# slow or unresponsive. # -# ── WHY IDEMPOTENT ──────────────────────────────────────────────────────────────────────────── -# This runs at every array start. If the value is already correct there is nothing to do — -# no config write, no PHP-FPM restart. Restarting PHP-FPM unnecessarily disrupts active -# WebGUI sessions and is annoying on every boot. +# PHP_MAX_CHILDREN=250 is appropriate for 128GB RAM: ~2MB per worker = ~500MB +# total. Too high wastes RAM; too low causes slowdowns. # -# ── APPLY SEQUENCE ──────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Idempotent Content Check +# Reads the current pm.max_children value before writing. If already at +# target → silent exit, no PHP-FPM restart. Restarting PHP-FPM unnecessarily +# disrupts active WebGUI sessions on every boot. +# +# Pattern Match Before Write +# Verifies the sed pattern finds pm.max_children in the config before +# applying any change. Prevents silent failures where sed succeeds but +# writes nothing because the key was missing or commented out. +# +# Apply Sequence # 1. Read current pm.max_children from PHP_CONF -# 2. If already at target → exit silently (idempotent) +# 2. If already at target → exit silently # 3. Verify sed pattern matches before writing # 4. Apply sed replacement # 5. Restart PHP-FPM via rc.php-fpm # 6. Verify PHP-FPM process running after restart -# 7. Verify config file reflects target value +# 7. Read back config to confirm value applied # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# Root check — writing to system config requires root -# acquire_lock — prevents concurrent runs at array start -# Idempotent check — only restarts PHP-FPM when value actually changes -# Pattern match check — verifies sed found pm.max_children before writing -# Process verify — confirms PHP-FPM running after restart -# Config verify — reads back config to confirm value applied -# validate_unraid — notify validated before use -# Silent on correct — runs every boot, no noise when already set ✅ +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# PHP_MAX_CHILDREN — target pm.max_children value (default 250) -# PHP_CONF — path to PHP-FPM www.conf (default /etc/php83/php-fpm.d/www.conf) +# Root Required +# Writing to /etc/php83/ requires root. +# +# Single Instance Lock +# acquire_lock prevents concurrent runs at array start. +# +# Process Verify +# Confirms PHP-FPM running after restart — errors if it failed to start. +# +# Config Verify +# Reads back config after restart to confirm the value was actually applied. +# +# Silent on Success +# Runs every boot — no noise when already correct. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# PHP_MAX_CHILDREN +# Target pm.max_children value. (default: 250) +# +# PHP_CONF +# Path to PHP-FPM www.conf. (default: /etc/php83/php-fpm.d/www.conf) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# php_fpm_max_children.sh +# Read current value. Update and restart PHP-FPM only if changed. Silent if correct. +# +# php_fpm_max_children.sh --dry-run +# Show current vs target value. No config write or restart. +# +# php_fpm_max_children.sh --status +# Show current pm.max_children, target, and PHP-FPM process state. +# +# php_fpm_max_children.sh --log +# Verbose output showing idempotent check, config write, and restart result. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# php_fpm_max_children.sh — normal run (idempotent) -# php_fpm_max_children.sh --dry-run — show what would change -# php_fpm_max_children.sh --status — show current vs target and process state -# php_fpm_max_children.sh --log — verbose output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/unRAID_Essentials/resource_watchdog.sh b/unRAID_Essentials/resource_watchdog.sh index 0cb5188..20d411a 100755 --- a/unRAID_Essentials/resource_watchdog.sh +++ b/unRAID_Essentials/resource_watchdog.sh @@ -2,65 +2,105 @@ # ============================================================================================== # ================================= Resource Manager =========================================== # ============================================================================================== -# Pressure reduction layer — detects rising system load and reduces it intelligently. -# Called by watchdog_orchestrator.sh every minute — single-pass, not a continuous loop. # -# ── RESPONSIBILITY ──────────────────────────────────────────────────────────────────────────── -# Reduce system pressure before things break. NOT fixing broken containers (docker_watchdog) -# and NOT rebooting (system_watchdog). The middle layer that keeps the system comfortable. +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Pressure reduction layer — detects rising system load and reduces it before +# things break. Called by watchdog_orchestrator.sh every minute as a single- +# pass run. The middle layer between docker_watchdog.sh (fixes broken +# containers) and system_watchdog.sh (reboots). Does neither of those things. # -# "Pressure is rising — reduce load intelligently." +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── THREE-LEVEL PRESSURE RESPONSE ───────────────────────────────────────────────────────────── +# Three-Level Pressure Response # # Level 1 — SOFT (RAM < RW_RAM_SOFT_GB OR load > RW_LOAD_SOFT_MULTIPLIER × cores): -# Throttle SABnzbd download speed to RW_SABNZBD_SPEED_SOFT -# Throttle qBittorrent download to RW_QBIT_DL_SOFT KB/s +# Throttle SABnzbd download speed to RW_SABNZBD_SPEED_SOFT. +# Throttle qBittorrent download to RW_QBIT_DL_SOFT KB/s. # # Level 2 — MEDIUM (RAM < RW_RAM_MEDIUM_GB OR load > RW_LOAD_MEDIUM_MULTIPLIER × cores): -# Further throttle SABnzbd + qBittorrent to medium limits -# docker pause RW_PAUSE_CONTAINERS — suspend without losing state, instant reversible +# Further throttle SABnzbd + qBittorrent to medium limits. +# docker pause RW_PAUSE_CONTAINERS — suspend without losing state, instantly reversible. # # Level 3 — HARD (RAM < RW_RAM_HARD_GB): -# docker stop RW_STOP_CONTAINERS — optional/heavy services (games, LocalAI, etc.) -# Write mem_shutdown_active=true — signals docker_watchdog to defer container restarts +# docker stop RW_STOP_CONTAINERS — optional/heavy services (games, LocalAI, etc.). +# Write mem_shutdown_active=true → signals docker_watchdog to defer container restarts. # -# ── RECOVERY ────────────────────────────────────────────────────────────────────────────────── -# Pressure must stay below current action threshold for RW_RECOVER_CYCLES consecutive runs -# before restoring. De-escalates one level at a time to avoid re-triggering immediately. -# Level 3 de-escalation additionally requires RAM >= RW_RAM_RECOVER_GB before un-stopping. +# Recovery +# Pressure must stay below current threshold for RW_RECOVER_CYCLES consecutive +# runs before restoring. De-escalates one level at a time — prevents re-triggering +# immediately after recovery. Level 3 additionally requires RAM >= RW_RAM_RECOVER_GB +# before containers are un-stopped. # -# ── COORDINATION WITH DOCKER WATCHDOG ───────────────────────────────────────────────────────── -# At level 3: writes mem_shutdown_active=true to RW_STATE_FILE. +# Coordination with docker_watchdog.sh +# At level 3, writes mem_shutdown_active=true to RW_STATE_FILE. # docker_watchdog.sh reads this and defers all container restart logic. -# Cleared when pressure fully resolves and containers are restarted. -# This prevents docker_watchdog from restarting containers that RM just stopped to free RAM. +# Without this, docker_watchdog would immediately restart containers that were +# just stopped to free RAM — defeating the purpose of level 3. +# Cleared when pressure resolves and containers are restarted. # -# ── NOT RESPONSIBLE FOR ─────────────────────────────────────────────────────────────────────── -# Restarting broken containers — docker_watchdog.sh -# Rebooting the system — system_watchdog.sh -# Reacting to single data points — RW_RECOVER_CYCLES prevents flip-flopping +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# Root Required +# docker pause/stop require root. +# +# Single Instance Lock +# acquire_lock prevents concurrent runs from racing on state file writes. +# +# RW_CRITICAL_CONTAINERS +# Containers listed here are never paused or stopped regardless of pressure level. +# +# RW_ENABLED Flag +# Set RW_ENABLED=false to disable the entire script without removing it from +# the orchestrator schedule. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf # RW_ENABLED, RW_STATE_FILE # RW_RAM_SOFT_GB, RW_RAM_MEDIUM_GB, RW_RAM_HARD_GB, RW_RAM_RECOVER_GB # RW_LOAD_SOFT_MULTIPLIER, RW_LOAD_MEDIUM_MULTIPLIER # RW_RECOVER_CYCLES # RW_SABNZBD_ENABLED, RW_SABNZBD_SPEED_SOFT, RW_SABNZBD_SPEED_MEDIUM # RW_QBIT_ENABLED, RW_QBIT_DL_SOFT, RW_QBIT_DL_MEDIUM -# RW_CRITICAL_CONTAINERS — never paused or stopped regardless of pressure +# RW_CRITICAL_CONTAINERS — never paused or stopped regardless of pressure # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── -# HOST*_RW_PAUSE_CONTAINERS — docker pause at medium pressure (aliased by detect_hosts) -# HOST*_RW_STOP_CONTAINERS — docker stop at hard pressure (aliased by detect_hosts) +# master_host*.conf (aliased by detect_hosts()) +# HOST*_RW_PAUSE_CONTAINERS — docker pause at medium pressure +# HOST*_RW_STOP_CONTAINERS — docker stop at hard pressure # HOST*_SABNZBD_URL, HOST*_SABNZBD_API_KEY # HOST*_QBIT_URL, HOST*_QBIT_USERNAME, HOST*_QBIT_PASSWORD # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# resource_watchdog.sh — normal run (via watchdog_orchestrator.sh) -# resource_watchdog.sh --dry-run — show what would happen without acting -# resource_watchdog.sh --status — current pressure level and active actions -# resource_watchdog.sh --log — verbose per-check output +# ============================================================================================== +# STATE FILES +# ============================================================================================== +# +# RW_STATE_FILE +# Pressure level, recovery cycle count, stopped container list, and the +# mem_shutdown_active coordination flag read by docker_watchdog.sh. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# resource_watchdog.sh +# Single-pass pressure check. Apply actions if threshold crossed. Silent if below. +# +# resource_watchdog.sh --dry-run +# Show current pressure level and what would be throttled/paused/stopped. No changes. +# +# resource_watchdog.sh --status +# Show current pressure level, active actions, recovery cycle count, stopped containers. +# +# resource_watchdog.sh --log +# Verbose per-check output — show RAM, load, each threshold comparison, each action. +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/unRAID_Essentials/rsync_stop.sh b/unRAID_Essentials/rsync_stop.sh index 3032516..1c332a9 100755 --- a/unRAID_Essentials/rsync_stop.sh +++ b/unRAID_Essentials/rsync_stop.sh @@ -2,49 +2,87 @@ # ============================================================================================== # ================================= Rsync Stop ================================================= # ============================================================================================== -# Stops rsync intelligently on both local and remote servers. -# Auto-detects orchestrators and chooses the safest stop strategy automatically. # -# ── TWO MODES ───────────────────────────────────────────────────────────────────────────────── +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Stops rsync intelligently on both local and remote servers. Auto-detects +# running orchestrators and chooses the safest stop strategy. If an +# orchestrator is running, kills only the rsync subprocess so the orchestrator +# exits cleanly after finishing the current share. Use --full-stop to kill +# everything immediately. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Two Stop Modes # Default (smart): -# Detects if an orchestrator (daily/weekly/critical sync) is running -# If orchestrator found → kills rsync subprocess only -# Orchestrator sees rsync died → moves to next share or exits cleanly -# If no orchestrator → kills rsync directly (standalone rsync.sh run) -# Cleans stale lock files after kill -# Recovers containers left stopped by interrupted rsync (local only) +# Detects if an orchestrator (daily/weekly/critical sync) is running. +# If orchestrator found → kills rsync subprocess only. Orchestrator sees +# rsync exit → moves to next share or exits cleanly on its own. +# If no orchestrator → kills rsync directly (standalone rsync.sh run). +# Cleans stale lock files after kill. +# Recovers containers left stopped by interrupted rsync (local only). # # --full-stop (nuclear): -# Kills orchestrator first → then kills rsync -# Orchestrator will NOT continue to next share -# Use when: you need everything dead immediately +# Kills orchestrator first → then kills rsync. +# Orchestrator will NOT continue to next share. +# Use when everything needs to stop immediately. # -# ── REMOTE HANDLING ─────────────────────────────────────────────────────────────────────────── -# Both local and remote handled in one run via SSH. -# Remote containers left as-is — docker_watchdog.sh handles remote container recovery. -# If remote unreachable → skips remote cleanly, logs warning. +# Orchestrator Detection +# detect_rsync_parent() scans all lock files to find which running process +# has rsync as a descendant. No hardcoded list — works for any orchestrator. +# Returns "script_name:parent_pid" if found, empty if standalone. # -# ── ORCHESTRATOR DETECTION ──────────────────────────────────────────────────────────────────── -# detect_rsync_parent() scans all lock files to find which running process -# has rsync as a descendant. No hardcoded list — works for any orchestrator. -# Returns: "script_name:parent_pid" if found, empty if rsync running standalone. +# Remote Handling +# Both local and remote handled in one run via SSH. +# Remote containers left as-is — docker_watchdog.sh handles remote recovery. +# If remote unreachable → skips remote cleanly, logs warning. # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# Root check — pkill and docker require root -# acquire_lock — prevents concurrent stop attempts racing -# DOCKER_TIMEOUT — all docker calls protected against hung daemon -# SSH_TIMEOUT — all remote SSH calls timeout-protected -# SIGTERM → SIGKILL — graceful then forced for orchestrators -# Container recovery — restarts local containers left stopped by killed rsync -# validate_unraid_cmd — notify validated before use +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root Required +# pkill and docker require root. +# +# Single Instance Lock +# acquire_lock prevents concurrent stop attempts racing each other. +# +# Timeout Protection +# DOCKER_TIMEOUT (15s) on all docker calls — hung daemon doesn't block. +# SSH_TIMEOUT (15s) on all remote SSH calls. +# +# SIGTERM → SIGKILL Sequence +# Orchestrators receive SIGTERM first, SIGKILL only if still running after 2s. +# +# Container Recovery +# Restarts local containers left stopped by the killed rsync session. +# Remote containers deferred to docker_watchdog.sh. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# rsync_stop.sh +# Auto-detect orchestrator. Kill rsync-only or full-stop accordingly. +# +# rsync_stop.sh --full-stop +# Kill orchestrator first, then kill rsync. Nothing continues after this. +# +# rsync_stop.sh --rsync-only +# Skip container recovery. Used when called by other scripts that handle +# recovery themselves. +# +# rsync_stop.sh --dry-run +# Show what would be killed without killing anything. +# +# rsync_stop.sh --status +# Show local and remote rsync PIDs, running orchestrators, and lock files. +# +# rsync_stop.sh --full-stop --dry-run +# Preview full-stop sequence without making any changes. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# rsync_stop.sh — smart stop (auto-detect) -# rsync_stop.sh --full-stop — kill orchestrator + rsync -# rsync_stop.sh --rsync-only — skip container recovery (called by other scripts) -# rsync_stop.sh --dry-run — preview without changes -# rsync_stop.sh --status — show what's currently running -# rsync_stop.sh --full-stop --dry-run — preview full stop # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/unRAID_Essentials/server_reboot.sh b/unRAID_Essentials/server_reboot.sh index 1ece51f..dd0be54 100644 --- a/unRAID_Essentials/server_reboot.sh +++ b/unRAID_Essentials/server_reboot.sh @@ -2,55 +2,88 @@ # ============================================================================================== # ================================= Server Reboot ============================================== # ============================================================================================== -# Gracefully reboots the unRAID server with full pre-flight checks and clean shutdown sequence. -# Warns all users, checks for active processes, stops services, syncs disks, then reboots. # -# ── SHUTDOWN SEQUENCE ───────────────────────────────────────────────────────────────────────── -# 1. Pre-flight warnings — rsync, mover, active Emby sessions (warn not block) +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Gracefully reboots the unRAID server with pre-flight checks, user warnings, +# clean service shutdown, and disk sync. Use instead of raw /sbin/reboot — +# gives users warning time and ensures services stop cleanly before the kernel +# drops. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Shutdown Sequence +# 1. Pre-flight warnings — rsync, mover, active Emby sessions (warn, not block) # 2. Wall message to all logged-in terminal users -# 3. unRAID notification to dashboard -# 4. Wait REBOOT_SLEEP seconds (default 30) — gives users time to save work -# 5. Gracefully shutdown VMs (virsh shutdown each, then wait) +# 3. unRAID dashboard notification +# 4. Wait REBOOT_SLEEP seconds — users time to save work +# 5. Graceful VM shutdown via virsh — ACPI signal, then wait REBOOT_VM_WAIT # 6. Stop libvirt (VM Manager) # 7. Stop Docker service -# 8. Sync filesystem buffers to disk -# 9. Reboot +# 8. sync — filesystem buffers flushed to disk +# 9. /sbin/reboot # -# ── PRE-FLIGHT WARNINGS ─────────────────────────────────────────────────────────────────────── -# The following are warnings only — they do not block the reboot. You called this script, -# so you know what you're doing. The warnings give you context before the countdown starts. -# - rsync running → partial files possible if mid-transfer -# - mover running → files may be left on cache or array mid-move -# - Emby sessions → active streams/transcodes will be interrupted +# Pre-flight Warnings (informational — do not block) +# rsync running → partial files possible if mid-transfer +# mover running → files may be left mid-move on cache or array +# Emby sessions → active streams/transcodes interrupted +# Warnings do not block the reboot — you called this script, you know. # -# ── VM GRACEFUL SHUTDOWN ────────────────────────────────────────────────────────────────────── -# virsh shutdown sends ACPI power button signal to each VM — same as pressing power button. -# VM gets a chance to flush its own buffers and shutdown cleanly. -# Waits REBOOT_VM_WAIT seconds (default 30) for VMs to shut down before stopping libvirt. -# If VMs don't shut down in time libvirt stops anyway — system reboot takes priority. +# VM Graceful Shutdown +# virsh shutdown sends the ACPI power button signal — same as pressing the +# physical power button. VM gets a chance to flush buffers and shut down. +# After REBOOT_VM_WAIT seconds, libvirt stops anyway — reboot takes priority. # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID — used in wall message, notification, and summary. -# Critical on a two-server setup — wall and notifications show WHICH server is rebooting. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# Root check — reboot requires root -# acquire_lock — prevents concurrent reboot calls -# detect_hosts() — MY_ID in all user-facing messages -# validate_unraid_cmd — notify validated before use -# Graceful VM shutdown — VMs get clean ACPI signal before libvirt stops -# sync before reboot — filesystem buffers flushed to disk +# Root Required +# /sbin/reboot requires root. # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# REBOOT_SLEEP — seconds to warn users before starting shutdown sequence (default 30) -# REBOOT_VM_WAIT — seconds to wait for VMs to shut down gracefully (default 30) +# Single Instance Lock +# acquire_lock prevents concurrent reboot calls. +# +# Host Identity in All Messages +# detect_hosts() sets MY_ID — wall and notifications show which server is +# rebooting. Critical on a two-server setup. +# +# sync Before Reboot +# filesystem buffers flushed to disk before reboot command. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# REBOOT_SLEEP +# Seconds between warning and shutdown sequence start. (default: 30) +# +# REBOOT_VM_WAIT +# Seconds to wait for VMs to shut down gracefully. (default: 30) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# server_reboot.sh +# Run pre-flight, warn users, stop services cleanly, then reboot. +# +# server_reboot.sh --dry-run +# Walk through the entire shutdown sequence without stopping anything or rebooting. +# +# server_reboot.sh --status +# Show running processes that would be affected: rsync, mover, VMs, containers. +# +# server_reboot.sh --reason="maintenance" +# Include reason in wall message and notification. Defaults to "manual". +# +# server_reboot.sh --log +# Verbose output — show each step of the shutdown sequence. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# server_reboot.sh — reboot with 30s warning -# server_reboot.sh --dry-run — walk through sequence without rebooting -# server_reboot.sh --status — show running processes that would be affected -# server_reboot.sh --reason="maintenance" — log reason for reboot -# server_reboot.sh --log — verbose output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/unRAID_Essentials/system_watchdog.sh b/unRAID_Essentials/system_watchdog.sh index 008b616..a129dd2 100755 --- a/unRAID_Essentials/system_watchdog.sh +++ b/unRAID_Essentials/system_watchdog.sh @@ -2,71 +2,107 @@ # ============================================================================================== # ================================= System Watchdog ============================================ # ============================================================================================== -# Last line of defense — reboots the system cleanly if it is about to become unstable. -# Runs continuously as a background process — started by array_started.sh at array start. -# Works alongside docker_watchdog.sh which handles container-level healing first. # -# ── THREE-TIER RESPONSE SYSTEM ──────────────────────────────────────────────────────────────── +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Last line of defense — reboots the system cleanly if it is about to become +# unstable. Runs continuously as a background process started by +# array_started.sh at array start. Works alongside docker_watchdog.sh which +# handles container-level healing first. Only escalates to reboot when +# docker_watchdog.sh cannot resolve the condition. # -# TIER 1 — CRITICAL (bypass ALL strikes, reboot immediately) -# Docker daemon unresponsive — nothing can be healed, letting it run makes it worse -# rootfs at 99%+ — writes failing, SSH may stop, no recovery options +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Three-Tier Response System +# +# Tier 1 — CRITICAL (bypass all strikes, reboot immediately) +# Docker daemon unresponsive — nothing can be healed; running it longer makes it worse +# rootfs at 99%+ — writes failing; SSH may stop; no recovery options # Kernel oops/BUG in dmesg — kernel running with corrupted state # File descriptor exhaustion — new connections and processes failing silently # /boot read-only unexpectedly — state files and config writes silently failing # -# TIER 2 — URGENT (bypass strikes when OOM confirms active crisis) -# RAM < MEM_GB AND OOM kills >= OOM_LIMIT in this cycle -# Rationale: OOM kills at this rate means system is dying faster than watchdogs heal -# Without OOM confirmation → standard strike system applies +# Tier 2 — URGENT (bypass strikes when OOM confirms active crisis) +# RAM < MEM_GB AND OOM kills >= OOM_LIMIT in this cycle. +# OOM kills at this rate means the system is dying faster than watchdogs can heal. +# Without OOM confirmation → standard strike system applies. # -# TIER 3 — STANDARD (N consecutive failures → reboot) -# RAM tiers, load, CPU temp, zombies, /var/log, /tmp, containers, NIC, mdstat +# Tier 3 — STANDARD (N consecutive failures → reboot) +# RAM tiers, load, CPU temp, zombies, /var/log, /tmp, containers, NIC, mdstat. # -# ── RAM TIERS ───────────────────────────────────────────────────────────────────────────────── +# RAM Tiers # MEM_WARN_GB (10GB) — warn + notify only # MEM_SHUTDOWN_GB (6GB) — stop non-essential containers, wait for recovery -# MEM_GB (4GB) — strike system → reboot (bypass if OOM confirms) -# MEM_RECOVER_GB (30GB) — RAM must reach this before containers restart +# MEM_GB (4GB) — strike system → reboot (bypass with OOM confirmation) +# MEM_RECOVER_GB (30GB) — RAM must reach this before stopped containers restart # -# ── CONTAINER SHUTDOWN LOGIC ────────────────────────────────────────────────────────────────── -# At MEM_SHUTDOWN_GB: stop all containers NOT in SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED -# Excluded: NginxProxyManager, Authelia, Mariadb, Redis, Emby, Dispatcharr -# Stopped containers tracked in shutdown list — won't restart until RAM recovers -# Strike system prevents flip-flopping — shutdown only happens once per degradation event +# Container Shutdown Logic (at MEM_SHUTDOWN_GB) +# Stops all containers not in SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED. +# Stopped containers tracked in shutdown list — won't restart until RAM recovers. +# Strike system prevents flip-flopping — shutdown only once per degradation event. # -# ── OOM TRACKING ────────────────────────────────────────────────────────────────────────────── -# /proc/vmstat oom_kill counter — read each cycle, delta = kills this cycle -# Included in reboot message with process names from dmesg (diagnostic context) -# Bypass trigger: RAM critical AND kills this cycle >= SYS_WATCHDOG_OOM_LIMIT +# Abort Conditions (prevent reboot during sensitive operations) +# ZFS pool unhealthy, parity running, mover running — each toggleable. +# CRITICAL tier bypasses all abort conditions — imminent crash overrides data safety. # -# ── NEW CHECKS THIS VERSION ─────────────────────────────────────────────────────────────────── -# OOM rate tracking — delta from /proc/vmstat each cycle -# /boot read-only — write test on /boot each cycle -# Kernel oops detection — dmesg BUG/Oops count delta each cycle -# File descriptor exhaustion — /proc/sys/fs/file-nr utilisation -# /tmp usage — tmpfs fill detection with auto-clear attempt -# Array disk errors — mdstat error delta each cycle -# Runaway process — single process >N% CPU sustained (disabled by default) -# NIC state check — primary interface operstate -# sshd check — restart attempt before escalating +# Checks Run Every Cycle +# rootfs usage, /var/log, /tmp, free RAM, ZFS ARC, CPU temp, load avg, +# zombie processes, Docker daemon, OOM rate, /boot read-only, kernel oops, +# file descriptor exhaustion, array disk errors, NIC state, required containers. # -# ── EXISTING CHECKS ─────────────────────────────────────────────────────────────────────────── -# rootfs usage, /var/log, free RAM, ZFS ARC, CPU temp, load avg, -# zombie processes, Docker daemon, required containers from skip list +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── ABORT CONDITIONS ────────────────────────────────────────────────────────────────────────── -# ZFS pool unhealthy, parity running, mover running — toggleable -# CRITICAL tier bypasses abort conditions — imminent crash overrides data safety +# Root Required +# Reboot and container stop require root. # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# Full config under System Watchdog section — see master.conf for all vars +# Single Instance Lock +# acquire_lock prevents a second watchdog instance from starting. +# +# State File Verification +# All state files verified writable at startup — errors if any cannot be created. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf — System Watchdog section +# Full variable listing in master.conf. Key variables: +# +# SYS_WATCHDOG_REBOOT_WINDOW_HRS — reboot rate limit window (default: 2) +# SYS_WATCHDOG_MAX_REBOOTS — max reboots in window before giving up (default: 3) +# SYS_WATCHDOG_STRIKES — consecutive failures before reboot (default: 3) +# SYS_WATCHDOG_OOM_LIMIT — OOM kills/cycle to trigger URGENT bypass (default: 3) +# SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED — containers exempt from memory shutdown +# +# ============================================================================================== +# STATE FILES +# ============================================================================================== +# +# SYS_WATCHDOG_STATE_FILE — strike counters and cycle state +# SYS_WATCHDOG_REBOOT_LOG — reboot history for rate limiting +# SYS_WATCHDOG_FAILED_FILE — containers confirmed down for skip list integration +# SYS_WATCHDOG_OOM_FILE — OOM kill counter from previous cycle +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# system_watchdog.sh +# Start continuous monitoring loop. Runs until stopped or system reboots. +# +# system_watchdog.sh --dry-run +# Run detection logic without rebooting or stopping containers. +# +# system_watchdog.sh --status +# Show config, thresholds, current system state, and strike counts. +# +# system_watchdog.sh --log +# Verbose per-cycle output — show every check result and threshold comparison. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# system_watchdog.sh — normal start (continuous loop) -# system_watchdog.sh --dry-run — trigger detection without rebooting -# system_watchdog.sh --status — show config and thresholds -# system_watchdog.sh --log — verbose per-cycle output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/unRAID_Essentials/user_scripts_stop.sh b/unRAID_Essentials/user_scripts_stop.sh index 1991646..8065386 100755 --- a/unRAID_Essentials/user_scripts_stop.sh +++ b/unRAID_Essentials/user_scripts_stop.sh @@ -2,45 +2,70 @@ # ============================================================================================== # ============================= User Scripts Stop ============================================== # ============================================================================================== -# Stops all running User Script processes spawned by the unRAID User Scripts plugin. -# Identifies processes by their /tmp/user.scripts path signature. -# Shows script names not just PIDs — you know what's being stopped. # -# ── WHEN TO USE ─────────────────────────────────────────────────────────────────────────────── -# - Before a planned reboot when scripts are running mid-cycle -# - When a script is stuck and won't respond to the Abort button in the UI -# - Called automatically by server_reboot.sh as part of shutdown sequence -# - Emergency stop of all background ecosystem scripts +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Stops all running User Script processes spawned by the unRAID User Scripts +# plugin. Shows script names not just PIDs so you know what's being stopped. +# Called automatically by server_reboot.sh as part of the shutdown sequence, +# and useful directly when a script is stuck and won't respond to the UI. # -# ── HOW IT IDENTIFIES PROCESSES ─────────────────────────────────────────────────────────────── -# Scans /proc/*/cmdline for processes whose command line contains "/tmp/user.scripts". -# The unRAID User Scripts plugin stages all scripts in /tmp/user.scripts/ before execution. -# This is more reliable than process name matching which can vary. +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── STOP SEQUENCE PER PROCESS ───────────────────────────────────────────────────────────────── -# 1. Send SIGTERM — allows script to trap and clean up gracefully +# Process Identification +# Scans /proc/*/cmdline for processes whose command line contains +# "/tmp/user.scripts". The User Scripts plugin stages all scripts in +# /tmp/user.scripts/ before execution — more reliable than process name +# matching which can vary. +# +# Stop Sequence Per Process +# 1. Send SIGTERM — allows the script to trap and clean up gracefully # 2. Wait 5 seconds -# 3. Check if still running → SIGKILL (force) if SIGTERM ignored -# 4. Verify dead after SIGKILL +# 3. If still running → SIGKILL (force) +# 4. Verify dead after SIGKILL — error if still running # -# ── SELF-EXCLUSION ──────────────────────────────────────────────────────────────────────────── -# If this script itself is run via the User Scripts plugin it would find its own PID. -# Self-exclusion prevents this script from killing itself mid-execution. +# Self-Exclusion +# If this script is run via the User Scripts plugin it would find its own +# PID in the scan. Self-exclusion by PID prevents killing its own process +# tree mid-execution. # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# Root check — kill requires root for other users' processes -# acquire_lock — prevents concurrent stop attempts -# Self-exclusion — never kills its own process tree -# SIGTERM → SIGKILL — graceful then forced -# Verify after kill — confirms processes are actually dead -# validate_unraid_cmd — notify validated before use -# Silent when clean — no processes running = log() only ✅ +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root Required +# kill requires root for other users' processes. +# +# Single Instance Lock +# acquire_lock prevents concurrent stop attempts. +# +# SIGTERM → SIGKILL Sequence +# Graceful first. Forced only if SIGTERM ignored after 5 seconds. +# +# Post-Kill Verify +# Confirms each process is actually dead. Errors and notifies if unkillable. +# +# Silent When Clean +# No processes running = log() only, no visible output. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# user_scripts_stop.sh +# Find and stop all User Script processes. Silent if none running. +# +# user_scripts_stop.sh --dry-run +# Show which processes would be stopped, with names and runtimes. No kills. +# +# user_scripts_stop.sh --status +# Show currently running User Script processes with names and elapsed time. +# +# user_scripts_stop.sh --log +# Verbose output — show each process found, each signal sent, each result. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# user_scripts_stop.sh — stop all user scripts -# user_scripts_stop.sh --dry-run — show what would be stopped -# user_scripts_stop.sh --status — show currently running user scripts -# user_scripts_stop.sh --log — verbose output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/unRAID_Essentials/webgui_restart.sh b/unRAID_Essentials/webgui_restart.sh index 4fd9991..4a2e708 100644 --- a/unRAID_Essentials/webgui_restart.sh +++ b/unRAID_Essentials/webgui_restart.sh @@ -2,57 +2,92 @@ # ============================================================================================== # ================================= WebGUI Watchdog ============================================ # ============================================================================================== -# Monitors the unRAID WebGUI and restarts services if unresponsive. -# Uses a three-step escalating strategy — lightest fix first, heaviest last. -# Run every 5-10 minutes via User Scripts plugin. -# Silent when healthy — only produces output when something needs fixing. # -# ── ESCALATION PATH ─────────────────────────────────────────────────────────────────────────── -# Check WebGUI → responding → log() + exit 0 (completely silent ✅) +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Monitors the unRAID WebGUI and restarts services if unresponsive. Uses a +# three-step escalating strategy — lightest fix first, heaviest last. Run +# every 5–10 minutes via the User Scripts plugin. Silent when healthy. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Escalation Path +# WebGUI responding → log() + exit 0 (completely silent ✅) # # Not responding: -# Step 1 — Restart nginx -# Lightest fix — handles most transient WebGUI failures -# nginx crash, worker stuck, connection timeout -# Wait WEBGUI_NGINX_WAIT seconds → recheck +# Step 1 — nginx restart +# Lightest fix — handles most transient WebGUI failures: +# nginx crash, worker stuck, connection timeout. +# Wait WEBGUI_NGINX_WAIT seconds → recheck. # -# Step 2 — Restart php-fpm -# WebGUI runs through PHP-FPM — worker exhaustion causes silent failure -# php-fpm workers saturated → new requests queue → WebGUI appears frozen -# system_tuning_monitor.sh tracks usage — this recovers it -# Wait WEBGUI_PHP_WAIT seconds → recheck +# Step 2 — php-fpm restart +# WebGUI runs through PHP-FPM. Worker exhaustion causes silent +# failure — requests queue and the WebGUI appears frozen. +# Wait WEBGUI_PHP_WAIT seconds → recheck. # -# Step 3 — Restart emhttp -# Heaviest fix — emhttp is the unRAID management daemon -# Array, Docker, shares stay running — only WebGUI management restarts -# Takes longer to recover — WEBGUI_EMHTTP_WAIT gives it time -# Wait WEBGUI_EMHTTP_WAIT seconds → recheck +# Step 3 — emhttp restart +# Heaviest fix. emhttp is the unRAID management daemon. +# Array, Docker, and shares stay running — only WebGUI +# management restarts. Takes longer — WEBGUI_EMHTTP_WAIT. +# Wait WEBGUI_EMHTTP_WAIT seconds → recheck. # -# All three failed → notify warning, manual intervention needed → exit 1 +# All three failed → notify, manual intervention needed → exit 1. # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID — used in all notifications and summary. -# Critical on two-server setup — which server's WebGUI failed? +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# acquire_lock — prevents concurrent runs double-restarting services -# detect_hosts() — MY_ID in all notifications -# Process verify — pgrep check after each service restart -# Silent healthy — completely silent on healthy cycle ✅ -# validate_unraid_cmd — notify validated before use +# Root Required +# Service restart commands require root. # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# WEBGUI_URL — URL to check (default http://localhost) -# WEBGUI_TIMEOUT — curl timeout in seconds (default 5) -# WEBGUI_NGINX_WAIT — seconds after nginx restart before rechecking (default 15) -# WEBGUI_PHP_WAIT — seconds after php-fpm restart before rechecking (default 10) -# WEBGUI_EMHTTP_WAIT — seconds after emhttp restart before rechecking (default 30) +# Single Instance Lock +# acquire_lock prevents concurrent runs double-restarting services. +# +# Process Verify After Each Restart +# pgrep check after each rc.* command — errors if process not running. +# +# Silent When Healthy +# Completely silent on healthy cycles. Only produces output when recovering. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# WEBGUI_URL +# URL to check for WebGUI response. (default: http://localhost) +# +# WEBGUI_TIMEOUT +# curl timeout in seconds. (default: 5) +# +# WEBGUI_NGINX_WAIT +# Seconds after nginx restart before rechecking. (default: 15) +# +# WEBGUI_PHP_WAIT +# Seconds after php-fpm restart before rechecking. (default: 10) +# +# WEBGUI_EMHTTP_WAIT +# Seconds after emhttp restart before rechecking. (default: 30) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# webgui_restart.sh +# Check WebGUI. Escalate through nginx → php-fpm → emhttp if unresponsive. +# +# webgui_restart.sh --dry-run +# Show which services would be restarted. No restarts, no waits. +# +# webgui_restart.sh --status +# Show current WebGUI response state and nginx/php-fpm/emhttp process states. +# +# webgui_restart.sh --log +# Verbose output — show each check, each restart attempt, each wait. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# webgui_restart.sh — check and recover if needed -# webgui_restart.sh --dry-run — show what would be restarted -# webgui_restart.sh --status — show current WebGUI and service states -# webgui_restart.sh --log — verbose output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"