#!/bin/bash # ============================================================================================== # ============================= USER SCRIPTS MASTER TEMPLATE =================================== # ============================================================================================== # # Paste this file into a User Script entry. Uncomment ONE script block and set the schedule. # Every script in the ecosystem is listed here — from the orchestrators that run it all, # down to the individual scripts you can run standalone for specific tasks. # # ── HOW THIS ECOSYSTEM WORKS ────────────────────────────────────────────────────────────────── # When used as intended, only a handful of orchestrators need to be scheduled. # The orchestrators handle everything else in the correct order — they call child scripts, # manage timing and dependencies, track pass/fail, and send one notification per window. # # You do not need to schedule every script below. The orchestrators cover it all: # # array_started.sh at array start — launches ALL startup scripts in order # array_stopping.sh at array stop — graceful shutdown sequence # watchdog_orchestrator.sh every minute — resource + docker + system watchdog # transcode_management.sh every 7 min — cleanup then manager (order critical) # critical_sync_maintenance.sh every 30 min — auth + Emby dirty sync + partnership # intermediate_sync_maintenance.sh every 4 hours — arr library sync + artwork fetch # daily_sync_maintenance.sh 1am daily — git + rsync + media + restart # rsync.sh --profile=emby-fallback every 30 min — Emby watch state dirty sync # weekly_sync_maintenance.sh 2:30am Sunday — clean sync + image updates # sunday_morning_coffee_report.sh 7am Sunday — full weekly digest # weekly_health_digest.sh 8am daily — profile-controlled health notification # system_tuning_monitor.sh every 6 hours — inotify + php-fpm trend tracking # monthly_maintenance.sh 15th monthly — uptime-gated heavy tasks (ZFS scrub, SMART long test) # # ── INDIVIDUAL SCRIPTS ──────────────────────────────────────────────────────────────────────── # Every child script is also listed below, individually. # If you only want part of the ecosystem — say, just the arr cleanup scripts, or just # the transcode manager — uncomment only those blocks and schedule them directly. # All scripts are fully self-contained and work standalone. # # ── ALL SCRIPTS SUPPORT ─────────────────────────────────────────────────────────────────────── # --dry-run run without making any changes — shows exactly what would happen # --status show current configuration and state, then exit # --log verbose output — every decision logged # # ── PATHS ───────────────────────────────────────────────────────────────────────────────────── # Scripts: /mnt/user/appdata/unraid_scripts/ # Configuration: /mnt/user/appdata/unraid_scripts/master.conf # Per-host: /mnt/user/appdata/unraid_scripts/host1.conf (or host2.conf) # Library: /mnt/user/appdata/unraid_scripts/common.sh # # ── PLUGIN SETTINGS (apply to every entry) ──────────────────────────────────────────────────── # Run in background: YES — required for all scripts, even one-shots # Run as root: YES — all scripts require root # # ── CHANGELOG ───────────────────────────────────────────────────────────────────────────────── # v1.0 — Initial template # v1.1 — MAX_RSYNC_PROCS removed, --status flag, full directory tree, changelog # v1.2 — Docker Essentials, Media, Transcodes, System Watchdog added # v1.3 — Failover script added # v1.4 — WebGUI watchdog, ZFS snapshot, Docker network connect, schedules section # v1.5 — media_management.sh orchestrator, corrected filenames, git repo section # v1.6 — lidarr_cleanup.sh, sonarr_cleanup.sh, radarr_cleanup.sh added # v1.7 — cert_monitor.sh added # v1.8 — Monitors/ folder with all monitoring scripts # v1.9 — transcode_management.sh added to Orchestrators/ # v2.0 — Major restructure: array_started.sh single entry point, daily/weekly orchestrators, # sunday_morning_coffee_report.sh, docker_network_connect combined, # inotify_tuning.sh, arr path translation, version checking, nuclear mode flags # v2.1 — critical_sync_maintenance.sh, Partnership/ folder, system_tuning_monitor.sh, # arrs_failed_stalled_recovery.sh, rsync_stop.sh orchestrator-aware, # WEEKLY_SYNC_UPDATES, temp thresholds from dynamix.cfg # v2.2 — Orchestrators at top with full inline detail. All child scripts listed # individually below for standalone use. Inline descriptions on every entry. # Correct schedules verified against script headers. sunday_morning_coffee_report # and rsync_emby_failover added as separate scheduled entries. # v2.3 — intermediate_sync_maintenance.sh new 4-hour orchestrator (arr_sync + artwork). # docker_update.sh + docker_update_remaining.sh: daily/weekly image pulls. # downloaders_reset.sh: slskd + SABnzbd + qBittorrent maintenance. # arr_sync.sh: bidirectional Lidarr/Sonarr/Radarr library mesh. # lidarr_missing_art.sh: fetch missing album/artist artwork. # radarr_tmdb_removed.sh + sonarr_tvdb_removed.sh: dropped-entry cleanup. # Initial_run/ section: ssh_setup.sh + partnership_onboard.sh. # Moved: arrs_failed_stalled_recovery → Media/, fallback_state_reset + # watchdog_skip_list_manager → Tools/. Renamed: partnership_manager.sh. # v2.4 — array_stopping.sh: At Stopping of Array orchestrator (graceful shutdown sequence). # watchdog_orchestrator.sh: every-minute cron replacing continuous loops in # system_watchdog + docker_watchdog (both now single-pass, no longer started by # array_started.sh — remove from ARRAY_START_SCRIPTS in master.conf). # resource_watchdog.sh: three-level pressure reduction (soft/medium/hard). # docker_container_stop.sh: sequential verified container stop for shutdown. # Path fixes: git_pull_execute.sh (root), continuous_scripts_status.sh (Tools/). # v2.5 — fallback.sh --stop: added to ARRAY_STOP_SCRIPTS as step 2 (after user_scripts_stop, # before rsync_stop) — fallback runs as a bare subprocess, not via /tmp/user.scripts, # so it is not caught by user_scripts_stop.sh. # v2.6 — monthly_maintenance.sh: uptime-gated orchestrator for heavy monthly tasks (ZFS scrub, # SMART long test). Schedule 0 0 15 * * — 15th of each month at midnight. # storage_watchdog.sh + network_watchdog.sh: pool growth + connectivity watchdogs, # called via SYSTEM_WATCHDOG_SCRIPTS by system_watchdog.sh (watchdog chain, no cron). # webgui_watchdog.sh moved from standalone cron to SYSTEM_WATCHDOG_SCRIPTS chain. # arrs_failed_stalled_recovery.sh moved to INTERMEDIATE_MAINTENANCE_SCRIPTS (every 4hr). # clear_logs.sh moved to WEEKLY_MAINTENANCE_SCRIPTS (Sunday weekly window). # Sunday monitor block replaced by COFFEE_REPORT_SCRIPTS (sunday_morning_coffee_report). # Playback-aware discovery (lidarr/radarr/sonarr) added to WEEKLY_MAINTENANCE_SCRIPTS. # Partnership/: gitea_ssh_setup.sh, partnership_offboard.sh, partnership_transfer.sh. # Tools/: emby_to_lidarr_sync.sh, emby_to_radarr_sync.sh, emby_to_sonarr_sync.sh, # ramdisk_stop.sh. Monitors/: mesh_monitor.sh. Path fix: Initial_run/ → Partnership/. # ============================================================================================== # ============================================================================================== # ██ ORCHESTRATORS — RUN THE WHOLE ECOSYSTEM ██████████████████████████████████████████████████ # ============================================================================================== # # These are the only entries you need to schedule when using the ecosystem as intended. # Each orchestrator calls child scripts in the correct order with correct timing. # Configure what runs via master.conf — never by editing the orchestrator scripts. # # ============================================================================================== # ── ARRAY START ─────────────────────────────────────────────────────────────────────────────── # Schedule: At Startup of Array # Background: YES — required (starts continuous background processes) # # The ONLY "At Startup of Array" entry needed. Reads ARRAY_START_SCRIPTS from master.conf # and launches each in order. Detects one-shot vs continuous scripts automatically. # Auto-fixes execute permissions on any script missing them. # # Launches in order: # inotify_tuning.sh raise kernel inotify limits BEFORE containers inherit them # docker_syslog_filter.sh suppress Docker veth noise BEFORE containers create interfaces # php_fpm_max_children.sh set PHP-FPM workers to 250 BEFORE WebGUI serves requests # ramdisk_setup.sh create tmpfs ramdisk + transcode symlink BEFORE Emby starts # docker_network_connect.sh ensure custom networks exist and containers are connected # fallback.sh [continuous] mutual fallback state machine # # NOTE: system_watchdog.sh and docker_watchdog.sh are no longer started by array_started.sh. # They are now single-pass scripts called every minute by watchdog_orchestrator.sh (see below). # Remove them from ARRAY_START_SCRIPTS in master.conf if present. # # bash /mnt/user/appdata/unraid_scripts/Orchestrators/array_started.sh # ── ARRAY STOPPING ──────────────────────────────────────────────────────────────────────────── # Schedule: At Stopping of Array # Background: YES # # The ONLY "At Stopping of Array" entry needed. Reads ARRAY_STOP_SCRIPTS from master.conf # and runs each sequentially — each must complete before the next starts. # # Runs in order: # user_scripts_stop.sh stop background user scripts first (prevents new operations) # fallback.sh --stop gracefully stop fallback (not caught by user_scripts_stop) # rsync_stop.sh --rsync-only kill rsync without triggering container recovery # mover_stop.sh stop mover after rsync (both write to same paths) # docker_container_stop.sh stop all containers one-by-one with verification # # A failed step is logged and notified but does not prevent remaining steps from running. # Unlike array_started.sh, all steps run in the foreground — each confirmed done before next. # # bash /mnt/user/appdata/unraid_scripts/Orchestrators/array_stopping.sh # bash /mnt/user/appdata/unraid_scripts/Orchestrators/array_stopping.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Orchestrators/array_stopping.sh --status # ── WATCHDOG ORCHESTRATOR ───────────────────────────────────────────────────────────────────── # Schedule: * * * * * (every minute) # Background: YES # # Runs the three-layer watchdog system in the correct sequence each cycle. Replaces the # continuous loops that used to run inside system_watchdog.sh and docker_watchdog.sh — # those scripts are now single-pass and called here. # # Execution order (non-negotiable): # 1. resource_watchdog.sh free RAM and reduce CPU pressure BEFORE healing attempts # 2. docker_watchdog.sh restart containers with pressure already reduced # 3. system_watchdog.sh storage + WebGUI + network checks via SYSTEM_WATCHDOG_SCRIPTS: # storage_watchdog.sh pool growth + runaway log detection # webgui_watchdog.sh WebGUI availability (nginx → php-fpm → emhttp) # network_watchdog.sh internet, DDNS, Tailscale, NPM proxy # 4. stability_watchdog.sh reboot if all else failed — last line of defense # # Startup grace: no action until uptime >= WATCHDOG_STARTUP_GRACE (600s). Exits early # so all three sub-scripts don't each log startup-grace skips on every minute. # Overlap protection: acquire_lock() exits immediately if a prior cycle is still running — # prevents pile-up when a restart attempt or daemon check takes longer than 60 seconds. # # bash /mnt/user/appdata/unraid_scripts/Orchestrators/watchdog_orchestrator.sh # bash /mnt/user/appdata/unraid_scripts/Orchestrators/watchdog_orchestrator.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Orchestrators/watchdog_orchestrator.sh --status # ── TRANSCODE MANAGEMENT ────────────────────────────────────────────────────────────────────── # Schedule: */7 * * * * (every 7 minutes) # Background: YES # # Cleanup first, then manager — order is non-negotiable. Stale segment files are removed # BEFORE the manager checks ramdisk usage, so threshold decisions are based on real active # session load, not inflated counts from sessions that already ended. # # transcode_cleanup.sh: # Remove HLS segment files older than TRANSCODE_MAX_AGE (20min) not open by any process. # One lsof call per location (not one per file — critical on busy Live TV systems). # transcoding-temp directory NEVER deleted even when empty (prevents SSD session drift). # After cleanup: triggers flip-back to ramdisk if usage dropped below RAMDISK_LOW_GB. # # transcode_manager.sh: # Check ramdisk usage vs RAMDISK_WARN_GB. Flip /mnt/ram-transcode symlink if needed. # Hysteresis: WARN→SSD flip, usage must drop to LOW before flipping back (prevents oscillation). # Safety every cycle: missing symlink → recreate, ramdisk gone → auto-flip SSD, # missing transcoding-temp → recreate immediately. # Show active Emby sessions with play method. Write daily stats to TRANSCODE_DAILY_LOG. # # NOTE: Emby Extra Parameters MUST include: # --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared # Without shared propagation, symlink flips are silently ignored after the first flip. # # bash /mnt/user/appdata/unraid_scripts/Orchestrators/transcode_management.sh # ── CRITICAL SYNC MAINTENANCE ───────────────────────────────────────────────────────────────── # Schedule: */30 * * * * (every 30 minutes) # Background: YES # # Keeps fallback-critical data current on HOST2. 30 minutes is the maximum staleness # for the auth stack and Emby watch states. Run on HOST1 — bidirectional via detect_hosts(). # # rsync Critical-Data: # Auth stack to HOST2: NPM proxy rules, TLS certs, LLDAP user accounts, # Authelia config and policies. HOST2 auth always within 30min of HOST1. # # rsync Emby dirty sync: # Watch states, user activity, library delta — Emby stays running on both sides. # WAL/SHM files excluded (unsafe to copy mid-write). HOST2 Emby restarts after sync # to pick up config changes. # # partnership --check: # Reads remote state file. Increments offline counter on failed sync. # Triggers auto-offboard at PARTNERSHIP_OFFLINE_THRESHOLD (30 days) unreachable. # # Silent on clean cycles: runs 48 times per day, no output when healthy. # # bash /mnt/user/appdata/unraid_scripts/Orchestrators/critical_sync_maintenance.sh # ── INTERMEDIATE SYNC MAINTENANCE ──────────────────────────────────────────────────────────── # Schedule: 0 */4 * * * (every 4 hours) # Background: YES # # Arr library reconciliation, artwork fetching, and optional mid-day rsync. # Run on BOTH servers independently — each handles its own arr types. # # Execution order: # 1. arr_sync.sh bidirectional Lidarr/Sonarr/Radarr mesh — all nodes agree # Union model: if any node tracks it, all nodes get it. # Remote API keys read via SSH on the fly — never stored. # 2. Rsync window (optional) INTERMEDIATE_SYNC_SHARES — empty by default, skipped # Add shares here if a subset needs mid-day propagation. # 3. INTERMEDIATE_MAINTENANCE_SCRIPTS — 4-hour jobs (add any script that needs sub-daily cadence). # arrs_failed_stalled_recovery.sh — blocklist + re-search failed # and stalled arr imports. Every 4 hours. Configured in master.conf. # lidarr_missing_art.sh — idempotent artwork fetch. Skips existing files. # # Why separate from daily: arr libraries need to converge more than once a day. If a remote # node adds something at 2am, waiting 23 hours means both arrs search for content already owned. # 4-hour cycles close that gap. Artwork and failed imports catch up within 4 hours. # # Silent on success — runs 6x/day, only failures produce notification. # # bash /mnt/user/appdata/unraid_scripts/Orchestrators/intermediate_sync_maintenance.sh # ── DAILY SYNC MAINTENANCE ──────────────────────────────────────────────────────────────────── # Schedule: 0 1 * * * (1am daily — runs on BOTH servers independently) # Background: YES # # Full nightly maintenance window. detect_hosts() sets correct sync direction per server. # HOST1 pushes: Movies, Tv_Shows, Music, Kids content, Sports, stand-up comedy → HOST2 # HOST2 pushes: Anime_Shows, Anime_Movies → HOST1 # Personal encrypted shares: appended after standard shares — remote receives ciphertext only. # # Execution order (order is not arbitrary — each step depends on the previous): # 1. git_pull_execute.sh pull latest scripts FIRST — everything below runs on current code # 2. rsync DAILY_SYNC_SHARES push this server's truth shares to other server # Temp exit codes: 1=WARN (skip share, continue), 2=CRIT (abort all syncs, notify) # 3. media_shares_permissions.sh apply nobody:users 755/664 — arr cleanup depends on this # 4. media_cleaner.sh anime remove scene junk BEFORE orphan detection (cleaner results) # media_cleaner.sh media # 5. lidarr_cleanup.sh delete orphaned media — 7 safety checks before any deletion # sonarr_cleanup.sh notify_emby_scan() after each — ghost entries cleared immediately # radarr_cleanup.sh # 6. docker_update.sh pull latest images for DAILY_RESTART_CONTAINERS (no downtime) # 7. docker_daily_restart.sh restart: NPM, Authelia, Dispatcharr, Dispatcharr-Basic, ErsatzTV-Emby # # Notify: silent on success. One failure notification with job list on any failure. # # bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh # ── RSYNC EMBY FALLBACK ─────────────────────────────────────────────────────────────────────── # Schedule: */30 * * * * (every 30 minutes) # Background: YES # # Keeps HOST2 Emby within 30 minutes of HOST1 on watch states and library changes. # Direct rsync.sh call (not an orchestrator). Emby stays running on both sides. # # Syncs: users.db, library.db, authentication.db, config/ # Skips: *.wal *.shm (unsafe mid-write), transcodes/, logs/, cache/ (volatile/local only) # After: HOST2 Emby restarts to pick up any config changes from the sync. # Result: if HOST1 fails, users resume from at most 30 minutes stale. # # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ # /mnt/user/Media_Server/Emby --profile=emby-fallback # ── WEEKLY SYNC MAINTENANCE ─────────────────────────────────────────────────────────────────── # Schedule: 30 2 * * 0 (Sunday 2:30am — 4.5 hours before coffee report) # Background: YES # # Full Sunday maintenance window. Containers stop for clean sync. Image updates happen # at zero extra downtime since containers are already stopped for the rsync anyway. # # Execution order: # 1. Pre-flight checks connectivity, remote Docker daemon, remote rootfs space # 2. Stop local containers Emby + auth stack on this server # 3. Stop remote containers Emby + auth stack on remote via SSH # 4. Pull images locally if WEEKLY_SYNC_UPDATES=true (already stopped = free downtime) # 5. Pull images remotely if WEEKLY_SYNC_UPDATES_REMOTE=true (SSH to remote) # 6. rsync WEEKLY_SYNC_SHARES # Emby full clean mirror — databases checkpointed on shutdown, all config included # Critical-Data full clean auth stack mirror # 7. Start remote containers dependency order, new image, verify each container up # 8. Start local containers dependency order, new image, verify each container up # 9. docker_weekly_restart.sh restart less-critical services: NextCloud, AdGuard, Immich # 10. docker_update_remaining.sh pull latest images for all containers not in daily/weekly lists # 11. WEEKLY_MAINTENANCE_SCRIPTS — clear_logs.sh + playback_aware_*.sh discovery scripts. # playback_aware_lidarr_discovery.sh: 0–5 music adds/week # based on your Emby play history + Last.fm similar artists. # playback_aware_radarr_discovery.sh: 0–5 movie adds/week # based on recently watched + TMDB recommendations. # playback_aware_sonarr_discovery.sh: 0–3 TV adds/week # based on recently watched + TMDB recommendations. # # Why weekly not nightly for Emby: HOST2 builds warm thumbnail cache throughout the week. # Nightly sync resets that cache — cold loads every morning. Weekly: cache warm 6 days, # resets Sunday night while users sleep. # # bash /mnt/user/appdata/unraid_scripts/Orchestrators/weekly_sync_maintenance.sh # ── MONTHLY MAINTENANCE ─────────────────────────────────────────────────────────────────────── # Schedule: 0 0 15 * * (15th of each month at midnight) # Background: YES # # Runs heavy tasks that require a long-stable, settled system. Fires only when BOTH gates pass: # 1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS (default: 30 days) # 2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS ago (default: 30 days) # # If either gate is not met on the 15th, the run is skipped until next month's 15th. # A reboot between the 14th and 15th will defer maintenance — which is intentional; # heavy tasks (ZFS scrub, SMART long test) should not run on a freshly rebooted system. # # Configure via master.conf MONTHLY_MAINTENANCE_SCRIPTS. Scripts planned but not yet built: # zfs_pool_scrub.sh ZFS pool integrity scrub (Tools/) # smart_long_test.sh SMART extended drive health test (Tools/) # # bash /mnt/user/appdata/unraid_scripts/Orchestrators/monthly_maintenance.sh # bash /mnt/user/appdata/unraid_scripts/Orchestrators/monthly_maintenance.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Orchestrators/monthly_maintenance.sh --status # bash /mnt/user/appdata/unraid_scripts/Orchestrators/monthly_maintenance.sh --force # ── SUNDAY MORNING COFFEE REPORT ───────────────────────────────────────────────────────────── # Schedule: 0 7 * * 0 (Sunday 7am — after 2:30am maintenance completes) # Background: YES # # Full weekly system overview. Reads data from the week — state files, logs, APIs, live checks. # # Sections: # System uptime, memory, boot/cache drive usage, watchdog-triggered reboots this week # Array disk count, parity status, ZFS pool health, all drive temperatures # Transcodes weekly ramdisk peak, flip count, ramdisk vs SSD session split # Media arr orphan stats (count removed, GB freed), current queue depth per arr # Rsync weekly transfer totals, per-profile breakdown, failures, large transfers # Watchdog system + docker watchdog running state, active strikes, skip list # Security SSL cert days remaining per domain (live TLS connection — not NPM's API) # Emby version, active streams now (direct vs transcode), weekly play count, top users # Health inotify peak/avg/warnings, php-fpm peak/avg, SMART pass/fail, git sync state # Issues anything requiring attention collected from above sections # # Footer: ✅ enjoy your Sunday ☕ OR ⚠️ N issue(s) need attention # Delivery: unRAID notification + Discord webhook (if DISCORD_WEBHOOK configured) # # bash /mnt/user/appdata/unraid_scripts/Orchestrators/sunday_morning_coffee_report.sh # ── WEEKLY HEALTH DIGEST ───────────────────────────────────────────────────────────────────── # Schedule: 0 8 * * * (8am daily — DIGEST_PROFILE controls when it actually notifies) # Background: YES # # Reads ecosystem state files and compiles a digest notification. Read-only, writes nothing. # Schedule never changes — switch behaviour by changing DIGEST_PROFILE in master.conf only. # # DIGEST_PROFILE options (master.conf): # always notify every run regardless — daily digest even when healthy # smart notify only when something worth reporting is found: # active watchdog strikes, non-NORMAL failover, cert within CERT_WARN_DAYS, # bandwidth event exceeded BANDWIDTH_WARN_GB # weekly notify once per week on DIGEST_DAY (Sunday), silent all other days # # Data sources: fallback state, watchdog strike files, container skip list, # bandwidth log, transcode daily log, cert expiry (live openssl check) # # bash /mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh # ── SYSTEM TUNING MONITOR ──────────────────────────────────────────────────────────────────── # Schedule: 0 */6 * * * (every 6 hours) # Background: YES # # Snapshots inotify and PHP-FPM utilisation. Writes one bounded log entry. # sunday_morning_coffee_report.sh reads this log for weekly peak/avg/warning counts. # # inotify: current instances vs kernel limit. Warns above INOTIFY_WARN_PCT (80%). # Exhaustion symptom: downloads complete but arrs don't detect them, Live TV stutters, # library updates stop — no obvious error message anywhere. # # php-fpm: active workers vs PHP_MAX_CHILDREN. Warns above PHP_FPM_WARN_PCT (80%). # Exhaustion symptom: WebGUI slows, settings saves hang, container UI starts timeout. # # Log: TUNING_MONITOR_LOG — bounded, trimmed to TUNING_LOG_RETENTION days on each write. # # bash /mnt/user/appdata/unraid_scripts/Monitors/system_tuning_monitor.sh # ── SUNDAY MONITOR BLOCK ───────────────────────────────────────────────────────────────────── # These monitors are now driven by sunday_morning_coffee_report.sh via COFFEE_REPORT_SCRIPTS. # No separate User Script entries needed — configure which monitors run in master.conf. # All monitors are read-only — observe and report, never act. Still safe to run individually. # # Current COFFEE_REPORT_SCRIPTS (master.conf): # Monitors/zfs_memory_snapshot.sh ZFS pool health + ARC + Docker memory snapshot # Monitors/smart_health.sh drive SMART attributes — reallocated, pending, temp # Monitors/cert_monitor.sh SSL certificate expiry for all configured domains # Monitors/backup_verify.sh rsync mirror integrity via independent MD5 checksums # Monitors/bandwidth_monitor.sh weekly rsync transfer totals and per-share breakdown # Monitors/emby_session_report.sh Emby usage — streams, users, library, transcode ratio # Monitors/weekly_health_digest.sh aggregated digest — watchdogs, fallback, skip list # ============================================================================================== # ██ INDIVIDUAL SCRIPTS — STANDALONE AND SELECTIVE USE ███████████████████████████████████████ # ============================================================================================== # # Everything below is called by the orchestrators above when using the full ecosystem. # Listed here so you can: # → Run one script manually without triggering the full orchestrator # → Use only specific parts of the ecosystem (just arr cleanup, just transcode manager) # → Test individual scripts before scheduling the orchestrator # → Schedule individual scripts on different timing than the orchestrator uses # # Every script is fully standalone — sources its own config, runs its own pre-flight checks. # # ============================================================================================== # ────────────────────────────────────────────────────────────────────────────────────────────── # FALLBACK # ────────────────────────────────────────────────────────────────────────────────────────────── # fallback.sh — [continuous] mutual fallback state machine # Started by array_started.sh on both servers independently. # Every FALLBACK_CHECK_INTERVAL (120s) pings: remote Tailscale IP + 8.8.8.8 # States: NORMAL / FALLBACK / NO_INTERNET / DARK # FALLBACK: starts remote containers in tiers across 24 hours: # Tier 1 immediate: DDNS + Emby + auth (NPM/Authelia/LLDAP) + VaultWarden + Live TV # Tier 2 at 4hr: NextCloud + Immich + Jellyseerr # Tier 3 at 12hr: AdGuard + Gitea + uptime monitoring # Tier 4 at 24hr: arrs + downloaders # Auth stack (NPM, Authelia, LLDAP) runs warm on both servers always — no failover startup delay. # DDNS: one domain, one DDNS container active at all times. Split-brain prevented by design. # Handback: stop remote DDNS → rsync writeback → start containers → start local DDNS. # # bash /mnt/user/appdata/unraid_scripts/Fallback/fallback.sh # bash /mnt/user/appdata/unraid_scripts/Fallback/fallback.sh --status # bash /mnt/user/appdata/unraid_scripts/Fallback/fallback.sh --dry-run # fallback_test.sh — controlled full-lifecycle fallback test using iptables simulation # ALWAYS run --dry-run first. Live test starts/stops real containers — brief service interruption. # iptables safety trap removes the block rule on ANY exit including ctrl-c and crashes. # Remote is always reachable after the test completes regardless of what happened. # 8 phases: pre-flight → block remote (iptables) → detect FALLBACK → verify Tier 1 started # → restore remote → wait handback → verify NORMAL restored → full pass/fail report. # FALLBACK_TEST_BLOCK_WAIT must be > FALLBACK_CHECK_INTERVAL + buffer (default: 150s). # FALLBACK_TEST_HANDBACK_WAIT must cover: strike confirmation + rsync + container start (default: 360s). # # bash /mnt/user/appdata/unraid_scripts/Fallback/fallback_test.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Fallback/fallback_test.sh # fallback_state_reset.sh — reset fallback state file to NORMAL # Resets state file ONLY — does NOT start or stop any containers. # Use when: state stuck in FALLBACK after testing, after killing fallback.sh mid-cycle, # after manual intervention left state inconsistent, after a failed handback. # Shows current state file before asking for confirmation. # Verify before resetting: right containers on right server, DDNS correct, # both servers Tailscale visible, fallback.sh not currently running. # # bash /mnt/user/appdata/unraid_scripts/Tools/fallback_state_reset.sh --status # bash /mnt/user/appdata/unraid_scripts/Tools/fallback_state_reset.sh # bash /mnt/user/appdata/unraid_scripts/Tools/fallback_state_reset.sh --force # ────────────────────────────────────────────────────────────────────────────────────────────── # RSYNC # ────────────────────────────────────────────────────────────────────────────────────────────── # rsync.sh — core sync script, call with share path and optional --profile=name # Pre-flight every run: connectivity, version parity, remote Docker daemon, # remote rootfs space, remote share exists. Profile matched by directory basename (lowercased). # Drive temperature: reads thresholds from dynamix.cfg. Exit 1=WARN (skip), 2=CRIT (caller aborts). # Logs each transfer to BANDWIDTH_LOG automatically (read by bandwidth_monitor --report). # # Profile behaviour — what containers stop: # critical-data Mariadb-Authelia, Redis-Authelia, NginxProxyManager, Lldap-Gmer4Lfe # + delayed restart: Authelia, Authelia-Secondary (30s after dependencies) # important-data Postgres-NextCloud + delayed: NextCloud # arrs_stack Sonarr, Radarr, Lidarr, Prowlarr, Bazarr, Pinchflat # emby Emby both sides (weekly clean sync — both instances stopped) # emby-fallback nothing stopped (Emby stays running — dirty sync, WAL/SHM excluded) # [no profile] no containers stopped (media shares, plain data) # # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ # /mnt/user/appdata-Fallback/Critical-Data # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ # /mnt/user/appdata-Fallback/Important-Data # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ # /mnt/user/appdata-Fallback/Arrs_Stack # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ # /mnt/user/Media_Server/Emby --profile=emby # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ # /mnt/user/Media_Server/Emby --profile=emby-fallback # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ # /mnt/user/appdata-Fallback/Gmer4Lfe # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Movies # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Tv_Shows # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Music # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Shows # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Movies # rsync_stop.sh — stop rsync intelligently on local and remote # Smart mode (default): kills rsync subprocess only, orchestrator exits cleanly, # any stopped containers are recovered by docker_watchdog.sh on next cycle. # --full-stop: kills orchestrator first then rsync — emergency full stop. # --rsync-only: stop rsync, skip container recovery check (used by partnership_manager --offboard). # # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/rsync_stop.sh --status # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/rsync_stop.sh # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/rsync_stop.sh --full-stop # ────────────────────────────────────────────────────────────────────────────────────────────── # DOCKER ESSENTIALS # ────────────────────────────────────────────────────────────────────────────────────────────── # docker_watchdog.sh — two-tier container self-healing monitor (single-pass) # Called every minute by watchdog_orchestrator.sh — NOT started by array_started.sh. Single-pass. # # Tier 1 — explicit per-container (configured in host*.conf): # Memory hard limits: immediate restart when exceeded — no strikes, no waiting # CPU strike system: 2 consecutive cycles above HARD_CPU_THRESHOLD → restart # HTTP health checks: curl to configured URL — 2 consecutive failures → restart # Required containers: found stopped → restart attempt → failure → skip list # Dependency ordering: database must be running before dependent is restarted # # Tier 2 — global scan of ALL running containers (when WATCHDOG_SCAN_ALL=true): # HEALTHCHECK unhealthy → restart # OOM kill detected → restart + notify # Crash loop (RestartCount climbing) → notify, above WATCHDOG_CRASH_LIMIT → restart → skip list # Dead state → remove + start # Non-zero exit → restart # # Startup grace: WATCHDOG_STARTUP_GRACE (600s) — restarts suppressed, checks still log # Restart loop: 3 restarts in 1hr → skip list on /boot/config → critical notification # RAM emergency: reads mem_shutdown_active from system_watchdog state file, defers all restarts # Silent on clean cycles — only outputs events and hourly heartbeat # # bash /mnt/user/appdata/unraid_scripts/Watchdogs/docker_watchdog.sh --status # bash /mnt/user/appdata/unraid_scripts/Watchdogs/docker_watchdog.sh --dry-run # watchdog_skip_list_manager.sh — view and manage the container skip list # When docker_watchdog restarts the same container 3 times in 1hr → skip-listed. # Skip list persists on /boot/config — survives reboots. Still broken after reboot = stays listed. # Auto-clear: watchdog removes a container from the skip list when it sees it running. # Manual clear needed only when container is stuck stopped after you've fixed the problem. # # Recovery workflow: # --status → understand situation (skip list, running/stopped state, restart history) # fix the underlying problem (check docker logs, disk space, database integrity) # --clear ContainerName → remove from skip list + clear restart history # docker start ContainerName → confirm fix works before handing back to watchdog # watchdog resumes normal monitoring on next cycle automatically # # bash /mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --status # bash /mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --clear ContainerName # bash /mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --clear ContainerName --force # bash /mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --clear-all # bash /mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --dry-run # docker_container_stop.sh — stop all running containers sequentially with verification # Called by array_stopping.sh as the final step in the planned shutdown sequence. # Safe to run standalone before array maintenance, disk replacement, or planned reboots. # Never moves to the next container until the current one is confirmed stopped. # # Stop sequence per container: # docker stop -t 30 (SIGTERM + 30s grace — docker sends SIGKILL if needed) # Retry up to RETRY_COUNT times if still running # docker kill (SIGKILL) if all retries exhausted # Notify on any container that cannot be stopped # # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_container_stop.sh --status # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_container_stop.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_container_stop.sh # docker_daily_restart.sh — restart degradation-prone containers nightly # Called by daily_sync_maintenance.sh. Also safe to run ad hoc. # Running → docker restart (graceful). Stopped → left stopped (state respected). Missing → skip. # Dependency ordering via WATCHDOG_DEPENDENCIES — databases before applications. # Restart verification: checks container still up after settle period, notifies if not. # Configured via HOST*_DAILY_RESTART_CONTAINERS (host*.conf): # NginxProxyManager, Authelia, Dispatcharr, Dispatcharr-Basic, ErsatzTV-Emby # # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_daily_restart.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_daily_restart.sh --status # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_daily_restart.sh # docker_weekly_restart.sh — restart less-critical services weekly # Called by weekly_sync_maintenance.sh after sync completes and containers are back up. # Targets services that benefit from weekly clean start but don't stop for the sync itself. # Same rules as daily: running→restart, stopped→leave, missing→skip. # Configured via HOST*_WEEKLY_RESTART_CONTAINERS (host*.conf): # NextCloud, AdGuard-Home, Immich # # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh --status # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh # docker_network_connect.sh — ensure custom networks exist and containers are connected # Called by array_started.sh. Idempotent — silent when everything is already correct. # Network missing → create (bridge driver, auto-assigned subnet) → notify. # Network creation should only happen after an unRAID update wiped networks — notify tells you. # Configured via HOST*_NETWORK_CONNECT_NETWORKS and HOST*_NETWORK_CONNECT_CONTAINERS. # Use after unRAID updates when containers can't reach each other on custom networks. # # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh --status # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh # docker_update.sh — pull latest images for daily restart containers, no downtime # Called by daily_sync_maintenance.sh BEFORE docker_daily_restart.sh. # Targets HOST*_DAILY_RESTART_CONTAINERS — same list as daily restart, no separate config. # Containers stay running during the pull. docker_daily_restart.sh picks up the new image. # Reports "updated" vs "already current" per container — useful to see what actually changed. # Toggle: DAILY_CONTAINER_UPDATES=false in master.conf skips all pulls, restart still runs. # # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_update.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_update.sh --status # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_update.sh # docker_update_remaining.sh — pull latest images for all containers not in daily/weekly lists # Called by weekly_sync_maintenance.sh at the end of the Sunday window. # Derives target set from `docker ps` minus DAILY_RESTART_CONTAINERS + WEEKLY_RESTART_CONTAINERS. # Only running containers — stopped containers are excluded (likely paused intentionally). # Together with docker_update.sh and the weekly image pull: every deployed container gets # at least one image pull per week with no second list to maintain. # Toggle: WEEKLY_REMAINING_UPDATES=false in master.conf skips all pulls. # # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_update_remaining.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_update_remaining.sh --status # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_update_remaining.sh # downloaders_reset.sh — maintenance reset for all download clients on this server # Called every 30 minutes by critical_sync_maintenance.sh via CRITICAL_MAINTENANCE_SCRIPTS. # Host-aware: sections skip gracefully if a downloader URL is empty for this server. # # slskd: # Stuck searches — clears Completed/Errored searches left by Soularr crashes # Dead transfers — removes completed/errored/aborted transfer records per user # NEVER removes InProgress or Queued transfers # Failed imports — purges albums Soularr downloaded but Lidarr rejected # SABnzbd: # Completed history — removes completed records older than DOWNLOADER_RETENTION_DAYS # Failed history — removes failed records older than DOWNLOADER_RETENTION_DAYS # Stalled queue — removes Paused or Stuck queue items (never active downloads) # qBittorrent: # Age failsafe — removes torrents older than QBIT_FAILSAFE_MIN_DAYS # deleteFiles=false — leaves files for arrs to manage # # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/downloaders_reset.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/downloaders_reset.sh --status # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/downloaders_reset.sh # ────────────────────────────────────────────────────────────────────────────────────────────── # UNRAID ESSENTIALS # ────────────────────────────────────────────────────────────────────────────────────────────── # system_watchdog.sh — third layer in the watchdog chain: runs SYSTEM_WATCHDOG_SCRIPTS each cycle # Called by watchdog_orchestrator.sh — NOT scheduled directly. Single-pass. # Iterates SYSTEM_WATCHDOG_SCRIPTS sequentially: storage_watchdog → webgui_watchdog → network_watchdog. # Each sub-script is independent — a failure in one does not skip the others. # # Tier 1 CRITICAL — bypass ALL strikes, reboot immediately: # Docker daemon hung → attempt rc.docker restart → still hung → reboot # rootfs >= 99% → reboot (writes failing, system cannot recover) # Kernel BUG/Oops delta → reboot (kernel running with corrupted state) # FD exhaustion >= 95% → reboot (new connections failing, Docker can't spawn) # /boot read-only → reboot (state file writes silently failing) # # Tier 2 URGENT — bypass strikes when OOM confirms RAM crisis: # RAM < MEM_GB (4GB) AND >= OOM_LIMIT (3) OOM kills this cycle → reboot # Without OOM confirmation: standard strike system applies. # # Tier 3 STANDARD — N consecutive failures → reboot: # RAM tiers: warn (10GB), stop containers (6GB, excluded list respected), reboot (4GB) # Load average, CPU temperature, /var/log usage, /tmp usage, zombie processes, # sshd, NIC operstate, mdstat errors, kernel oops rate, FD rate # # RAM emergency: writes mem_shutdown_active=true → docker_watchdog defers all restarts. # Reboot loop protection: N reboots in X hours → shutdown instead. # State file heartbeat: writes watchdog_cycle=N every cycle (docker_watchdog stale guard). # # bash /mnt/user/appdata/unraid_scripts/Watchdogs/stability_watchdog.sh --status # bash /mnt/user/appdata/unraid_scripts/Watchdogs/stability_watchdog.sh --dry-run # storage_watchdog.sh — pool growth + runaway log detection (single-pass, called by system_watchdog) # Part of SYSTEM_WATCHDOG_SCRIPTS — not scheduled directly. Called by system_watchdog.sh each cycle. # # Two complementary checks every cycle: # Growth rate (zero-config): du per container vs previous baseline — any container growing # more than WATCHDOG_APPDATA_GROWTH_GB triggers a focused *.log scan inside that dir. # New containers covered automatically from their second cycle. No per-container config. # Absolute log size: finds *.log files over WATCHDOG_APPDATA_LOG_MAX_GB anywhere in # WATCHDOG_APPDATA_PATHS. Catches logs already large but not actively growing. # # Strike system (same pattern as docker_watchdog CPU/HTTP strikes): # Strike 1 → warn + notify. Strike 2 → escalated notify. Strike 3 → action: # WATCHDOG_APPDATA_TRUNCATE_LOGS=true → truncate *.log in-place, clear strikes # WATCHDOG_APPDATA_TRUNCATE_LOGS=false → critical notify, hold strikes until resolved # Condition resolves (growth stops / log shrinks) → strikes auto-clear. # # bash /mnt/user/appdata/unraid_scripts/Watchdogs/System/storage_watchdog.sh --status # bash /mnt/user/appdata/unraid_scripts/Watchdogs/System/storage_watchdog.sh --dry-run # network_watchdog.sh — internet + DDNS + Tailscale + NPM proxy checks (called by system_watchdog) # Part of SYSTEM_WATCHDOG_SCRIPTS — not scheduled directly. Called by system_watchdog.sh each cycle. # # Check 1 — Internet: curl external endpoint. Fail → alert + skip all remaining (prevents cascades). # Check 2 — DDNS (Cloudflare): public IP via ifconfig.me vs DNS record via dig @1.1.1.1. # Mismatch → restart DDNS container (triggers immediate Cloudflare record update) + notify. # Check 3 — Tailscale: tailscale status → Running → pass. Not running → notify (no restart — human). # Check 4 — NPM proxy (external): curl external URL through the full stack. # 2-strike system: Strike 1 = warn + notify. Strike 2 = restart NginxProxyManager + notify. # Strikes auto-clear when the external URL becomes reachable again. # # bash /mnt/user/appdata/unraid_scripts/Watchdogs/System/network_watchdog.sh --status # bash /mnt/user/appdata/unraid_scripts/Watchdogs/System/network_watchdog.sh --dry-run # resource_watchdog.sh — three-level pressure reduction layer (single-pass, called by watchdog_orchestrator) # Reduces system load intelligently BEFORE docker_watchdog attempts container restarts. # Containers restarted into a resource-pressured system just fail again — this runs first. # # Three pressure levels (thresholds configured in master.conf): # Level 1 SOFT (low RAM or high load): throttle SABnzbd + qBittorrent download speeds # Level 2 MEDIUM (lower RAM or higher load): further throttle + docker pause non-critical containers # Level 3 HARD (critically low RAM): docker stop heavy services + signal docker_watchdog to defer restarts # # Recovery: pressure must stay below current threshold for RW_RECOVER_CYCLES consecutive runs # before de-escalating. One level at a time — prevents flip-flopping. # Coordination: at Level 3 writes mem_shutdown_active=true → docker_watchdog defers all restarts. # HOST*_RW_PAUSE_CONTAINERS and HOST*_RW_STOP_CONTAINERS configured per host in host*.conf. # # bash /mnt/user/appdata/unraid_scripts/Watchdogs/resource_watchdog.sh --status # bash /mnt/user/appdata/unraid_scripts/Watchdogs/resource_watchdog.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Watchdogs/resource_watchdog.sh # inotify_tuning.sh — raise Linux inotify kernel limits at array start # Called by array_started.sh FIRST — must run before containers start (they inherit limits). # Settings reset on each reboot — script reapplies on every array start. Idempotent. # # Limits set: # max_user_watches → 1048576 (default 8192 — VSCode alone needs 50-200K for large projects) # max_user_instances → 1024 (default 128) # max_queued_events → 32768 (default 16384) # # If Code-Server started before this ran → docker restart Code-Server to inherit new limits. # Exhaustion symptom: downloads complete but arrs don't detect them, Live TV stutters. # # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/inotify_tuning.sh --status # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/inotify_tuning.sh # php_fpm_max_children.sh — set PHP-FPM pm.max_children at array start # Called by array_started.sh. Default is 4-8 workers — inadequate for a busy multi-user server. # Sets PHP_MAX_CHILDREN (250). 250 × ~2MB idle = ~500MB. Acceptable on 64GB+. # Symptom of saturation: WebGUI slow, settings saves hang, container UI starts timeout. # Resets on each reboot — reapplied at array start. Idempotent: silent when already correct. # # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/php_fpm_max_children.sh --status # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/php_fpm_max_children.sh # docker_syslog_filter.sh — suppress Docker veth/docker0 interface log noise # Called by array_started.sh before containers start. Creates rsyslog drop rule. # Without this: 50+ containers at array start = 200-400 lines of kernel veth messages. # Real events (mount failures, permission errors) are invisible in that noise. # Idempotent: compares expected filter content exactly — only writes when changed. # # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/docker_syslog_filter.sh --status # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/docker_syslog_filter.sh # webgui_watchdog.sh — WebGUI availability watchdog (called by system_watchdog via SYSTEM_WATCHDOG_SCRIPTS) # Not scheduled directly — runs as part of the every-minute watchdog chain. # Escalation: nginx → php-fpm → emhttp, each with recheck before proceeding to next level. # bash /mnt/user/appdata/unraid_scripts/Watchdogs/System/webgui_watchdog.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Watchdogs/System/webgui_watchdog.sh --status # clear_logs.sh — size-threshold log cleanup (called by weekly_sync_maintenance via WEEKLY_MAINTENANCE_SCRIPTS) # Not scheduled directly — runs as part of the Sunday 2:30am weekly window. # System logs only cleared if > LOG_MIN_SIZE_MB (10MB). Docker logs only if > LOG_DOCKER_MAX_MB (100MB). # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/clear_logs.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/clear_logs.sh --status # mover_stop.sh — stop unRAID mover cleanly before array operations # Wall message to logged-in users. Waits MOVER_STOP_TIMEOUT seconds. # SIGTERM (allows mover to finish current file — no partial files). Verify. SIGKILL last resort. # Use before: planned reboots with mover running, disk replacement, array maintenance. # # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/mover_stop.sh --status # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/mover_stop.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/mover_stop.sh # server_reboot.sh — graceful reboot with pre-flight warnings and clean shutdown sequence # Pre-flight warnings (inform not block): rsync running, mover running, active Emby sessions. # Sequence: wall + unRAID notification → wait REBOOT_SLEEP (30s) → virsh shutdown VMs # → wait REBOOT_VM_WAIT → stop libvirt → stop Docker → sync → /sbin/reboot # --dry-run walks the full sequence without rebooting — shows all active processes. # # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/server_reboot.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/server_reboot.sh --status # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/server_reboot.sh # user_scripts_stop.sh — stop all running User Script processes # Finds processes by /tmp/user.scripts path signature. Shows script names, not just PIDs. # SIGTERM → verify → SIGKILL. Self-exclusion prevents killing itself. # Use when plugin Abort button didn't work, or before a reboot to clean up running scripts. # Called automatically by server_reboot.sh before reboot. # # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/user_scripts_stop.sh --status # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/user_scripts_stop.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/user_scripts_stop.sh # git_pull_execute.sh — pull latest scripts from Gitea and set execute permissions # Deployment mechanism for the ecosystem. Push from VS Code → Gitea → run on both servers. # Called automatically by daily_sync_maintenance.sh as the first job in the daily window. # # bash /mnt/user/appdata/unraid_scripts/git_pull_execute.sh # ────────────────────────────────────────────────────────────────────────────────────────────── # MEDIA # ────────────────────────────────────────────────────────────────────────────────────────────── # # All media scripts run via daily_sync_maintenance.sh in the correct order: # permissions → cleaners → arr cleanup # # ORDER MATTERS: arr cleanup depends on correct ownership (permissions first) and clean # folders (junk removal before orphan detection — otherwise junk files appear as orphans). # Listed here for standalone testing or selective use outside the daily window. # media_shares_permissions.sh — apply correct ownership and permissions to all media shares # Applies nobody:users 755 (dirs) / 664 (files) to HOST*_MEDIA_PERMISSION_SHARES. # Two separate passes: find -type d and find -type f — directories get 755, files get 664. # Idempotent: only changes what's wrong. On large libraries takes 20-30 minutes. # Common drift causes: scp/cp leaves root:root, container missing PUID=99 PGID=100, # rsync without --chown flag, admin copies directly into share. # Correct PUID/PGID in Docker template env vars is the permanent fix — this is the safety net. # # bash /mnt/user/appdata/unraid_scripts/Media/media_shares_permissions.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Media/media_shares_permissions.sh --status # bash /mnt/user/appdata/unraid_scripts/Media/media_shares_permissions.sh # media_cleaner.sh — remove scene junk files from media share folders # Run BEFORE arr cleanup — removes non-media files that would otherwise appear as orphans. # Patterns: *.sfv *.md5 *.sha1 *.nfo *.url *.lnk *.rar *.zip *.info *.torrent # *.sample* *.proof* *sync-conflict* *.scr *.exe *.srr *.log *.json # Two profiles with separate folder lists (configured in host*.conf): # anime HOST*_ANIME_CLEAN_FOLDERS — anime share folders # media HOST*_MEDIA_CLEAN_FOLDERS — Movies, Tv_Shows, Music, Sports etc. # ALWAYS --dry-run when adding new patterns or folders — verify before committing. # # bash /mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh anime --dry-run # bash /mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh media --dry-run # bash /mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh anime # bash /mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh media # lidarr_cleanup.sh — delete orphaned music files not tracked by Lidarr # HOST1 ONLY — HOST1 owns /mnt/user/Music and runs Lidarr. Exits cleanly on HOST2. # # Classification per file on disk: # TRACKED → in Lidarr API response → never touch # PROTECTED → matches LIDARR_PROTECTED_PATTERNS → never touch # (*.jpg *.png = cover art, *.nfo = metadata, *.lrc = synced lyrics) # ORPHAN → music extension, not tracked, older than LIDARR_ORPHAN_AGE (7 days), # not open by any process → DELETE # JUNK → not a music extension, not protected → DELETE (any age) # RECENT → not tracked, under 7 days → skip (may be mid-import) # # 7 safety checks ALL must pass before any deletion: # Lidarr running + healthy. API reachable. API version matches. Artist count > 0. # Tracked file count > 0. Count >= 80% of last known (detects database issues). # Total deletion size < LIDARR_MAX_DELETE_GB (50GB — catches path misconfigurations). # # After any deletion: notify_emby_scan() triggers Emby "Clean Missing Files" immediately. # NUCLEAR MODE (use with full understanding): --i-know-what-im-doing --skip-strike-list # Both flags: age check bypassed + size gate bypassed. Deletes everything not tracked. # # bash /mnt/user/appdata/unraid_scripts/Media/lidarr_cleanup.sh --dry-run --log # bash /mnt/user/appdata/unraid_scripts/Media/lidarr_cleanup.sh --status # bash /mnt/user/appdata/unraid_scripts/Media/lidarr_cleanup.sh # sonarr_cleanup.sh — delete orphaned TV files not tracked by Sonarr # HOST-AWARE: HOST1 manages Tv_Shows (via HOST1_SONARR_TV_ROOT). # HOST2 manages Anime_Shows (via HOST2_SONARR_TV_ROOT). # detect_hosts() selects correct root — same script works on both servers. # Same 7-check safety system as lidarr_cleanup. # Protected: *.jpg *.png (episode/show artwork), *.nfo, *.srt *.sub *.ass *.ssa (Bazarr subtitles) # After deletion: notify_emby_scan() clears ghost entries from Emby library immediately. # # bash /mnt/user/appdata/unraid_scripts/Media/sonarr_cleanup.sh --dry-run --log # bash /mnt/user/appdata/unraid_scripts/Media/sonarr_cleanup.sh --status # bash /mnt/user/appdata/unraid_scripts/Media/sonarr_cleanup.sh # radarr_cleanup.sh — delete orphaned movie files not tracked by Radarr # HOST-AWARE: HOST1 manages Movies (via HOST1_RADARR_MOVIES_ROOT). # HOST2 manages Anime_Movies (via HOST2_RADARR_MOVIES_ROOT). # Same 7-check safety system as lidarr_cleanup. # Protected: *.jpg *.png (movie artwork, fanart, thumbnails), *.nfo, subtitles # After deletion: notify_emby_scan() clears ghost entries from Emby library immediately. # # bash /mnt/user/appdata/unraid_scripts/Media/radarr_cleanup.sh --dry-run --log # bash /mnt/user/appdata/unraid_scripts/Media/radarr_cleanup.sh --status # bash /mnt/user/appdata/unraid_scripts/Media/radarr_cleanup.sh # arr_sync.sh — bidirectional arr library sync across all nodes in the ecosystem # Called by intermediate_sync_maintenance.sh every 4 hours. Can be run standalone. # Union mesh: every node syncs with every other — no primary, no hierarchy. # If any node tracks an item, all nodes get it (unless blocklisted). # # What syncs: library items keyed on stable IDs (MusicBrainz / TVDB / TMDB). # Adding to a remote: that remote's own quality profile, metadata profile, and root # folder are used — settings are never copied from the source node. # Remote API keys read via SSH from each arr's config.xml — never stored anywhere. # Key regeneration is picked up automatically on next run. # Blocklist: ARR_SYNC_BLOCKLIST tombstones IDs that must never be re-added anywhere. # --blocklist-add does three things: tombstones, deletes local, deletes all remotes via SSH. # Files become orphans on all nodes — arr_cleanup.sh removes them on next run. # # bash /mnt/user/appdata/unraid_scripts/Media/arr_sync.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Media/arr_sync.sh --status # bash /mnt/user/appdata/unraid_scripts/Media/arr_sync.sh # bash /mnt/user/appdata/unraid_scripts/Media/arr_sync.sh --blocklist-add radarr "reason" # bash /mnt/user/appdata/unraid_scripts/Media/arr_sync.sh --blocklist-add sonarr "reason" # bash /mnt/user/appdata/unraid_scripts/Media/arr_sync.sh --blocklist-add lidarr "reason" # bash /mnt/user/appdata/unraid_scripts/Media/arr_sync.sh --blocklist-remove radarr # bash /mnt/user/appdata/unraid_scripts/Media/arr_sync.sh --blocklist-list # arrs_failed_stalled_recovery.sh — blocklist + re-search failed/stalled arr imports # Called every 4 hours by intermediate_sync_maintenance.sh via INTERMEDIATE_MAINTENANCE_SCRIPTS. # Finds importFailed / importPending / error / stalled items across all configured arrs. # Age gate: items newer than ARR_IMPORT_RECOVERY_AGE (6hr) are skipped — arr may still self-retry. # bash /mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh --status # playback_aware_lidarr_discovery.sh — behavior-driven music discovery (runs via WEEKLY_MAINTENANCE_SCRIPTS) # Called by weekly_sync_maintenance.sh on Sunday. Can be run standalone. # Stage 1: score artists you actually played this week in Emby (play count + recency + user diversity). # Stage 2: run Last.fm artist.getSimilar on top seeds, score recommendations, add top results to Lidarr. # Goal: 0–5 meaningful adds per run. Not bulk import — only artists likely to match your taste. # bash /mnt/user/appdata/unraid_scripts/Media/playback_aware_lidarr_discovery.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Media/playback_aware_lidarr_discovery.sh --status # bash /mnt/user/appdata/unraid_scripts/Media/playback_aware_lidarr_discovery.sh # playback_aware_radarr_discovery.sh — behavior-driven movie discovery (runs via WEEKLY_MAINTENANCE_SCRIPTS) # Called by weekly_sync_maintenance.sh on Sunday. Can be run standalone. # Stage 1: score recently watched movies in Emby, weighted by recency and TMDB rating. # Stage 2: run TMDB recommendations on top seeds, score candidates, add top results to Radarr. # Goal: 0–5 meaningful adds per run. # bash /mnt/user/appdata/unraid_scripts/Media/playback_aware_radarr_discovery.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Media/playback_aware_radarr_discovery.sh --status # bash /mnt/user/appdata/unraid_scripts/Media/playback_aware_radarr_discovery.sh # playback_aware_sonarr_discovery.sh — behavior-driven TV discovery (runs via WEEKLY_MAINTENANCE_SCRIPTS) # Called by weekly_sync_maintenance.sh on Sunday. Can be run standalone. # Stage 1: score recently watched series in Emby (user diversity + recency + per-user-capped episodes). # Stage 2: run TMDB TV recommendations on top seeds, score candidates, add top results to Sonarr. # Goal: 0–3 meaningful adds per run. # bash /mnt/user/appdata/unraid_scripts/Media/playback_aware_sonarr_discovery.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Media/playback_aware_sonarr_discovery.sh --status # bash /mnt/user/appdata/unraid_scripts/Media/playback_aware_sonarr_discovery.sh # lidarr_missing_art.sh — fetch missing album and artist artwork for the Lidarr music library # Called by intermediate_sync_maintenance.sh via INTERMEDIATE_MAINTENANCE_SCRIPTS. # HOST1 only — exits cleanly on HOST2 (HOST2 does not run Lidarr). # READS from Lidarr only — no writes back, never modifies audio tags or renames files. # NEVER overwrites existing artwork — idempotent re-runs are safe. # # Artwork targets: # Album: cover.jpg cdart.png back.jpg # Artist: folder.jpg fanart.jpg logo.png banner.jpg # Sources: fanart.tv → iTunes (album) / Deezer / Last.fm (artist) fallbacks # # bash /mnt/user/appdata/unraid_scripts/Media/lidarr_missing_art.sh --dry-run --log # bash /mnt/user/appdata/unraid_scripts/Media/lidarr_missing_art.sh --status # bash /mnt/user/appdata/unraid_scripts/Media/lidarr_missing_art.sh # radarr_tmdb_removed.sh — remove movies dropped from TMDb (status="deleted") # These generate Radarr system health errors and can never be monitored or downloaded. # 99% are future/announced movies that were delisted before release. # Files NOT deleted by default — use --delete-files to also remove from disk. # Import exclusion added by default — prevents Radarr re-adding the same dropped movie. # HOST1 only — exits cleanly on HOST2. # # bash /mnt/user/appdata/unraid_scripts/Media/radarr_tmdb_removed.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Media/radarr_tmdb_removed.sh --status # bash /mnt/user/appdata/unraid_scripts/Media/radarr_tmdb_removed.sh # bash /mnt/user/appdata/unraid_scripts/Media/radarr_tmdb_removed.sh --delete-files # sonarr_tvdb_removed.sh — remove series dropped from TVDB (status="deleted") # Same pattern as radarr_tmdb_removed: Sonarr health errors, can't be monitored. # Files NOT deleted by default — use --delete-files to also remove from disk. # Import exclusion added by default. HOST1 only. # # bash /mnt/user/appdata/unraid_scripts/Media/sonarr_tvdb_removed.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Media/sonarr_tvdb_removed.sh --status # bash /mnt/user/appdata/unraid_scripts/Media/sonarr_tvdb_removed.sh # bash /mnt/user/appdata/unraid_scripts/Media/sonarr_tvdb_removed.sh --delete-files # ────────────────────────────────────────────────────────────────────────────────────────────── # TRANSCODES # ────────────────────────────────────────────────────────────────────────────────────────────── # ramdisk_setup.sh — create tmpfs ramdisk and transcode symlink at array start # Called by array_started.sh. MUST run before Emby starts. # Idempotent: if ramdisk already mounted → report status and exit cleanly, do not remount. # # Creates: # RAMDISK_PATH tmpfs mount (HOST*_RAMDISK_SIZE ceiling — only uses RAM actually needed) # transcoding-temp/ pre-created inside ramdisk — Emby finds this before the SSD version. # Without pre-creation: Emby creates transcoding-temp at first writable # location which may be SSD, routing ALL sessions there until restart. # TRANSCODE_SSD SSD fallback directory (and SSD/transcoding-temp for consistency) # TRANSCODE_LINK symlink → RAMDISK_PATH (always reset to ramdisk at array start) # # IMPORTANT: Emby Extra Parameters must include: # --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared # Without shared propagation: symlink flips silently ignored after first flip (rprivate locks inode). # # Verify setup: # mountpoint /mnt/ramdisk_transcodes should say "is a mountpoint" # readlink /mnt/ram-transcode should show /mnt/ramdisk_transcodes # ls /mnt/ramdisk_transcodes/ should show transcoding-temp/ # # bash /mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh --status # bash /mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh # transcode_manager.sh — manage ramdisk/SSD symlink based on current usage # Called by transcode_management.sh AFTER cleanup. Can be run standalone for testing. # Modes (TRANSCODE_MANAGER_MODE in master.conf): # smart auto-flip between ramdisk and SSD based on thresholds (default) # ramdisk always ramdisk, never flip to SSD (warn if above threshold) # ssd always SSD, never ramdisk (use post-flip drain or maintenance) # # Safety checks every cycle regardless of mode: # Missing/broken symlink → recreate pointing at ramdisk # Ramdisk disappeared → auto-flip to SSD, notify warning # SSD path missing → disable SSD fallback / error in ssd mode # transcoding-temp missing from ramdisk → recreate immediately (prevents SSD session drift) # Permissions drift → fix silently every run # # Multi-server: TRANSCODE_SERVERS array supports Emby, Jellyfin, Plex # DO NOT add Tdarr here — full-file encoding fills ramdisk rapidly. # # bash /mnt/user/appdata/unraid_scripts/Transcodes/transcode_manager.sh --status # bash /mnt/user/appdata/unraid_scripts/Transcodes/transcode_manager.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Transcodes/transcode_manager.sh # transcode_cleanup.sh — remove stale HLS segment files from ramdisk and SSD fallback # Called by transcode_management.sh BEFORE transcode_manager.sh (order critical). # A file is eligible only when: older than TRANSCODE_MAX_AGE (20min) AND not open by any process. # lsof called ONCE per location (not once per file — critical performance on Live TV systems). # transcoding-temp directory NEVER deleted even when empty — protects against SSD session drift. # After cleanup: if ramdisk usage dropped below RAMDISK_LOW_GB, triggers flip-back to ramdisk. # # bash /mnt/user/appdata/unraid_scripts/Transcodes/transcode_cleanup.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Transcodes/transcode_cleanup.sh # ramdisk_stop.sh — safely stop the transcode ramdisk without interrupting Emby # Flips the transcode symlink to the SSD fallback BEFORE unmounting so Emby continues writing. # Then unmounts the tmpfs and updates the state file. # Use before: planned reboots, ramdisk size changes, ramdisk troubleshooting. # ramdisk_setup.sh recreates and re-mounts on next array start. # # bash /mnt/user/appdata/unraid_scripts/Tools/ramdisk_stop.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Tools/ramdisk_stop.sh --status # bash /mnt/user/appdata/unraid_scripts/Tools/ramdisk_stop.sh # ────────────────────────────────────────────────────────────────────────────────────────────── # MONITORS # ────────────────────────────────────────────────────────────────────────────────────────────── # # All monitor scripts observe and report — they NEVER act on what they find. # Read-only except bandwidth_monitor (one bounded log append per rsync) and # system_tuning_monitor (one bounded log append per 6hr snapshot). # Safe to run at any time. # zfs_memory_snapshot.sh — weekly ZFS + ARC + memory diagnostic report # ZFS pool status (ONLINE/DEGRADED/FAULTED) per pool. Pools in ZFS_REPORT_IGNORE_POOLS excluded. # ARC: current size vs max, hit rate, metadata vs data ratio. Warns if ARC > ZFS_REPORT_ARC_WARN_PCT (90%). # Memory: total, free, available. Warns if RAM < ZFS_REPORT_FREE_WARN_GB. # Docker: top ZFS_REPORT_DOCKER_TOP (10) containers by memory usage. # Kernel pressure: vmstat 3-sample snapshot. # Output: console + /var/log/zfs-weekly-health.log (enables week-over-week comparison). # # bash /mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh --status # bash /mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh # smart_health.sh — drive SMART attribute monitoring # Checks every /dev/sd* and /dev/nvme*. Drives in HOST*_SMART_IGNORE_DRIVES are skipped. # Temperature thresholds read from /boot/config/plugins/dynamix/dynamix.cfg (same as unRAID UI). # Notifies on: Reallocated_Sector_Ct > 0 (bad sector swapped = warning sign), # Current_Pending_Sector > 0 (suspected bad, not confirmed), Offline_Uncorrectable > 0 (data loss risk), # temperature above thresholds, SMART overall-health FAILED (get data off immediately). # Silent when all drives pass all checks. # # bash /mnt/user/appdata/unraid_scripts/Monitors/smart_health.sh --status # bash /mnt/user/appdata/unraid_scripts/Monitors/smart_health.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Monitors/smart_health.sh # cert_monitor.sh — SSL certificate expiry via live TLS connection # Connects via openssl s_client — checks cert actually being served to clients. # Catches: renewed-but-not-reloaded (nginx not reloaded after certbot renewal), # wrong cert served, chain issues visible externally but not internally. # If a user would see a certificate error in their browser, this catches it first. # Configured via HOST*_CERT_MONITOR_DOMAINS in host*.conf. # Thresholds: > 30 days = silent, <= 30 = warning, <= CERT_CRIT_DAYS (7) = urgent. # # bash /mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh --status # bash /mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh # backup_verify.sh — rsync mirror integrity via independent MD5 checksums # Randomly samples BACKUP_VERIFY_SAMPLE (10) files per share. Computes MD5 both sides via SSH. # Does not trust rsync exit code — verifies actual file content matches. # rsync exit 0 does NOT guarantee content is correct. Hardware corruption produces files # that exist with matching size/mtime but different content. rsync misses this. MD5 doesn't. # Pre-flight: check_connectivity() and check_remote_array() prevent false MISSING alarms. # Notifies on MISMATCH or excessive MISSING files. # # bash /mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh --status # bash /mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh # emby_session_report.sh — Emby weekly streaming usage statistics # Queries HOST*_EMBY_URL for past EMBY_REPORT_DAYS (7) days of activity. # Reports: server version, active now (direct vs transcode), library size (movies/episodes/songs), # weekly play count, top EMBY_REPORT_TOP_N content, most active users. # Notifies if transcode ratio > 80% — usually indicates client/format compatibility issue. # # bash /mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh --status # bash /mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh # bandwidth_monitor.sh — rsync transfer history and weekly summary report # --report: reads BANDWIDTH_LOG and generates weekly summary per profile. # Shows: run count, total GB transferred, average duration, failure count, large transfers. # --log-transfer: called AUTOMATICALLY by rsync.sh after each sync — never call manually. # Log format: DATE|TIME|profile|duration|status|bytes_transferred # Version-stable field names — survives rsync output format changes between versions. # BANDWIDTH_LOG bounded to BANDWIDTH_LOG_RETENTION (90) days — never grows unbounded. # # bash /mnt/user/appdata/unraid_scripts/Monitors/bandwidth_monitor.sh --report # bash /mnt/user/appdata/unraid_scripts/Monitors/bandwidth_monitor.sh --status # weekly_health_digest.sh — profile-controlled ecosystem health digest (see ORCHESTRATORS) # bash /mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh --status # system_tuning_monitor.sh — inotify + PHP-FPM trend tracking (see ORCHESTRATORS) # bash /mnt/user/appdata/unraid_scripts/Monitors/system_tuning_monitor.sh --status # mesh_monitor.sh — at-a-glance view of all servers in the mesh (read-only, on demand) # No schedule — run any time you want a current mesh snapshot. No SSH or API calls. # Shows each node: hostname, Tailscale IP, role (owner/mirror), services covered in fallback tiers. # Reads directly from all three conf files — does not use detect_hosts(). # # bash /mnt/user/appdata/unraid_scripts/Monitors/mesh_monitor.sh # continuous_scripts_status.sh — legacy live dashboard (Old_Arch_Still_Works/) # NOTE: watchdog architecture changed to single-pass in v2.4. This dashboard was written # for the continuous-loop design and is no longer kept current. Use --status flags on # individual scripts instead: docker_watchdog --status, stability_watchdog --status, etc. # # bash /mnt/user/appdata/unraid_scripts/Old_Arch_Still_Works/continuous_scripts_status.sh # ────────────────────────────────────────────────────────────────────────────────────────────── # PARTNERSHIP # ────────────────────────────────────────────────────────────────────────────────────────────── # partnership_manager.sh — two-server auth-sharing relationship lifecycle manager # Automated --check runs inside critical_sync_maintenance.sh. Below are manual operation modes. # Blocklist: after offboard, former partner hostname is written to partnership_blocklist.db. # --onboard is hard-blocked if remote is on blocklist. --unblock clears it. # # --status: show both server state files (ACTIVE/INACTIVE), offline counter, owner server, # current WebUI redirect state on HOST2, whether critical sync is enabled. # # --onboard: HOST1/owner only. Reconfigures HOST2 Docker template WebUI URLs to redirect # to HOST1 via Tailscale (click NPM on HOST2 → opens HOST1's NPM). # Verifies WebUI routing works. Writes ACTIVE state on both servers. # ALWAYS --dry-run first — shows exact template changes before anything is touched. # # --offboard: either server can initiate. Stops rsync. Final sync to ensure HOST2 has # current state. Reconfigures HOST2 WebUI URLs back to localhost (HOST2 manages # its own auth independently from now on). Grace period then Tailscale removal. # Deferred finalisation: if remote unreachable, completes automatically when reachable. # ALWAYS --dry-run first. 10-second countdown before live execution. # # --transfer: owner only. Flips auth ownership (HOST1→HOST2 or back). Health strike system # verifies both servers healthy before proceeding. Requires confirmation string: # "i-understand-this-transfers-ownership" (long by design — cannot type accidentally). # # bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manager.sh --status # bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manager.sh --onboard --dry-run # bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manager.sh --onboard # bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manager.sh --unblock hostname # partnership_offboard.sh — cleanly end a partnership (standalone version of --offboard) # Role is detected automatically — run on either server. Owner path runs the full sequence: # stop rsync → final sync → reconfigure mirror WebUIs → disable sync → local cleanup → # Tailscale removal. Mirror path handles local side and signals owner. # Deferred finalisation: if remote unreachable, completes automatically when reachable. # ALWAYS --dry-run first. 10-second countdown before live execution. # # bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_offboard.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_offboard.sh --status # bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_offboard.sh # partnership_transfer.sh — transfer ownership from current owner to current mirror # Owner only — mirror cannot initiate. Config and WebUI targets updated; containers stay. # Health strike system verifies both servers healthy before proceeding. # Requires confirmation string: "i-understand-this-transfers-ownership" (long by design). # After transfer: sync direction reverses on the next fallback/critical_sync cycle. # # bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_transfer.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_transfer.sh --status # bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_transfer.sh \ # --confirm=i-understand-this-transfers-ownership # gitea_ssh_setup.sh — generate Gitea SSH keypair and register it with Gitea via API # Idempotent: skips generation if key exists, skips registration if same key already registered. # Use --force to regenerate and re-register (e.g. after Gitea reinstall or key compromise). # Called during onboard setup or when re-keying a server. Run after Gitea container is up. # # bash /mnt/user/appdata/unraid_scripts/Partnership/gitea_ssh_setup.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Partnership/gitea_ssh_setup.sh --status # bash /mnt/user/appdata/unraid_scripts/Partnership/gitea_ssh_setup.sh # bash /mnt/user/appdata/unraid_scripts/Partnership/gitea_ssh_setup.sh --force # ────────────────────────────────────────────────────────────────────────────────────────────── # INITIAL RUN # ────────────────────────────────────────────────────────────────────────────────────────────── # # Run ONCE when setting up a new server or joining a new partnership. Not scheduled. # All initial-run scripts now live in Partnership/ alongside the lifecycle scripts they set up. # ssh_setup.sh — generate the rsync SSH keypair and install it on the remote server # Key named from hostname: unRAID-Gmer4Lfe → gmer4lfe_rsync_automation # Idempotent — skips generation if key already exists. --force to regenerate and re-copy. # # --validate: health check mode. Called during partnership --check to detect broken SSH auth. # Remote Tailscale IP unreachable = network issue → NOT a strike. # Remote reachable but SSH auth fails = key issue → strike incremented. # At SSH_MAX_STRIKES (5): notify + return exit 2 for caller to escalate. # Strikes reset after SSH_STRIKE_RESET_HRS (24) of clean connectivity. # # bash /mnt/user/appdata/unraid_scripts/Partnership/ssh_setup.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Partnership/ssh_setup.sh --status # bash /mnt/user/appdata/unraid_scripts/Partnership/ssh_setup.sh # bash /mnt/user/appdata/unraid_scripts/Partnership/ssh_setup.sh --force # bash /mnt/user/appdata/unraid_scripts/Partnership/ssh_setup.sh --validate # partnership_onboard.sh — full partnership setup orchestrator (run once to join a partner server) # Orchestrates the complete setup sequence in order: # Step 1: ssh_setup.sh generate keypair, install on remote, update conf # Step 2: partnership_manager.sh --onboard # verify both servers, reconfigure auth WebUIs → owner IP, # write ACTIVE state, create FolderView3 folder if enabled # Step 3: arr_sync.sh bidirectional library bootstrap — both sides start with the # merged library from day one (non-fatal: skipped if arrs not live) # # Pre-requisites: both servers on same Tailscale tailnet, remote has password auth for root, # PARTNERSHIP_OWNER_HOST correct in master.conf. # ALWAYS --dry-run first. # # bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_onboard.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_onboard.sh # bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_onboard.sh --skip-ssh # bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_onboard.sh --skip-arr-sync # ────────────────────────────────────────────────────────────────────────────────────────────── # TOOLS # ────────────────────────────────────────────────────────────────────────────────────────────── # # Situational utilities — run when something specific needs fixing, not on a schedule. # Each exists because a specific situation arose that required bash commands to resolve. # All support --dry-run and --status. # bulk_permissions_repair.sh — apply nobody:users 755/664 to a specific path immediately # Targeted fix — do not wait for next nightly media_shares_permissions.sh run. # Use when: scp/cp left root:root files blocking arr operations right now. # Accepts one or multiple paths on the same command line. # # bash /mnt/user/appdata/unraid_scripts/Tools/bulk_permissions_repair.sh /mnt/user/Movies --dry-run # bash /mnt/user/appdata/unraid_scripts/Tools/bulk_permissions_repair.sh /mnt/user/Movies # bash /mnt/user/appdata/unraid_scripts/Tools/bulk_permissions_repair.sh /mnt/user/Movies /mnt/user/Music # container_data_export.sh — stop container, archive appdata, verify archive, restart # Use before: major version updates with "no rollback" migrations, pool migrations, # container removal, risky config changes that need a clean restore point. # Sequence: space check → stop container → tar.gz archive → verify integrity → restart. # Container is always restarted even if archiving fails — never leaves container stopped. # Syntax: container_data_export.sh ContainerName /path/to/appdata /output/directory # # bash /mnt/user/appdata/unraid_scripts/Tools/container_data_export.sh \ # Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/ --dry-run # bash /mnt/user/appdata/unraid_scripts/Tools/container_data_export.sh \ # Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/ # emby_database_repair.sh — SQLite integrity check on all Emby databases # Use when: Emby crashes repeatedly with database errors, after power loss with Emby running. # Stops Emby. Runs PRAGMA integrity_check on every .db file. Restarts Emby. Reports per database. # Recovery guidance per database (this script does NOT auto-delete — human decision required): # library.db → CORRUPT: safe to delete (Emby fully rebuilds from media files) # users.db → CORRUPT: deleting resets ALL accounts + watch history (check backup) # authentication.db → CORRUPT: safe to delete (API keys regenerated on restart) # activity.db → CORRUPT: safe to delete (it's a log only) # Config path detected automatically from Docker volume mounts — no configuration needed. # # bash /mnt/user/appdata/unraid_scripts/Tools/emby_database_repair.sh --status # bash /mnt/user/appdata/unraid_scripts/Tools/emby_database_repair.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Tools/emby_database_repair.sh # zfs_pool_scrub.sh — trigger ZFS scrub on all pools, wait for completion # ZFS checksums every block and verifies — catches silent data corruption before you # discover it on reads. Safe to run while system is active (low I/O priority). # Pools in ZFS_REPORT_IGNORE_POOLS excluded from default all-pool scrub. # Specify a pool name to scrub it specifically (bypasses the ignore list). # Notifies when complete with error summary. # # bash /mnt/user/appdata/unraid_scripts/Tools/zfs_pool_scrub.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Tools/zfs_pool_scrub.sh # bash /mnt/user/appdata/unraid_scripts/Tools/zfs_pool_scrub.sh poolname # emby_to_lidarr_sync.sh — one-shot bootstrap: add all played Emby artists to Lidarr # Finds artists played on Emby that are not tracked in Lidarr and adds them. No scoring. # Run once after Lidarr is set up, or any time you suspect gaps between what you listen to # and what Lidarr monitors. Use --days N to limit lookback window. # bash /mnt/user/appdata/unraid_scripts/Tools/emby_to_lidarr_sync.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Tools/emby_to_lidarr_sync.sh --status # bash /mnt/user/appdata/unraid_scripts/Tools/emby_to_lidarr_sync.sh # emby_to_radarr_sync.sh — one-shot bootstrap: add all played Emby movies to Radarr # Same pattern as emby_to_lidarr_sync. Run once to ensure everything you've watched is monitored. # bash /mnt/user/appdata/unraid_scripts/Tools/emby_to_radarr_sync.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Tools/emby_to_radarr_sync.sh --status # bash /mnt/user/appdata/unraid_scripts/Tools/emby_to_radarr_sync.sh # emby_to_sonarr_sync.sh — one-shot bootstrap: add all played Emby series to Sonarr # Same pattern as emby_to_lidarr_sync. Run once to ensure all watched shows are monitored. # bash /mnt/user/appdata/unraid_scripts/Tools/emby_to_sonarr_sync.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Tools/emby_to_sonarr_sync.sh --status # bash /mnt/user/appdata/unraid_scripts/Tools/emby_to_sonarr_sync.sh # recreate_shares.sh — create share directories on disks after fresh install or rebuild # Use when: share .cfg files restored to fresh install but directories missing on disks. # Reads /boot/config/shares/*.cfg, parses shareInclude list, creates /mnt/diskN/sharename. # chown nobody:users on creation. Run once — then initial rsync populates the content. # Without this: rsync aborts "remote share missing" even though the share shows in the UI. # # bash /mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh --status # bash /mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh # ============================================================================================== # ██ QUICK REFERENCE — COMPLETE SCHEDULE █████████████████████████████████████████████████████ # ============================================================================================== # # At Startup of Array: # Orchestrators/array_started.sh (single entry — handles everything) # # At Stopping of Array: # Orchestrators/array_stopping.sh (single entry — graceful shutdown) # # * * * * * every minute: # Orchestrators/watchdog_orchestrator.sh # └─ resource_watchdog → docker_watchdog → system_watchdog (storage/webgui/network) → stability_watchdog # # */7 * * * * every 7 minutes: # Orchestrators/transcode_management.sh # # */30 * * * * every 30 minutes: # Orchestrators/critical_sync_maintenance.sh # Rsync/rsync.sh /mnt/user/Media_Server/Emby --profile=emby-fallback # # 0 */4 * * * every 4 hours: # Orchestrators/intermediate_sync_maintenance.sh # └─ arr_sync → arrs_failed_stalled_recovery + lidarr_missing_art (INTERMEDIATE_MAINTENANCE_SCRIPTS) # # 0 */6 * * * every 6 hours: # Monitors/system_tuning_monitor.sh # # 0 1 * * * 1am daily: # Orchestrators/daily_sync_maintenance.sh # # 0 0 15 * * 15th of each month at midnight: # Orchestrators/monthly_maintenance.sh # # 0 7 * * 0 Sunday 7am: # Orchestrators/sunday_morning_coffee_report.sh # └─ runs all monitors via COFFEE_REPORT_SCRIPTS (master.conf): # zfs_memory_snapshot / smart_health / cert_monitor / backup_verify # bandwidth_monitor / emby_session_report / weekly_health_digest # # 0 8 * * * 8am daily: # Monitors/weekly_health_digest.sh # # 30 2 * * 0 Sunday 2:30am: # Orchestrators/weekly_sync_maintenance.sh # └─ runs WEEKLY_MAINTENANCE_SCRIPTS after sync: # clear_logs / playback_aware_lidarr / playback_aware_radarr / playback_aware_sonarr # # ==============================================================================================