#!/bin/bash
# ==============================================================================================
# ============================= USER SCRIPTS MASTER TEMPLATE ===================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Paste this file into a User Script entry. Uncomment ONE script block and set
# the schedule. Every script in the ecosystem is listed here — from the
# orchestrators that run it all, down to the individual scripts you can run
# standalone for specific tasks.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# When used as intended, only a handful of orchestrators need to be scheduled.
# The orchestrators handle everything else in the correct order — they call child
# scripts, manage timing and dependencies, track pass/fail, and send one
# notification per window.
#
# You do not need to schedule every script below. The orchestrators cover it all:
#
#   array_started.sh                    at array start — launches ALL startup scripts in order
#   array_stopping.sh                   at array stop  — graceful shutdown sequence
#   watchdog_orchestrator.sh            every 15 min   — resource + docker + system watchdog
#   transcode_management.sh             every 7 min    — cleanup then manager (order critical)
#   critical_sync_maintenance.sh        every 30 min   — auth + play_state_sync + partnership
#   intermediate_sync_maintenance.sh    every 4 hours  — arr library sync + artwork fetch
#   daily_sync_maintenance.sh           1am daily      — git + rsync + media + restart
#   weekly_sync_maintenance.sh          2:30am Sunday  — clean sync + image updates
#   sunday_morning_coffee_report.sh     7am Sunday     — full weekly digest
#   weekly_health_digest.sh             8am daily      — profile-controlled health notification
#   system_tuning_monitor.sh            every 6 hours  — inotify + php-fpm trend tracking
#   monthly_maintenance.sh              15th monthly   — uptime-gated heavy tasks (ZFS scrub, SMART long test)
#
# ── INDIVIDUAL SCRIPTS ────────────────────────────────────────────────────────────────────────
# Every child script is also listed below, individually.
# If you only want part of the ecosystem — say, just the arr cleanup scripts, or just
# the transcode manager — uncomment only those blocks and schedule them directly.
# All scripts are fully self-contained and work standalone.
#
# ── ALL SCRIPTS SUPPORT ───────────────────────────────────────────────────────────────────────
#   --dry-run    run without making any changes — shows exactly what would happen
#   --status     show current configuration and state, then exit
#   --log        verbose output — every decision logged
#
# ── PATHS ─────────────────────────────────────────────────────────────────────────────────────
#   Scripts:       /boot/config/plugins/varaverk/
#   Configuration: /boot/config/plugins/varaverk/Configurations/master.conf
#   Per-host:      /boot/config/plugins/varaverk/Configurations/host1.conf (or host2.conf)
#   Library:       /boot/config/plugins/varaverk/common.sh
#
# ── PLUGIN SETTINGS (apply to every entry) ────────────────────────────────────────────────────
#   Run in background: YES   — required for all scripts, even one-shots
#   Run as root:       YES   — all scripts require root
#
# ── CHANGELOG ─────────────────────────────────────────────────────────────────────────────────
#   v1.0 — Initial template
#   v1.1 — MAX_RSYNC_PROCS removed, --status flag, full directory tree, changelog
#   v1.2 — Docker Essentials, Media, Transcodes, System Watchdog added
#   v1.3 — Fallback script added
#   v1.4 — WebGUI watchdog, ZFS snapshot, Docker network connect, schedules section
#   v1.5 — media_management.sh orchestrator, corrected filenames, git repo section
#   v1.6 — lidarr_cleanup.sh, sonarr_cleanup.sh, radarr_cleanup.sh added
#   v1.7 — cert_monitor.sh added
#   v1.8 — Monitors/ folder with all monitoring scripts
#   v1.9 — transcode_management.sh added to Orchestrators/
#   v2.0 — Major restructure: array_started.sh single entry point, daily/weekly orchestrators,
#           sunday_morning_coffee_report.sh, docker_network_connect combined,
#           inotify_tuning.sh, arr path translation, version checking, nuclear mode flags
#   v2.1 — critical_sync_maintenance.sh, Partnership/ folder, system_tuning_monitor.sh,
#           arrs_failed_stalled_recovery.sh, rsync_stop.sh orchestrator-aware,
#           WEEKLY_SYNC_UPDATES, temp thresholds from dynamix.cfg
#   v2.2 — Orchestrators at top with full inline detail. All child scripts listed
#           individually below for standalone use. Inline descriptions on every entry.
#           Correct schedules verified against script headers. sunday_morning_coffee_report
#           and rsync_emby_fallback added as separate scheduled entries.
#   v2.3 — intermediate_sync_maintenance.sh new 4-hour orchestrator (arr_sync + artwork).
#           docker_update.sh (normal + --remainder): daily/monthly image pulls.
#           downloaders_reset.sh: slskd + SABnzbd + qBittorrent maintenance.
#           arr_sync.sh: bidirectional Lidarr/Sonarr/Radarr library mesh.
#           lidarr_missing_art.sh: fetch missing album/artist artwork.
#           radarr_tmdb_removed.sh + sonarr_tvdb_removed.sh: dropped-entry cleanup.
#           Initial_run/ section: ssh_setup.sh + partnership_onboard.sh.
#           Moved: arrs_failed_stalled_recovery → Media/, fallback_state_reset +
#           watchdog_skip_list_manager → Tools/. Renamed: partnership_manager.sh.
#   v2.4 — array_stopping.sh: At Stopping of Array orchestrator (graceful shutdown sequence).
#           watchdog_orchestrator.sh: every-minute cron replacing continuous loops in
#           system_watchdog + docker_watchdog (both now single-pass, no longer started by
#           array_started.sh — remove from ARRAY_START_SCRIPTS in master.conf).
#           resource_watchdog.sh: three-level pressure reduction (soft/medium/hard).
#           docker_container_stop.sh: sequential verified container stop for shutdown.
#           Path fixes: git_pull_execute.sh (root), continuous_scripts_status.sh (Tools/).
#   v2.5 — fallback.sh --stop: added to ARRAY_STOP_SCRIPTS as step 2 (after user_scripts_stop,
#           before rsync_stop) — fallback runs as a bare subprocess, not via /tmp/user.scripts,
#           so it is not caught by user_scripts_stop.sh.
#   v2.6 — monthly_maintenance.sh: uptime-gated orchestrator for heavy monthly tasks (ZFS scrub,
#           SMART long test). Schedule 0 0 15 * * — 15th of each month at midnight.
#           storage_watchdog.sh + network_watchdog.sh: pool growth + connectivity watchdogs,
#           called via SYSTEM_WATCHDOG_SCRIPTS by system_watchdog.sh (watchdog chain, no cron).
#           webgui_watchdog.sh moved from standalone cron to SYSTEM_WATCHDOG_SCRIPTS chain.
#           arrs_failed_stalled_recovery.sh moved to INTERMEDIATE_MAINTENANCE_SCRIPTS (every 4hr).
#           clear_logs.sh moved to WEEKLY_MAINTENANCE_SCRIPTS (Sunday weekly window).
#           Sunday monitor block replaced by COFFEE_REPORT_SCRIPTS (sunday_morning_coffee_report).
#           Playback-aware discovery (lidarr/radarr/sonarr) added to WEEKLY_MAINTENANCE_SCRIPTS.
#           Partnership/: gitea_ssh_setup.sh, partnership_offboard.sh, partnership_transfer.sh.
#           Tools/: emby_to_lidarr_sync.sh, emby_to_radarr_sync.sh, emby_to_sonarr_sync.sh,
#           ramdisk_stop.sh. Monitors/: mesh_monitor.sh. Path fix: Initial_run/ → Partnership/.
# ==============================================================================================


# ==============================================================================================
# ██  ORCHESTRATORS — RUN THE WHOLE ECOSYSTEM  ██████████████████████████████████████████████████
# ==============================================================================================
#
# These are the only entries you need to schedule when using the ecosystem as intended.
# Each orchestrator calls child scripts in the correct order with correct timing.
# Configure what runs via master.conf — never by editing the orchestrator scripts.
#
# ==============================================================================================


# ── ARRAY START ───────────────────────────────────────────────────────────────────────────────
# Schedule:   At Startup of Array
# Background: YES — required (starts continuous background processes)
#
# The ONLY "At Startup of Array" entry needed. Reads ARRAY_START_SCRIPTS from master.conf
# and launches each in order. Detects one-shot vs continuous scripts automatically.
# Auto-fixes execute permissions on any script missing them.
#
# Launches in order:
#   inotify_tuning.sh         raise kernel inotify limits BEFORE containers inherit them
#   docker_syslog_filter.sh   suppress Docker veth noise BEFORE containers create interfaces
#   php_fpm_max_children.sh   set PHP-FPM workers to 250 BEFORE WebGUI serves requests
#   ramdisk_setup.sh          create tmpfs ramdisk + transcode symlink BEFORE Emby starts
#   docker_network_connect.sh ensure custom networks exist and containers are connected
#   fallback.sh               [continuous] mutual fallback state machine
#
# NOTE: system_watchdog.sh and docker_watchdog.sh are no longer started by array_started.sh.
# They are now single-pass scripts called every 15 minutes by watchdog_orchestrator.sh (see below).
# Remove them from ARRAY_START_SCRIPTS in master.conf if present.
#
# bash /boot/config/plugins/varaverk/Orchestrators/array_started.sh


# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Schedule Orchestrators, Not Scripts
#   The catalog below lists every script in the ecosystem, but only a handful of
#   orchestrators are meant to be scheduled. The orchestrators own ordering, dependencies,
#   pass/fail tracking and one notification per window — scheduling their children directly
#   bypasses all of that and produces overlapping runs the locks then have to fight.
#
# Everything Listed, Most Commented
#   Individual scripts are documented here so they can be run standalone for a specific
#   task, not so they can be scheduled. The commented-out default is the safe state: paste
#   the file, uncomment exactly one block, set the schedule.
#
# One Block Per User Script Entry
#   Each entry runs one thing. Combining blocks defeats the per-entry schedule and makes a
#   failure in the first silently skip the rest.
#
# Documentation That Cannot Drift Silently
#   Schedules here are stated alongside what the script does, so a mismatch with
#   varaverk.cron is visible when read. Cross-check against the cron before trusting a
#   cadence quoted in a comment — the cron is authoritative.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Inert by Default
#   Every command in this file is commented out. Pasted as-is it does nothing — the
#   operator must deliberately uncomment a block. That is the safeguard: there is no state
#   in which this file runs something unintended.
#
# No Root, No Lock, No detect_hosts — Deliberate
#   This is a reference template, never executed as a file. Each uncommented block invokes
#   a real script that enforces its own root check and takes its own lock. Adding them here
#   would gate a file that is only ever read.
#
# Full Paths Throughout
#   Every example uses an absolute path. User Scripts entries run with an unpredictable
#   working directory, and a relative path would resolve differently depending on how the
#   plugin happened to invoke it.
#
# Background Flag Documented Per Entry
#   Continuous scripts are marked Background: YES and single-pass ones NO. Getting this
#   wrong either blocks the array-start sequence on a script that never exits, or detaches
#   one whose exit code the sequence needed.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# None — this file configures nothing and reads nothing. It is documentation shaped like a
# shell script so it can be pasted into a User Scripts entry.
#
# What the scheduled orchestrators actually read:
#
#   master.conf   ARRAY_START_SCRIPTS, ARRAY_STOP_SCRIPTS, WATCHDOG_ORCHESTRATOR_SCRIPTS,
#                 CRITICAL_/INTERMEDIATE_/DAILY_/WEEKLY_/MONTHLY_MAINTENANCE_SCRIPTS
#
# Add or remove a script by editing those lists — not by adding another User Scripts entry.
# That is the whole point of the orchestrator model.
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# Not executed. Paste into a User Scripts entry, uncomment ONE block, set its schedule.
#
# The blocks below are grouped by area (Fallback, Rsync, Docker Essentials, Unraid
# Essentials, Media, Transcodes, Monitors, Partnership). Each carries its own recommended
# schedule and Background flag.
#
# ==============================================================================================


# ── ARRAY STOPPING ────────────────────────────────────────────────────────────────────────────
# Schedule:   At Stopping of Array
# Background: YES
#
# The ONLY "At Stopping of Array" entry needed. Reads ARRAY_STOP_SCRIPTS from master.conf
# and runs each sequentially — each must complete before the next starts.
#
# Runs in order:
#   user_scripts_stop.sh       stop background user scripts first (prevents new operations)
#   fallback.sh --stop         gracefully stop fallback (not caught by user_scripts_stop)
#   rsync_stop.sh --rsync-only kill rsync without triggering container recovery
#   mover_stop.sh              stop mover after rsync (both write to same paths)
#   docker_container_stop.sh   stop all containers one-by-one with verification
#
# A failed step is logged and notified but does not prevent remaining steps from running.
# Unlike array_started.sh, all steps run in the foreground — each confirmed done before next.
#
# bash /boot/config/plugins/varaverk/Orchestrators/array_stopping.sh
# bash /boot/config/plugins/varaverk/Orchestrators/array_stopping.sh --dry-run
# bash /boot/config/plugins/varaverk/Orchestrators/array_stopping.sh --status


# ── WATCHDOG ORCHESTRATOR ─────────────────────────────────────────────────────────────────────
# Schedule:   */15 * * * *  (every 15 minutes)
# Background: YES
#
# Runs the three-layer watchdog system in the correct sequence each cycle. Replaces the
# continuous loops that used to run inside system_watchdog.sh and docker_watchdog.sh —
# those scripts are now single-pass and called here.
#
# Execution order (non-negotiable):
#   1. resource_watchdog.sh   free RAM and reduce CPU pressure BEFORE healing attempts
#   2. docker_watchdog.sh     restart containers with pressure already reduced
#   3. system_watchdog.sh     storage + WebGUI + network checks via SYSTEM_WATCHDOG_SCRIPTS:
#                               storage_watchdog.sh  pool growth + runaway log detection
#                               webgui_watchdog.sh   WebGUI availability (nginx → php-fpm → emhttp)
#                               network_watchdog.sh  internet, DDNS, Tailscale, NPM proxy
#   4. stability_watchdog.sh  reboot if all else failed — last line of defense
#
# Startup grace: no action until uptime >= WATCHDOG_STARTUP_GRACE (600s). Exits early
# so all three sub-scripts don't each log startup-grace skips on every cycle.
# Overlap protection: acquire_lock() exits immediately if a prior cycle is still running —
# prevents pile-up when a restart attempt or daemon check takes longer than 15 minutes.
#
# bash /boot/config/plugins/varaverk/Orchestrators/watchdog_orchestrator.sh
# bash /boot/config/plugins/varaverk/Orchestrators/watchdog_orchestrator.sh --dry-run
# bash /boot/config/plugins/varaverk/Orchestrators/watchdog_orchestrator.sh --status


# ── TRANSCODE MANAGEMENT ──────────────────────────────────────────────────────────────────────
# Schedule:   */7 * * * *  (every 7 minutes)
# Background: YES
#
# Cleanup first, then manager — order is non-negotiable. Stale segment files are removed
# BEFORE the manager checks ramdisk usage, so threshold decisions are based on real active
# session load, not inflated counts from sessions that already ended.
#
# transcode_cleanup.sh:
#   Remove HLS segment files older than TRANSCODE_MAX_AGE (20min) not open by any process.
#   One lsof call per location (not one per file — critical on busy Live TV systems).
#   transcoding-temp directory NEVER deleted even when empty (prevents SSD session drift).
#   After cleanup: triggers flip-back to ramdisk if usage dropped below RAMDISK_LOW_GB.
#
# transcode_manager.sh:
#   Check ramdisk usage vs RAMDISK_WARN_GB. Flip /mnt/ram-transcode symlink if needed.
#   Hysteresis: WARN→SSD flip, usage must drop to LOW before flipping back (prevents oscillation).
#   Safety every cycle: missing symlink → recreate, ramdisk gone → auto-flip SSD,
#   missing transcoding-temp → recreate immediately.
#   Show active Emby sessions with play method. Write daily stats to TRANSCODE_DAILY_LOG.
#
# NOTE: Emby Extra Parameters MUST include:
#   --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode
#   Do NOT add bind-propagation=shared — runc v1.3.5+ (Docker 29.x, Unraid 7.3+) rejects it.
#   The ramdisk mount is already shared at the kernel level (MS_SHARED) — propagation is automatic.
#
# bash /boot/config/plugins/varaverk/Orchestrators/transcode_management.sh


# ── CRITICAL SYNC MAINTENANCE ─────────────────────────────────────────────────────────────────
# Schedule:   */30 * * * *  (every 30 minutes)
# Background: YES
#
# Keeps fallback-critical data current on HOST2. 30 minutes is the maximum staleness
# for the auth stack and Emby watch states. Run on HOST1 — bidirectional via detect_hosts().
#
# rsync Critical-Data:
#   Auth stack to HOST2: NPM proxy rules, TLS certs, LLDAP user accounts,
#   Authelia config and policies. HOST2 auth always within 30min of HOST1.
#
# play_state_sync:
#   API-based watch state and resume position sync. Runs as a CRITICAL_MAINTENANCE_SCRIPT
#   in this window. No rsync — uses Emby API to sync per-user watch state directly.
#
# partnership --check:
#   Reads remote state file. Increments offline counter on failed sync.
#   Triggers auto-offboard at PARTNERSHIP_OFFLINE_THRESHOLD (30 days) unreachable.
#
# Silent on clean cycles: runs 48 times per day, no output when healthy.
#
# bash /boot/config/plugins/varaverk/Orchestrators/critical_sync_maintenance.sh


# ── INTERMEDIATE SYNC MAINTENANCE ────────────────────────────────────────────────────────────
# Schedule:   0 */4 * * *  (every 4 hours)
# Background: YES
#
# Arr library reconciliation, artwork fetching, and optional mid-day rsync.
# Run on BOTH servers independently — each handles its own arr types.
#
# Execution order:
#   1. arr_sync.sh               bidirectional Lidarr/Sonarr/Radarr mesh — all nodes agree
#                                Union model: if any node tracks it, all nodes get it.
#                                Remote API keys read via SSH on the fly — never stored.
#   2. Rsync window (optional)   INTERMEDIATE_SYNC_SHARES — empty by default, skipped
#                                Add shares here if a subset needs mid-day propagation.
#   3. INTERMEDIATE_MAINTENANCE_SCRIPTS — 4-hour jobs (add any script that needs sub-daily cadence).
#                                arrs_failed_stalled_recovery.sh — blocklist + re-search failed
#                                  and stalled arr imports. Every 4 hours. Configured in master.conf.
#                                lidarr_missing_art.sh — idempotent artwork fetch. Skips existing files.
#
# Why separate from daily: arr libraries need to converge more than once a day. If a remote
# node adds something at 2am, waiting 23 hours means both arrs search for content already owned.
# 4-hour cycles close that gap. Artwork and failed imports catch up within 4 hours.
#
# Silent on success — runs 6x/day, only failures produce notification.
#
# bash /boot/config/plugins/varaverk/Orchestrators/intermediate_sync_maintenance.sh


# ── DAILY SYNC MAINTENANCE ────────────────────────────────────────────────────────────────────
# Schedule:   0 1 * * *  (1am daily — runs on BOTH servers independently)
# Background: YES
#
# Full nightly maintenance window. detect_hosts() sets correct sync direction per server.
# HOST1 pushes: Movies, Tv_Shows, Music, Kids content, Sports, stand-up comedy → HOST2
# HOST2 pushes: Anime_Shows, Anime_Movies → HOST1
# Personal encrypted shares: appended after standard shares — remote receives ciphertext only.
#
# Execution order (order is not arbitrary — each step depends on the previous):
#   1. git_pull_execute.sh           pull latest scripts FIRST — everything below runs on current code
#   2. rsync DAILY_SYNC_SHARES       push this server's truth shares to other server
#        Temp exit codes: 1=WARN (skip share, continue), 2=CRIT (abort all syncs, notify)
#   3. media_shares_permissions.sh   apply nobody:users 755/664 — arr cleanup depends on this
#   4. media_cleaner.sh anime        remove scene junk BEFORE orphan detection (cleaner results)
#      media_cleaner.sh media
#   5. lidarr_cleanup.sh             delete orphaned media — 7 safety checks before any deletion
#      sonarr_cleanup.sh             notify_emby_scan() after each — ghost entries cleared immediately
#      radarr_cleanup.sh
#   6. docker_update.sh              pull latest images for DAILY_RESTART_CONTAINERS (no downtime)
#   7. docker_daily_restart.sh       restart: NPM, Authelia, Dispatcharr, Dispatcharr-Basic, ErsatzTV-Emby
#
# Notify: silent on success. One failure notification with job list on any failure.
#
# bash /boot/config/plugins/varaverk/Orchestrators/daily_sync_maintenance.sh


# ── WEEKLY SYNC MAINTENANCE ───────────────────────────────────────────────────────────────────
# Schedule:   30 2 * * 0  (Sunday 2:30am — 4.5 hours before coffee report)
# Background: YES
#
# Full Sunday maintenance window. Containers stop for clean sync. Image updates happen
# at zero extra downtime since containers are already stopped for the rsync anyway.
#
# Execution order:
#   1. Pre-flight checks        connectivity, remote Docker daemon, remote rootfs space
#   2. Stop local containers    Emby + auth stack on this server
#   3. Stop remote containers   Emby + auth stack on remote via SSH
#   4. Pull images locally      if WEEKLY_SYNC_UPDATES=true (already stopped = free downtime)
#   5. Pull images remotely     if WEEKLY_SYNC_UPDATES_REMOTE=true (SSH to remote)
#   6. rsync WEEKLY_SYNC_SHARES
#        Emby        full clean mirror — databases checkpointed on shutdown, all config included
#        Critical-Data full clean auth stack mirror
#   7. Start remote containers  dependency order, new image, verify each container up
#   8. Start local containers   dependency order, new image, verify each container up
#   9. docker_update.sh --weekly  pull latest images for WEEKLY_RESTART_CONTAINERS
#  10. docker_weekly_restart.sh  restart less-critical services: NextCloud, AdGuard, Immich
#  11. WEEKLY_MAINTENANCE_SCRIPTS — clear_logs.sh + playback_aware_*.sh discovery scripts.
#                                playback_aware_lidarr_discovery.sh: 0–5 music adds/week
#                                  based on your Emby play history + Last.fm similar artists.
#                                playback_aware_radarr_discovery.sh: 0–5 movie adds/week
#                                  based on recently watched + TMDB recommendations.
#                                playback_aware_sonarr_discovery.sh: 0–3 TV adds/week
#                                  based on recently watched + TMDB recommendations.
#
# Why weekly not nightly for Emby: HOST2 builds warm thumbnail cache throughout the week.
# Nightly sync resets that cache — cold loads every morning. Weekly: cache warm 6 days,
# resets Sunday night while users sleep.
#
# bash /boot/config/plugins/varaverk/Orchestrators/weekly_sync_maintenance.sh


# ── MONTHLY MAINTENANCE ───────────────────────────────────────────────────────────────────────
# Schedule:   0 0 15 * *  (15th of each month at midnight)
# Background: YES
#
# Runs heavy tasks that require a long-stable, settled system. Fires only when BOTH gates pass:
#   1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS (default: 30 days)
#   2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS ago (default: 30 days)
#
# If either gate is not met on the 15th, the run is skipped until next month's 15th.
# A reboot between the 14th and 15th will defer maintenance — which is intentional;
# heavy tasks (ZFS scrub, SMART long test) should not run on a freshly rebooted system.
#
# Configure via master.conf MONTHLY_MAINTENANCE_SCRIPTS — see Tools/zfs_pool_scrub.sh and Tools/smart_long_test.sh.
#
# bash /boot/config/plugins/varaverk/Orchestrators/monthly_maintenance.sh
# bash /boot/config/plugins/varaverk/Orchestrators/monthly_maintenance.sh --dry-run
# bash /boot/config/plugins/varaverk/Orchestrators/monthly_maintenance.sh --status
# bash /boot/config/plugins/varaverk/Orchestrators/monthly_maintenance.sh --force


# ── SUNDAY MORNING COFFEE REPORT ─────────────────────────────────────────────────────────────
# Schedule:   0 7 * * 0  (Sunday 7am — after 2:30am maintenance completes)
# Background: YES
#
# Full weekly system overview. Reads data from the week — state files, logs, APIs, live checks.
#
# Sections:
#   System      uptime, memory, boot/cache drive usage, watchdog-triggered reboots this week
#   Array       disk count, parity status, ZFS pool health, all drive temperatures
#   Transcodes  weekly ramdisk peak, flip count, ramdisk vs SSD session split
#   Media       arr orphan stats (count removed, GB freed), current queue depth per arr
#   Rsync       weekly transfer totals, per-profile breakdown, failures, large transfers
#   Watchdog    system + docker watchdog running state, active strikes, skip list
#   Security    SSL cert days remaining per domain (live TLS connection — not NPM's API)
#   Emby        version, active streams now (direct vs transcode), weekly play count, top users
#   Health      inotify peak/avg/warnings, php-fpm peak/avg, SMART pass/fail, git sync state
#   Issues      anything requiring attention collected from above sections
#
# Footer: ✅ enjoy your Sunday ☕  OR  ⚠️ N issue(s) need attention
# Delivery: unRAID notification + Discord webhook (if DISCORD_WEBHOOK configured)
#
# bash /boot/config/plugins/varaverk/Orchestrators/sunday_morning_coffee_report.sh


# ── WEEKLY HEALTH DIGEST ─────────────────────────────────────────────────────────────────────
# Schedule:   0 8 * * *  (8am daily — DIGEST_PROFILE controls when it actually notifies)
# Background: YES
#
# Reads ecosystem state files and compiles a digest notification. Read-only, writes nothing.
# Schedule never changes — switch behaviour by changing DIGEST_PROFILE in master.conf only.
#
# DIGEST_PROFILE options (master.conf):
#   always    notify every run regardless — daily digest even when healthy
#   smart     notify only when something worth reporting is found:
#               active watchdog strikes, non-NORMAL fallback state, cert within CERT_WARN_DAYS,
#               bandwidth event exceeded BANDWIDTH_WARN_GB
#   weekly    notify once per week on DIGEST_DAY (Sunday), silent all other days
#
# Data sources: fallback state, watchdog strike files, container skip list,
#               bandwidth log, transcode daily log, cert expiry (live openssl check)
#
# bash /boot/config/plugins/varaverk/Monitors/weekly_health_digest.sh


# ── SYSTEM TUNING MONITOR ────────────────────────────────────────────────────────────────────
# Schedule:   0 */6 * * *  (every 6 hours)
# Background: YES
#
# Snapshots inotify and PHP-FPM utilisation. Writes one bounded log entry.
# sunday_morning_coffee_report.sh reads this log for weekly peak/avg/warning counts.
#
# inotify: current instances vs kernel limit. Warns above INOTIFY_WARN_PCT (80%).
#   Exhaustion symptom: downloads complete but arrs don't detect them, Live TV stutters,
#   library updates stop — no obvious error message anywhere.
#
# php-fpm: active workers vs PHP_MAX_CHILDREN. Warns above PHP_FPM_WARN_PCT (80%).
#   Exhaustion symptom: WebGUI slows, settings saves hang, container UI starts timeout.
#
# Log: TUNING_MONITOR_LOG — bounded, trimmed to TUNING_LOG_RETENTION days on each write.
#
# bash /boot/config/plugins/varaverk/Monitors/system_tuning_monitor.sh


# ── SUNDAY MONITOR BLOCK ─────────────────────────────────────────────────────────────────────
# These monitors are now driven by sunday_morning_coffee_report.sh via COFFEE_REPORT_SCRIPTS.
# No separate User Script entries needed — configure which monitors run in master.conf.
# All monitors are read-only — observe and report, never act. Still safe to run individually.
#
# Current COFFEE_REPORT_SCRIPTS (master.conf):
#   Monitors/zfs_memory_snapshot.sh     ZFS pool health + ARC + Docker memory snapshot
#   Monitors/smart_health.sh            drive SMART attributes — reallocated, pending, temp
#   Monitors/cert_monitor.sh            SSL certificate expiry for all configured domains
#   Monitors/backup_verify.sh           rsync mirror integrity via independent MD5 checksums
#   Monitors/bandwidth_monitor.sh       weekly rsync transfer totals and per-share breakdown
#   Monitors/emby_session_report.sh     Emby usage — streams, users, library, transcode ratio
#   Monitors/weekly_health_digest.sh    aggregated digest — watchdogs, fallback, skip list


# ==============================================================================================
# ██  INDIVIDUAL SCRIPTS — STANDALONE AND SELECTIVE USE  ███████████████████████████████████████
# ==============================================================================================
#
# Everything below is called by the orchestrators above when using the full ecosystem.
# Listed here so you can:
#   → Run one script manually without triggering the full orchestrator
#   → Use only specific parts of the ecosystem (just arr cleanup, just transcode manager)
#   → Test individual scripts before scheduling the orchestrator
#   → Schedule individual scripts on different timing than the orchestrator uses
#
# Every script is fully standalone — sources its own config, runs its own pre-flight checks.
#
# ==============================================================================================


# ──────────────────────────────────────────────────────────────────────────────────────────────
# FALLBACK
# ──────────────────────────────────────────────────────────────────────────────────────────────

# fallback.sh — [continuous] mutual fallback state machine
# Started by array_started.sh on both servers independently.
# Every FALLBACK_CHECK_INTERVAL (120s) pings: remote Tailscale IP + 8.8.8.8
# States: NORMAL / FALLBACK / NO_INTERNET / DARK
# FALLBACK: starts remote containers in tiers across 24 hours:
#   Tier 1 immediate: DDNS + Emby + auth (NPM/Authelia/LLDAP) + VaultWarden + Live TV
#   Tier 2 at 4hr:   NextCloud + Immich + Jellyseerr
#   Tier 3 at 12hr:  AdGuard + Gitea + uptime monitoring
#   Tier 4 at 24hr:  arrs + downloaders
# Auth stack (NPM, Authelia, LLDAP) runs warm on both servers always — no fallback startup delay.
# DDNS: one domain, one DDNS container active at all times. Split-brain prevented by design.
# Handback: stop remote DDNS → rsync writeback → start containers → start local DDNS.
#
# bash /boot/config/plugins/varaverk/Fallback/fallback.sh
# bash /boot/config/plugins/varaverk/Fallback/fallback.sh --status
# bash /boot/config/plugins/varaverk/Fallback/fallback.sh --dry-run

# fallback_test.sh — controlled full-lifecycle fallback test using iptables simulation
# ALWAYS run --dry-run first. Live test starts/stops real containers — brief service interruption.
# iptables safety trap removes the block rule on ANY exit including ctrl-c and crashes.
# Remote is always reachable after the test completes regardless of what happened.
# 8 phases: pre-flight → block remote (iptables) → detect FALLBACK → verify Tier 1 started
#           → restore remote → wait handback → verify NORMAL restored → full pass/fail report.
# FALLBACK_TEST_BLOCK_WAIT must be > FALLBACK_CHECK_INTERVAL + buffer (default: 150s).
# FALLBACK_TEST_HANDBACK_WAIT must cover: strike confirmation + rsync + container start (default: 360s).
#
# bash /boot/config/plugins/varaverk/Fallback/fallback_test.sh --dry-run
# bash /boot/config/plugins/varaverk/Fallback/fallback_test.sh

# fallback_state_reset.sh — reset fallback state file to NORMAL
# Resets state file ONLY — does NOT start or stop any containers.
# Use when: state stuck in FALLBACK after testing, after killing fallback.sh mid-cycle,
# after manual intervention left state inconsistent, after a failed handback.
# Shows current state file before asking for confirmation.
# Verify before resetting: right containers on right server, DDNS correct,
# both servers Tailscale visible, fallback.sh not currently running.
#
# bash /boot/config/plugins/varaverk/Tools/fallback_state_reset.sh --status
# bash /boot/config/plugins/varaverk/Tools/fallback_state_reset.sh
# bash /boot/config/plugins/varaverk/Tools/fallback_state_reset.sh --force


# ──────────────────────────────────────────────────────────────────────────────────────────────
# RSYNC
# ──────────────────────────────────────────────────────────────────────────────────────────────

# rsync.sh — core sync script, call with share path and optional --profile=name
# Pre-flight every run: connectivity, version parity, remote Docker daemon,
# remote rootfs space, remote share exists. Profile matched by directory basename (lowercased).
# Drive temperature: reads thresholds from dynamix.cfg. Exit 1=WARN (skip), 2=CRIT (caller aborts).
# Logs each transfer to BANDWIDTH_LOG automatically (read by bandwidth_monitor --report).
#
# Profile behaviour — what containers stop:
#   critical-data    Mariadb-Authelia, Redis-Authelia, NginxProxyManager, Lldap
#                    + delayed restart: Authelia, Authelia-Secondary (30s after dependencies)
#   important-data   Postgres-NextCloud + delayed: NextCloud
#   arrs_stack       Sonarr, Radarr, Lidarr, Prowlarr, Bazarr, Pinchflat
#   emby             Emby both sides (weekly clean sync — both instances stopped)
#   [no profile]     no containers stopped (media shares, plain data)
#
# bash /boot/config/plugins/varaverk/Rsync/rsync.sh \
#     /mnt/user/appdata-Fallback/Critical-Data
# bash /boot/config/plugins/varaverk/Rsync/rsync.sh \
#     /mnt/user/appdata-Fallback/Important-Data
# bash /boot/config/plugins/varaverk/Rsync/rsync.sh \
#     /mnt/user/appdata-Fallback/Arrs_Stack
# bash /boot/config/plugins/varaverk/Rsync/rsync.sh \
#     /mnt/user/Media_Server/Emby --profile=emby
# bash /boot/config/plugins/varaverk/Rsync/rsync.sh \
#     /mnt/user/appdata-Fallback/Gmer4Lfe
# bash /boot/config/plugins/varaverk/Rsync/rsync.sh /mnt/user/Movies
# bash /boot/config/plugins/varaverk/Rsync/rsync.sh /mnt/user/Tv_Shows
# bash /boot/config/plugins/varaverk/Rsync/rsync.sh /mnt/user/Music
# bash /boot/config/plugins/varaverk/Rsync/rsync.sh /mnt/user/Anime_Shows
# bash /boot/config/plugins/varaverk/Rsync/rsync.sh /mnt/user/Anime_Movies

# rsync_stop.sh — stop rsync intelligently on local and remote
# Smart mode (default): kills rsync subprocess only, orchestrator exits cleanly,
#   any stopped containers are recovered by docker_watchdog.sh on next cycle.
# --full-stop: kills orchestrator first then rsync — emergency full stop.
# --rsync-only: stop rsync, skip container recovery check (used by partnership_manager --offboard).
#
# bash /boot/config/plugins/varaverk/System_Essentials/rsync_stop.sh --status
# bash /boot/config/plugins/varaverk/System_Essentials/rsync_stop.sh
# bash /boot/config/plugins/varaverk/System_Essentials/rsync_stop.sh --full-stop


# ──────────────────────────────────────────────────────────────────────────────────────────────
# DOCKER ESSENTIALS
# ──────────────────────────────────────────────────────────────────────────────────────────────

# docker_watchdog.sh — two-tier container self-healing monitor (single-pass)
# Called every 15 minutes by watchdog_orchestrator.sh — NOT started by array_started.sh. Single-pass.
#
# Tier 1 — explicit per-container (configured in host*.conf):
#   Memory hard limits:  immediate restart when exceeded — no strikes, no waiting
#   CPU strike system:   2 consecutive cycles above HARD_CPU_THRESHOLD → restart
#   HTTP health checks:  curl to configured URL — 2 consecutive failures → restart
#   Required containers: found stopped → restart attempt → failure → skip list
#   Dependency ordering: database must be running before dependent is restarted
#
# Tier 2 — global scan of ALL running containers (when WATCHDOG_SCAN_ALL=true):
#   HEALTHCHECK unhealthy → restart
#   OOM kill detected → restart + notify
#   Crash loop (RestartCount climbing) → notify, above WATCHDOG_CRASH_LIMIT → restart → skip list
#   Dead state → remove + start
#   Non-zero exit → restart
#
# Startup grace: WATCHDOG_STARTUP_GRACE (600s) — restarts suppressed, checks still log
# Restart loop: 3 restarts in 1hr → skip list on /boot/config → critical notification
# RAM emergency: reads mem_shutdown_active from system_watchdog state file, defers all restarts
# Silent on clean cycles — only outputs events and hourly heartbeat
#
# bash /boot/config/plugins/varaverk/Watchdogs/docker_watchdog.sh --status
# bash /boot/config/plugins/varaverk/Watchdogs/docker_watchdog.sh --dry-run

# watchdog_skip_list_manager.sh — view and manage the container skip list
# When docker_watchdog restarts the same container 3 times in 1hr → skip-listed.
# Skip list persists on /boot/config — survives reboots. Still broken after reboot = stays listed.
# Auto-clear: watchdog removes a container from the skip list when it sees it running.
# Manual clear needed only when container is stuck stopped after you've fixed the problem.
#
# Recovery workflow:
#   --status → understand situation (skip list, running/stopped state, restart history)
#   fix the underlying problem (check docker logs, disk space, database integrity)
#   --clear ContainerName → remove from skip list + clear restart history
#   docker start ContainerName → confirm fix works before handing back to watchdog
#   watchdog resumes normal monitoring on next cycle automatically
#
# bash /boot/config/plugins/varaverk/Tools/watchdog_skip_list_manager.sh --status
# bash /boot/config/plugins/varaverk/Tools/watchdog_skip_list_manager.sh --clear ContainerName
# bash /boot/config/plugins/varaverk/Tools/watchdog_skip_list_manager.sh --clear ContainerName --force
# bash /boot/config/plugins/varaverk/Tools/watchdog_skip_list_manager.sh --clear-all
# bash /boot/config/plugins/varaverk/Tools/watchdog_skip_list_manager.sh --dry-run

# docker_container_stop.sh — stop all running containers sequentially with verification
# Called by array_stopping.sh as the final step in the planned shutdown sequence.
# Safe to run standalone before array maintenance, disk replacement, or planned reboots.
# Never moves to the next container until the current one is confirmed stopped.
#
# Stop sequence per container:
#   docker stop -t 30 (SIGTERM + 30s grace — docker sends SIGKILL if needed)
#   Retry up to RETRY_COUNT times if still running
#   docker kill (SIGKILL) if all retries exhausted
#   Notify on any container that cannot be stopped
#
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_container_stop.sh --status
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_container_stop.sh --dry-run
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_container_stop.sh

# docker_daily_restart.sh — restart degradation-prone containers nightly
# Called by daily_sync_maintenance.sh. Also safe to run ad hoc.
# Running → docker restart (graceful). Stopped → left stopped (state respected). Missing → skip.
# Dependency ordering via WATCHDOG_DEPENDENCIES — databases before applications.
# Restart verification: checks container still up after settle period, notifies if not.
# Configured via HOST*_DAILY_RESTART_CONTAINERS (host*.conf):
#   NginxProxyManager, Authelia, Dispatcharr, Dispatcharr-Basic, ErsatzTV-Emby
#
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_daily_restart.sh --dry-run
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_daily_restart.sh --status
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_daily_restart.sh

# docker_weekly_restart.sh — restart less-critical services weekly
# Called by weekly_sync_maintenance.sh after sync completes and containers are back up.
# Targets services that benefit from weekly clean start but don't stop for the sync itself.
# Same rules as daily: running→restart, stopped→leave, missing→skip.
# Configured via HOST*_WEEKLY_RESTART_CONTAINERS (host*.conf):
#   NextCloud, AdGuard-Home, Immich
#
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_weekly_restart.sh --dry-run
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_weekly_restart.sh --status
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_weekly_restart.sh

# docker_network_connect.sh — ensure custom networks exist and containers are connected
# Called by array_started.sh. Idempotent — silent when everything is already correct.
# Network missing → create (bridge driver, auto-assigned subnet) → notify.
# Network creation should only happen after an unRAID update wiped networks — notify tells you.
# Configured via HOST*_NETWORK_CONNECT_NETWORKS and HOST*_NETWORK_CONNECT_CONTAINERS.
# Use after unRAID updates when containers can't reach each other on custom networks.
#
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_network_connect.sh --status
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_network_connect.sh --dry-run
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_network_connect.sh

# docker_update.sh — pull latest images for managed containers, rebuild those that got a new digest
# Two modes, one script:
#
#   Normal (daily) — targets HOST*_DAILY_RESTART_CONTAINERS.
#     Containers stay running during the pull. Rebuilds containers that received a new image.
#     docker_daily_restart.sh restarts the rest. Toggle: DAILY_CONTAINER_UPDATES in master.conf.
#
#   Weekly (--weekly) — targets WEEKLY_RESTART_CONTAINERS before docker_weekly_restart.sh.
#     Rebuilds containers that received a new image. Toggle: WEEKLY_CONTAINER_UPDATES in master.conf.
#
#   Remainder (--remainder, monthly) — targets all running containers not in managed lists.
#     Excludes daily, weekly, emby+critical-data sync-window, and fallback containers.
#     Rebuilds containers that received a new image. Toggle: MONTHLY_REMAINING_UPDATES in master.conf.
#
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh --dry-run
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh --status
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh --remainder
# bash /boot/config/plugins/varaverk/Docker_Essentials/docker_update.sh --remainder --dry-run

# downloaders_reset.sh — maintenance reset for all download clients on this server
# Called every 30 minutes by critical_sync_maintenance.sh via CRITICAL_MAINTENANCE_SCRIPTS.
# Host-aware: sections skip gracefully if a downloader URL is empty for this server.
#
# slskd:
#   Stuck searches    — clears Completed/Errored searches left by Soularr crashes
#   Dead transfers    — removes completed/errored/aborted transfer records per user
#                       NEVER removes InProgress or Queued transfers
#   Failed imports    — purges albums Soularr downloaded but Lidarr rejected
# SABnzbd:
#   Completed history — removes completed records older than DOWNLOADER_RETENTION_DAYS
#   Failed history    — removes failed records older than DOWNLOADER_RETENTION_DAYS
#   Stalled queue     — removes Paused or Stuck queue items (never active downloads)
# qBittorrent:
#   Age failsafe      — removes torrents older than QBIT_FAILSAFE_MIN_DAYS
#                       deleteFiles=false — leaves files for arrs to manage
#
# bash /boot/config/plugins/varaverk/Docker_Essentials/downloaders_reset.sh --dry-run
# bash /boot/config/plugins/varaverk/Docker_Essentials/downloaders_reset.sh --status
# bash /boot/config/plugins/varaverk/Docker_Essentials/downloaders_reset.sh


# ──────────────────────────────────────────────────────────────────────────────────────────────
# UNRAID ESSENTIALS
# ──────────────────────────────────────────────────────────────────────────────────────────────

# system_watchdog.sh — third layer in the watchdog chain: runs SYSTEM_WATCHDOG_SCRIPTS each cycle
# Called by watchdog_orchestrator.sh — NOT scheduled directly. Single-pass.
# Iterates SYSTEM_WATCHDOG_SCRIPTS sequentially: storage_watchdog → webgui_watchdog → network_watchdog.
# Each sub-script is independent — a failure in one does not skip the others.
#
# Tier 1 CRITICAL — bypass ALL strikes, reboot immediately:
#   Docker daemon hung     → attempt rc.docker restart → still hung → reboot
#   rootfs >= 99%          → reboot (writes failing, system cannot recover)
#   Kernel BUG/Oops delta  → reboot (kernel running with corrupted state)
#   FD exhaustion >= 95%   → reboot (new connections failing, Docker can't spawn)
#   /boot read-only        → reboot (state file writes silently failing)
#
# Tier 2 URGENT — bypass strikes when OOM confirms RAM crisis:
#   RAM < MEM_GB (4GB) AND >= OOM_LIMIT (3) OOM kills this cycle → reboot
#   Without OOM confirmation: standard strike system applies.
#
# Tier 3 STANDARD — N consecutive failures → reboot:
#   RAM tiers: warn (10GB), stop containers (6GB, excluded list respected), reboot (4GB)
#   Load average, CPU temperature, /var/log usage, /tmp usage, zombie processes,
#   sshd, NIC operstate, mdstat errors, kernel oops rate, FD rate
#
# RAM emergency: writes mem_shutdown_active=true → docker_watchdog defers all restarts.
# Reboot loop protection: N reboots in X hours → shutdown instead.
# State file heartbeat: writes watchdog_cycle=N every cycle (docker_watchdog stale guard).
#
# bash /boot/config/plugins/varaverk/Watchdogs/stability_watchdog.sh --status
# bash /boot/config/plugins/varaverk/Watchdogs/stability_watchdog.sh --dry-run

# storage_watchdog.sh — pool growth + runaway log detection (single-pass, called by system_watchdog)
# Part of SYSTEM_WATCHDOG_SCRIPTS — not scheduled directly. Called by system_watchdog.sh each cycle.
#
# Two complementary checks every cycle:
#   Growth rate (zero-config): du per container vs previous baseline — any container growing
#     more than WATCHDOG_APPDATA_GROWTH_GB triggers a focused *.log scan inside that dir.
#     New containers covered automatically from their second cycle. No per-container config.
#   Absolute log size: finds *.log files over WATCHDOG_APPDATA_LOG_MAX_GB anywhere in
#     WATCHDOG_APPDATA_PATHS. Catches logs already large but not actively growing.
#
# Strike system (same pattern as docker_watchdog CPU/HTTP strikes):
#   Strike 1 → warn + notify. Strike 2 → escalated notify. Strike 3 → action:
#   WATCHDOG_APPDATA_TRUNCATE_LOGS=true  → truncate *.log in-place, clear strikes
#   WATCHDOG_APPDATA_TRUNCATE_LOGS=false → critical notify, hold strikes until resolved
# Condition resolves (growth stops / log shrinks) → strikes auto-clear.
#
# bash /boot/config/plugins/varaverk/Watchdogs/System/storage_watchdog.sh --status
# bash /boot/config/plugins/varaverk/Watchdogs/System/storage_watchdog.sh --dry-run

# network_watchdog.sh — internet + DDNS + Tailscale + NPM proxy checks (called by system_watchdog)
# Part of SYSTEM_WATCHDOG_SCRIPTS — not scheduled directly. Called by system_watchdog.sh each cycle.
#
# Check 1 — Internet: curl external endpoint. Fail → alert + skip all remaining (prevents cascades).
# Check 2 — DDNS (Cloudflare): public IP via ifconfig.me vs DNS record via dig @1.1.1.1.
#   Mismatch → restart DDNS container (triggers immediate Cloudflare record update) + notify.
# Check 3 — Tailscale: tailscale status → Running → pass. Not running → notify (no restart — human).
# Check 4 — NPM proxy (external): curl external URL through the full stack.
#   2-strike system: Strike 1 = warn + notify. Strike 2 = restart NginxProxyManager + notify.
#   Strikes auto-clear when the external URL becomes reachable again.
#
# bash /boot/config/plugins/varaverk/Watchdogs/System/network_watchdog.sh --status
# bash /boot/config/plugins/varaverk/Watchdogs/System/network_watchdog.sh --dry-run

# resource_watchdog.sh — three-level pressure reduction layer (single-pass, called by watchdog_orchestrator)
# Reduces system load intelligently BEFORE docker_watchdog attempts container restarts.
# Containers restarted into a resource-pressured system just fail again — this runs first.
#
# Three pressure levels (thresholds configured in master.conf):
#   Level 1 SOFT   (low RAM or high load): throttle SABnzbd + qBittorrent download speeds
#   Level 2 MEDIUM (lower RAM or higher load): further throttle + docker pause non-critical containers
#   Level 3 HARD   (critically low RAM): docker stop heavy services + signal docker_watchdog to defer restarts
#
# Recovery: pressure must stay below current threshold for RW_RECOVER_CYCLES consecutive runs
# before de-escalating. One level at a time — prevents flip-flopping.
# Coordination: at Level 3 writes mem_shutdown_active=true → docker_watchdog defers all restarts.
# HOST*_RW_PAUSE_CONTAINERS and HOST*_RW_STOP_CONTAINERS configured per host in host*.conf.
#
# bash /boot/config/plugins/varaverk/Watchdogs/resource_watchdog.sh --status
# bash /boot/config/plugins/varaverk/Watchdogs/resource_watchdog.sh --dry-run
# bash /boot/config/plugins/varaverk/Watchdogs/resource_watchdog.sh

# inotify_tuning.sh — raise Linux inotify kernel limits at array start
# Called by array_started.sh FIRST — must run before containers start (they inherit limits).
# Settings reset on each reboot — script reapplies on every array start. Idempotent.
#
# Limits set:
#   max_user_watches   → 1048576 (default 8192 — VSCode alone needs 50-200K for large projects)
#   max_user_instances → 1024    (default 128)
#   max_queued_events  → 32768   (default 16384)
#
# If Code-Server started before this ran → docker restart Code-Server to inherit new limits.
# Exhaustion symptom: downloads complete but arrs don't detect them, Live TV stutters.
#
# bash /boot/config/plugins/varaverk/System_Essentials/inotify_tuning.sh --status
# bash /boot/config/plugins/varaverk/System_Essentials/inotify_tuning.sh

# php_fpm_max_children.sh — set PHP-FPM pm.max_children at array start
# Called by array_started.sh. Default is 4-8 workers — inadequate for a busy multi-user server.
# Sets PHP_MAX_CHILDREN (250). 250 × ~2MB idle = ~500MB. Acceptable on 64GB+.
# Symptom of saturation: WebGUI slow, settings saves hang, container UI starts timeout.
# Resets on each reboot — reapplied at array start. Idempotent: silent when already correct.
#
# bash /boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/php_fpm_max_children.sh --status
# bash /boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/php_fpm_max_children.sh

# docker_syslog_filter.sh — suppress Docker veth/docker0 interface log noise
# Called by array_started.sh before containers start. Creates rsyslog drop rule.
# Without this: 50+ containers at array start = 200-400 lines of kernel veth messages.
# Real events (mount failures, permission errors) are invisible in that noise.
# Idempotent: compares expected filter content exactly — only writes when changed.
#
# bash /boot/config/plugins/varaverk/System_Essentials/docker_syslog_filter.sh --status
# bash /boot/config/plugins/varaverk/System_Essentials/docker_syslog_filter.sh

# webgui_watchdog.sh — WebGUI availability watchdog (called by system_watchdog via SYSTEM_WATCHDOG_SCRIPTS)
# Not scheduled directly — runs as part of the every-15-minute watchdog chain.
# Escalation: nginx → php-fpm → emhttp, each with recheck before proceeding to next level.
# bash /boot/config/plugins/varaverk/Plugin/unraid/Watchdogs/System/webgui_watchdog.sh --dry-run
# bash /boot/config/plugins/varaverk/Plugin/unraid/Watchdogs/System/webgui_watchdog.sh --status

# clear_logs.sh — size-threshold log cleanup (called by weekly_sync_maintenance via WEEKLY_MAINTENANCE_SCRIPTS)
# Not scheduled directly — runs as part of the Sunday 2:30am weekly window.
# System logs only cleared if > LOG_MIN_SIZE_MB (10MB). Docker logs only if > LOG_DOCKER_MAX_MB (100MB).
# bash /boot/config/plugins/varaverk/System_Essentials/clear_logs.sh --dry-run
# bash /boot/config/plugins/varaverk/System_Essentials/clear_logs.sh --status

# mover_stop.sh — stop unRAID mover cleanly before array operations
# Wall message to logged-in users. Waits MOVER_STOP_TIMEOUT seconds.
# SIGTERM (allows mover to finish current file — no partial files). Verify. SIGKILL last resort.
# Use before: planned reboots with mover running, disk replacement, array maintenance.
#
# bash /boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/mover_stop.sh --status
# bash /boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/mover_stop.sh --dry-run
# bash /boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/mover_stop.sh

# server_reboot.sh — graceful reboot with pre-flight warnings and clean shutdown sequence
# Pre-flight warnings (inform not block): rsync running, mover running, active Emby sessions.
# Sequence: wall + unRAID notification → wait REBOOT_SLEEP (30s) → virsh shutdown VMs
#           → wait REBOOT_VM_WAIT → stop libvirt → stop Docker → sync → /sbin/reboot
# --dry-run walks the full sequence without rebooting — shows all active processes.
#
# bash /boot/config/plugins/varaverk/System_Essentials/server_reboot.sh --dry-run
# bash /boot/config/plugins/varaverk/System_Essentials/server_reboot.sh --status
# bash /boot/config/plugins/varaverk/System_Essentials/server_reboot.sh

# user_scripts_stop.sh — stop all running User Script processes
# Finds processes by /tmp/user.scripts path signature. Shows script names, not just PIDs.
# SIGTERM → verify → SIGKILL. Self-exclusion prevents killing itself.
# Use when plugin Abort button didn't work, or before a reboot to clean up running scripts.
# Called automatically by server_reboot.sh before reboot.
#
# bash /boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/user_scripts_stop.sh --status
# bash /boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/user_scripts_stop.sh --dry-run
# bash /boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/user_scripts_stop.sh

# git_pull_execute.sh — pull latest scripts from Gitea and set execute permissions
# Deployment mechanism for the ecosystem. Push from VS Code → Gitea → run on both servers.
# Called automatically by daily_sync_maintenance.sh as the first job in the daily window.
#
# bash /boot/config/plugins/varaverk/git_pull_execute.sh


# ──────────────────────────────────────────────────────────────────────────────────────────────
# MEDIA
# ──────────────────────────────────────────────────────────────────────────────────────────────
#
# All media scripts run via daily_sync_maintenance.sh in the correct order:
#   permissions → cleaners → arr cleanup
#
# ORDER MATTERS: arr cleanup depends on correct ownership (permissions first) and clean
# folders (junk removal before orphan detection — otherwise junk files appear as orphans).
# Listed here for standalone testing or selective use outside the daily window.

# media_shares_permissions.sh — apply correct ownership and permissions to all media shares
# Applies nobody:users 755 (dirs) / 664 (files) to HOST*_MEDIA_PERMISSION_SHARES.
# Two separate passes: find -type d and find -type f — directories get 755, files get 664.
# Idempotent: only changes what's wrong. On large libraries takes 20-30 minutes.
# Common drift causes: scp/cp leaves root:root, container missing PUID=99 PGID=100,
# rsync without --chown flag, admin copies directly into share.
# Correct PUID/PGID in Docker template env vars is the permanent fix — this is the safety net.
#
# bash /boot/config/plugins/varaverk/Media/media_shares_permissions.sh --dry-run
# bash /boot/config/plugins/varaverk/Media/media_shares_permissions.sh --status
# bash /boot/config/plugins/varaverk/Media/media_shares_permissions.sh

# media_cleaner.sh — remove scene junk files from media share folders
# Run BEFORE arr cleanup — removes non-media files that would otherwise appear as orphans.
# Patterns: *.sfv *.md5 *.sha1 *.nfo *.url *.lnk *.rar *.zip *.info *.torrent
#           *.sample* *.proof* *sync-conflict* *.scr *.exe *.srr *.log *.json
# Two profiles with separate folder lists (configured in host*.conf):
#   anime  HOST*_ANIME_CLEAN_FOLDERS — anime share folders
#   media  HOST*_MEDIA_CLEAN_FOLDERS — Movies, Tv_Shows, Music, Sports etc.
# ALWAYS --dry-run when adding new patterns or folders — verify before committing.
#
# bash /boot/config/plugins/varaverk/Media/media_cleaner.sh anime --dry-run
# bash /boot/config/plugins/varaverk/Media/media_cleaner.sh media --dry-run
# bash /boot/config/plugins/varaverk/Media/media_cleaner.sh anime
# bash /boot/config/plugins/varaverk/Media/media_cleaner.sh media

# lidarr_cleanup.sh — delete orphaned music files not tracked by Lidarr
# HOST1 ONLY — HOST1 owns /mnt/user/Music and runs Lidarr. Exits cleanly on HOST2.
#
# Classification per file on disk:
#   TRACKED   → in Lidarr API response → never touch
#   PROTECTED → matches LIDARR_PROTECTED_PATTERNS → never touch
#               (*.jpg *.png = cover art, *.nfo = metadata, *.lrc = synced lyrics)
#   ORPHAN    → music extension, not tracked, older than LIDARR_ORPHAN_AGE (7 days),
#               not open by any process → DELETE
#   JUNK      → not a music extension, not protected → DELETE (any age)
#   RECENT    → not tracked, under 7 days → skip (may be mid-import)
#
# 7 safety checks ALL must pass before any deletion:
#   Lidarr running + healthy. API reachable. API version matches. Artist count > 0.
#   Tracked file count > 0. Count >= 80% of last known (detects database issues).
#   Total deletion size < LIDARR_MAX_DELETE_GB (50GB — catches path misconfigurations).
#
# After any deletion: notify_emby_scan() triggers Emby "Clean Missing Files" immediately.
# NUCLEAR MODE (use with full understanding): --i-know-what-im-doing --skip-strike-list
#   Both flags: age check bypassed + size gate bypassed. Deletes everything not tracked.
#
# bash /boot/config/plugins/varaverk/Arrs_Stack/lidarr_cleanup.sh --dry-run --log
# bash /boot/config/plugins/varaverk/Arrs_Stack/lidarr_cleanup.sh --status
# bash /boot/config/plugins/varaverk/Arrs_Stack/lidarr_cleanup.sh

# sonarr_cleanup.sh — delete orphaned TV files not tracked by Sonarr
# HOST-AWARE: HOST1 manages Tv_Shows (via HOST1_SONARR_TV_ROOT).
#             HOST2 manages Anime_Shows (via HOST2_SONARR_TV_ROOT).
#             detect_hosts() selects correct root — same script works on both servers.
# Same 7-check safety system as lidarr_cleanup.
# Protected: *.jpg *.png (episode/show artwork), *.nfo, *.srt *.sub *.ass *.ssa (Bazarr subtitles)
# After deletion: notify_emby_scan() clears ghost entries from Emby library immediately.
#
# bash /boot/config/plugins/varaverk/Arrs_Stack/sonarr_cleanup.sh --dry-run --log
# bash /boot/config/plugins/varaverk/Arrs_Stack/sonarr_cleanup.sh --status
# bash /boot/config/plugins/varaverk/Arrs_Stack/sonarr_cleanup.sh

# radarr_cleanup.sh — delete orphaned movie files not tracked by Radarr
# HOST-AWARE: HOST1 manages Movies (via HOST1_RADARR_MOVIES_ROOT).
#             HOST2 manages Anime_Movies (via HOST2_RADARR_MOVIES_ROOT).
# Same 7-check safety system as lidarr_cleanup.
# Protected: *.jpg *.png (movie artwork, fanart, thumbnails), *.nfo, subtitles
# After deletion: notify_emby_scan() clears ghost entries from Emby library immediately.
#
# bash /boot/config/plugins/varaverk/Arrs_Stack/radarr_cleanup.sh --dry-run --log
# bash /boot/config/plugins/varaverk/Arrs_Stack/radarr_cleanup.sh --status
# bash /boot/config/plugins/varaverk/Arrs_Stack/radarr_cleanup.sh

# arr_sync.sh — bidirectional arr library sync across all nodes in the ecosystem
# Called by intermediate_sync_maintenance.sh every 4 hours. Can be run standalone.
# Union mesh: every node syncs with every other — no primary, no hierarchy.
# If any node tracks an item, all nodes get it (unless blocklisted).
#
# What syncs: library items keyed on stable IDs (MusicBrainz / TVDB / TMDB).
#   Adding to a remote: that remote's own quality profile, metadata profile, and root
#   folder are used — settings are never copied from the source node.
# Remote API keys read via SSH from each arr's config.xml — never stored anywhere.
#   Key regeneration is picked up automatically on next run.
# Blocklist: ARR_SYNC_BLOCKLIST tombstones IDs that must never be re-added anywhere.
#   --blocklist-add does three things: tombstones, deletes local, deletes all remotes via SSH.
#   Files become orphans on all nodes — arr_cleanup.sh removes them on next run.
#
# bash /boot/config/plugins/varaverk/Arrs_Stack/arr_sync.sh --dry-run
# bash /boot/config/plugins/varaverk/Arrs_Stack/arr_sync.sh --status
# bash /boot/config/plugins/varaverk/Arrs_Stack/arr_sync.sh
# bash /boot/config/plugins/varaverk/Arrs_Stack/arr_sync.sh --blocklist-add radarr <tmdbId> "reason"
# bash /boot/config/plugins/varaverk/Arrs_Stack/arr_sync.sh --blocklist-add sonarr <tvdbId> "reason"
# bash /boot/config/plugins/varaverk/Arrs_Stack/arr_sync.sh --blocklist-add lidarr <mbid> "reason"
# bash /boot/config/plugins/varaverk/Arrs_Stack/arr_sync.sh --blocklist-remove radarr <id>
# bash /boot/config/plugins/varaverk/Arrs_Stack/arr_sync.sh --blocklist-list

# arrs_failed_stalled_recovery.sh — blocklist + re-search failed/stalled arr imports
# Called every 4 hours by intermediate_sync_maintenance.sh via INTERMEDIATE_MAINTENANCE_SCRIPTS.
# Finds importFailed / importPending / error / stalled items across all configured arrs.
# Age gate: items newer than ARR_IMPORT_RECOVERY_AGE (6hr) are skipped — arr may still self-retry.
# bash /boot/config/plugins/varaverk/Arrs_Stack/arrs_failed_stalled_recovery.sh --dry-run
# bash /boot/config/plugins/varaverk/Arrs_Stack/arrs_failed_stalled_recovery.sh --status

# playback_aware_lidarr_discovery.sh — behavior-driven music discovery (runs via WEEKLY_MAINTENANCE_SCRIPTS)
# Called by weekly_sync_maintenance.sh on Sunday. Can be run standalone.
# Stage 1: score artists you actually played this week in Emby (play count + recency + user diversity).
# Stage 2: run Last.fm artist.getSimilar on top seeds, score recommendations, add top results to Lidarr.
# Goal: 0–5 meaningful adds per run. Not bulk import — only artists likely to match your taste.
# bash /boot/config/plugins/varaverk/Arrs_Stack/playback_aware_lidarr_discovery.sh --dry-run
# bash /boot/config/plugins/varaverk/Arrs_Stack/playback_aware_lidarr_discovery.sh --status
# bash /boot/config/plugins/varaverk/Arrs_Stack/playback_aware_lidarr_discovery.sh

# playback_aware_radarr_discovery.sh — behavior-driven movie discovery (runs via WEEKLY_MAINTENANCE_SCRIPTS)
# Called by weekly_sync_maintenance.sh on Sunday. Can be run standalone.
# Stage 1: score recently watched movies in Emby, weighted by recency and TMDB rating.
# Stage 2: run TMDB recommendations on top seeds, score candidates, add top results to Radarr.
# Goal: 0–5 meaningful adds per run.
# bash /boot/config/plugins/varaverk/Arrs_Stack/playback_aware_radarr_discovery.sh --dry-run
# bash /boot/config/plugins/varaverk/Arrs_Stack/playback_aware_radarr_discovery.sh --status
# bash /boot/config/plugins/varaverk/Arrs_Stack/playback_aware_radarr_discovery.sh

# playback_aware_sonarr_discovery.sh — behavior-driven TV discovery (runs via WEEKLY_MAINTENANCE_SCRIPTS)
# Called by weekly_sync_maintenance.sh on Sunday. Can be run standalone.
# Stage 1: score recently watched series in Emby (user diversity + recency + per-user-capped episodes).
# Stage 2: run TMDB TV recommendations on top seeds, score candidates, add top results to Sonarr.
# Goal: 0–3 meaningful adds per run.
# bash /boot/config/plugins/varaverk/Arrs_Stack/playback_aware_sonarr_discovery.sh --dry-run
# bash /boot/config/plugins/varaverk/Arrs_Stack/playback_aware_sonarr_discovery.sh --status
# bash /boot/config/plugins/varaverk/Arrs_Stack/playback_aware_sonarr_discovery.sh

# lidarr_missing_art.sh — fetch missing album and artist artwork for the Lidarr music library
# Called by intermediate_sync_maintenance.sh via INTERMEDIATE_MAINTENANCE_SCRIPTS.
# HOST1 only — exits cleanly on HOST2 (HOST2 does not run Lidarr).
# READS from Lidarr only — no writes back, never modifies audio tags or renames files.
# NEVER overwrites existing artwork — idempotent re-runs are safe.
#
# Artwork targets:
#   Album:  cover.jpg  cdart.png  back.jpg
#   Artist: folder.jpg  fanart.jpg  logo.png  banner.jpg
# Sources:  fanart.tv → iTunes (album) / Deezer / Last.fm (artist) fallbacks
#
# bash /boot/config/plugins/varaverk/Arrs_Stack/lidarr_missing_art.sh --dry-run --log
# bash /boot/config/plugins/varaverk/Arrs_Stack/lidarr_missing_art.sh --status
# bash /boot/config/plugins/varaverk/Arrs_Stack/lidarr_missing_art.sh

# radarr_tmdb_removed.sh — remove movies dropped from TMDb (status="deleted")
# These generate Radarr system health errors and can never be monitored or downloaded.
# 99% are future/announced movies that were delisted before release.
# Files NOT deleted by default — use --delete-files to also remove from disk.
# Import exclusion added by default — prevents Radarr re-adding the same dropped movie.
# HOST1 only — exits cleanly on HOST2.
#
# bash /boot/config/plugins/varaverk/Arrs_Stack/radarr_tmdb_removed.sh --dry-run
# bash /boot/config/plugins/varaverk/Arrs_Stack/radarr_tmdb_removed.sh --status
# bash /boot/config/plugins/varaverk/Arrs_Stack/radarr_tmdb_removed.sh
# bash /boot/config/plugins/varaverk/Arrs_Stack/radarr_tmdb_removed.sh --delete-files

# sonarr_tvdb_removed.sh — remove series dropped from TVDB (status="deleted")
# Same pattern as radarr_tmdb_removed: Sonarr health errors, can't be monitored.
# Files NOT deleted by default — use --delete-files to also remove from disk.
# Import exclusion added by default. HOST1 only.
#
# bash /boot/config/plugins/varaverk/Arrs_Stack/sonarr_tvdb_removed.sh --dry-run
# bash /boot/config/plugins/varaverk/Arrs_Stack/sonarr_tvdb_removed.sh --status
# bash /boot/config/plugins/varaverk/Arrs_Stack/sonarr_tvdb_removed.sh
# bash /boot/config/plugins/varaverk/Arrs_Stack/sonarr_tvdb_removed.sh --delete-files


# ──────────────────────────────────────────────────────────────────────────────────────────────
# TRANSCODES
# ──────────────────────────────────────────────────────────────────────────────────────────────

# ramdisk_setup.sh — create tmpfs ramdisk and transcode symlink at array start
# Called by array_started.sh. MUST run before Emby starts.
# Idempotent: if ramdisk already mounted → report status and exit cleanly, do not remount.
#
# Creates:
#   RAMDISK_PATH  tmpfs mount (HOST*_RAMDISK_SIZE ceiling — only uses RAM actually needed)
#   transcoding-temp/  pre-created inside ramdisk — Emby finds this before the SSD version.
#                       Without pre-creation: Emby creates transcoding-temp at first writable
#                       location which may be SSD, routing ALL sessions there until restart.
#   TRANSCODE_SSD  SSD fallback directory (and SSD/transcoding-temp for consistency)
#   TRANSCODE_LINK symlink → RAMDISK_PATH (always reset to ramdisk at array start)
#
# IMPORTANT: Emby Extra Parameters must include:
#   --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode
#   Do NOT add bind-propagation=shared — runc v1.3.5+ (Docker 29.x, Unraid 7.3+) rejects it.
#   The ramdisk mount is already shared at the kernel level (MS_SHARED) — propagation is automatic.
#
# Verify setup:
#   mountpoint /mnt/ramdisk_transcodes     should say "is a mountpoint"
#   readlink /mnt/ram-transcode            should show /mnt/ramdisk_transcodes
#   ls /mnt/ramdisk_transcodes/            should show transcoding-temp/
#
# bash /boot/config/plugins/varaverk/Transcodes/ramdisk_setup.sh --status
# bash /boot/config/plugins/varaverk/Transcodes/ramdisk_setup.sh --dry-run
# bash /boot/config/plugins/varaverk/Transcodes/ramdisk_setup.sh

# transcode_manager.sh — manage ramdisk/SSD symlink based on current usage
# Called by transcode_management.sh AFTER cleanup. Can be run standalone for testing.
# Modes (TRANSCODE_MANAGER_MODE in master.conf):
#   smart    auto-flip between ramdisk and SSD based on thresholds (default)
#   ramdisk  always ramdisk, never flip to SSD (warn if above threshold)
#   ssd      always SSD, never ramdisk (use post-flip drain or maintenance)
#
# Safety checks every cycle regardless of mode:
#   Missing/broken symlink → recreate pointing at ramdisk
#   Ramdisk disappeared   → auto-flip to SSD, notify warning
#   SSD path missing      → disable SSD fallback / error in ssd mode
#   transcoding-temp missing from ramdisk → recreate immediately (prevents SSD session drift)
#   Permissions drift     → fix silently every run
#
# Multi-server: TRANSCODE_SERVERS array supports Emby, Jellyfin, Plex
# DO NOT add Tdarr here — full-file encoding fills ramdisk rapidly.
#
# bash /boot/config/plugins/varaverk/Transcodes/transcode_manager.sh --status
# bash /boot/config/plugins/varaverk/Transcodes/transcode_manager.sh --dry-run
# bash /boot/config/plugins/varaverk/Transcodes/transcode_manager.sh

# transcode_cleanup.sh — remove stale HLS segment files from ramdisk and SSD fallback
# Called by transcode_management.sh BEFORE transcode_manager.sh (order critical).
# A file is eligible only when: older than TRANSCODE_MAX_AGE (20min) AND not open by any process.
# lsof called ONCE per location (not once per file — critical performance on Live TV systems).
# transcoding-temp directory NEVER deleted even when empty — protects against SSD session drift.
# After cleanup: if ramdisk usage dropped below RAMDISK_LOW_GB, triggers flip-back to ramdisk.
#
# bash /boot/config/plugins/varaverk/Transcodes/transcode_cleanup.sh --dry-run
# bash /boot/config/plugins/varaverk/Transcodes/transcode_cleanup.sh

# ramdisk_stop.sh — safely stop the transcode ramdisk without interrupting Emby
# Flips the transcode symlink to the SSD fallback BEFORE unmounting so Emby continues writing.
# Then unmounts the tmpfs and updates the state file.
# Use before: planned reboots, ramdisk size changes, ramdisk troubleshooting.
# ramdisk_setup.sh recreates and re-mounts on next array start.
#
# bash /boot/config/plugins/varaverk/Tools/ramdisk_stop.sh --dry-run
# bash /boot/config/plugins/varaverk/Tools/ramdisk_stop.sh --status
# bash /boot/config/plugins/varaverk/Tools/ramdisk_stop.sh


# ──────────────────────────────────────────────────────────────────────────────────────────────
# MONITORS
# ──────────────────────────────────────────────────────────────────────────────────────────────
#
# All monitor scripts observe and report — they NEVER act on what they find.
# Read-only except bandwidth_monitor (one bounded log append per rsync) and
# system_tuning_monitor (one bounded log append per 6hr snapshot).
# Safe to run at any time.

# zfs_memory_snapshot.sh — weekly ZFS + ARC + memory diagnostic report
# ZFS pool status (ONLINE/DEGRADED/FAULTED) per pool. Pools in ZFS_REPORT_IGNORE_POOLS excluded.
# ARC: current size vs max, hit rate, metadata vs data ratio. Warns if ARC > ZFS_REPORT_ARC_WARN_PCT (90%).
# Memory: total, free, available. Warns if RAM < ZFS_REPORT_FREE_WARN_GB.
# Docker: top ZFS_REPORT_DOCKER_TOP (10) containers by memory usage.
# Kernel pressure: vmstat 3-sample snapshot.
# Output: console + /var/log/zfs-weekly-health.log (enables week-over-week comparison).
#
# bash /boot/config/plugins/varaverk/Monitors/zfs_memory_snapshot.sh --status
# bash /boot/config/plugins/varaverk/Monitors/zfs_memory_snapshot.sh --dry-run
# bash /boot/config/plugins/varaverk/Monitors/zfs_memory_snapshot.sh

# smart_health.sh — drive SMART attribute monitoring
# Checks every /dev/sd* and /dev/nvme*. Drives in HOST*_SMART_IGNORE_DRIVES are skipped.
# Temperature thresholds read from /boot/config/plugins/dynamix/dynamix.cfg (same as unRAID UI).
# Notifies on: Reallocated_Sector_Ct > 0 (bad sector swapped = warning sign),
# Current_Pending_Sector > 0 (suspected bad, not confirmed), Offline_Uncorrectable > 0 (data loss risk),
# temperature above thresholds, SMART overall-health FAILED (get data off immediately).
# Silent when all drives pass all checks.
#
# bash /boot/config/plugins/varaverk/Monitors/smart_health.sh --status
# bash /boot/config/plugins/varaverk/Monitors/smart_health.sh --dry-run
# bash /boot/config/plugins/varaverk/Monitors/smart_health.sh

# cert_monitor.sh — SSL certificate expiry via live TLS connection
# Connects via openssl s_client — checks cert actually being served to clients.
# Catches: renewed-but-not-reloaded (nginx not reloaded after certbot renewal),
#          wrong cert served, chain issues visible externally but not internally.
# If a user would see a certificate error in their browser, this catches it first.
# Configured via HOST*_CERT_MONITOR_DOMAINS in host*.conf.
# Thresholds: > 30 days = silent, <= 30 = warning, <= CERT_CRIT_DAYS (7) = urgent.
#
# bash /boot/config/plugins/varaverk/Monitors/cert_monitor.sh --status
# bash /boot/config/plugins/varaverk/Monitors/cert_monitor.sh --dry-run
# bash /boot/config/plugins/varaverk/Monitors/cert_monitor.sh

# backup_verify.sh — rsync mirror integrity via independent MD5 checksums
# Randomly samples BACKUP_VERIFY_SAMPLE (10) files per share. Computes MD5 both sides via SSH.
# Does not trust rsync exit code — verifies actual file content matches.
# rsync exit 0 does NOT guarantee content is correct. Hardware corruption produces files
# that exist with matching size/mtime but different content. rsync misses this. MD5 doesn't.
# Pre-flight: check_connectivity() and check_remote_array() prevent false MISSING alarms.
# Notifies on MISMATCH or excessive MISSING files.
#
# bash /boot/config/plugins/varaverk/Monitors/backup_verify.sh --status
# bash /boot/config/plugins/varaverk/Monitors/backup_verify.sh --dry-run
# bash /boot/config/plugins/varaverk/Monitors/backup_verify.sh

# emby_session_report.sh — Emby weekly streaming usage statistics
# Queries HOST*_EMBY_URL for past EMBY_REPORT_DAYS (7) days of activity.
# Reports: server version, active now (direct vs transcode), library size (movies/episodes/songs),
# weekly play count, top EMBY_REPORT_TOP_N content, most active users.
# Notifies if transcode ratio > 80% — usually indicates client/format compatibility issue.
#
# bash /boot/config/plugins/varaverk/Monitors/emby_session_report.sh --status
# bash /boot/config/plugins/varaverk/Monitors/emby_session_report.sh --dry-run
# bash /boot/config/plugins/varaverk/Monitors/emby_session_report.sh

# bandwidth_monitor.sh — rsync transfer history and weekly summary report
# --report: reads BANDWIDTH_LOG and generates weekly summary per profile.
#   Shows: run count, total GB transferred, average duration, failure count, large transfers.
# --log-transfer: called AUTOMATICALLY by rsync.sh after each sync — never call manually.
#   Log format: DATE|TIME|profile|duration|status|bytes_transferred
#   Version-stable field names — survives rsync output format changes between versions.
# BANDWIDTH_LOG bounded to BANDWIDTH_LOG_RETENTION (90) days — never grows unbounded.
#
# bash /boot/config/plugins/varaverk/Monitors/bandwidth_monitor.sh --report
# bash /boot/config/plugins/varaverk/Monitors/bandwidth_monitor.sh --status

# weekly_health_digest.sh — profile-controlled ecosystem health digest (see ORCHESTRATORS)
# bash /boot/config/plugins/varaverk/Monitors/weekly_health_digest.sh --dry-run
# bash /boot/config/plugins/varaverk/Monitors/weekly_health_digest.sh --status

# system_tuning_monitor.sh — inotify + PHP-FPM trend tracking (see ORCHESTRATORS)
# bash /boot/config/plugins/varaverk/Monitors/system_tuning_monitor.sh --status

# mesh_monitor.sh — at-a-glance view of all servers in the mesh (read-only, on demand)
# No schedule — run any time you want a current mesh snapshot. No SSH or API calls.
# Shows each node: hostname, Tailscale IP, role (owner/mirror), services covered in fallback tiers.
# Reads directly from all three conf files — does not use detect_hosts().
#
# bash /boot/config/plugins/varaverk/Monitors/mesh_monitor.sh

# continuous_scripts_status.sh — legacy live dashboard (Old_Arch_Still_Works/)
# NOTE: watchdog architecture changed to single-pass in v2.4. This dashboard was written
# for the continuous-loop design and is no longer kept current. Use --status flags on
# individual scripts instead: docker_watchdog --status, stability_watchdog --status, etc.
#
# bash /boot/config/plugins/varaverk/Old_Arch_Still_Works/continuous_scripts_status.sh


# ──────────────────────────────────────────────────────────────────────────────────────────────
# PARTNERSHIP
# ──────────────────────────────────────────────────────────────────────────────────────────────

# partnership_manager.sh — two-server auth-sharing relationship lifecycle manager
# Automated --check runs inside critical_sync_maintenance.sh. Below are manual operation modes.
# Blocklist: after offboard, former partner hostname is written to partnership_blocklist.db.
# --onboard is hard-blocked if remote is on blocklist. --unblock <hostname> clears it.
#
# --status: show both server state files (ACTIVE/INACTIVE), offline counter, owner server,
#           current WebUI redirect state on HOST2, whether critical sync is enabled.
#
# --onboard: HOST1/owner only. Reconfigures HOST2 Docker template WebUI URLs to redirect
#            to HOST1 via Tailscale (click NPM on HOST2 → opens HOST1's NPM).
#            Verifies WebUI routing works. Writes ACTIVE state on both servers.
#            ALWAYS --dry-run first — shows exact template changes before anything is touched.
#
# --offboard: either server can initiate. Stops rsync. Final sync to ensure HOST2 has
#             current state. Reconfigures HOST2 WebUI URLs back to localhost (HOST2 manages
#             its own auth independently from now on). Grace period then Tailscale removal.
#             Deferred finalisation: if remote unreachable, completes automatically when reachable.
#             ALWAYS --dry-run first. 10-second countdown before live execution.
#
# --transfer: owner only. Flips auth ownership (HOST1→HOST2 or back). Health strike system
#             verifies both servers healthy before proceeding. Requires confirmation string:
#             "i-understand-this-transfers-ownership" (long by design — cannot type accidentally).
#
# bash /boot/config/plugins/varaverk/Partnership/partnership_manager.sh --status
# bash /boot/config/plugins/varaverk/Partnership/partnership_manager.sh --onboard --dry-run
# bash /boot/config/plugins/varaverk/Partnership/partnership_manager.sh --onboard
# bash /boot/config/plugins/varaverk/Partnership/partnership_manager.sh --unblock hostname

# partnership_offboard.sh — cleanly end a partnership (standalone version of --offboard)
# Role is detected automatically — run on either server. Owner path runs the full sequence:
# stop rsync → final sync → reconfigure mirror WebUIs → disable sync → local cleanup →
# Tailscale removal. Mirror path handles local side and signals owner.
# Deferred finalisation: if remote unreachable, completes automatically when reachable.
# ALWAYS --dry-run first. 10-second countdown before live execution.
#
# bash /boot/config/plugins/varaverk/Partnership/partnership_offboard.sh --dry-run
# bash /boot/config/plugins/varaverk/Partnership/partnership_offboard.sh --status
# bash /boot/config/plugins/varaverk/Partnership/partnership_offboard.sh

# partnership_transfer.sh — transfer ownership from current owner to current mirror
# Owner only — mirror cannot initiate. Config and WebUI targets updated; containers stay.
# Health strike system verifies both servers healthy before proceeding.
# Requires confirmation string: "i-understand-this-transfers-ownership" (long by design).
# After transfer: sync direction reverses on the next fallback/critical_sync cycle.
#
# bash /boot/config/plugins/varaverk/Partnership/partnership_transfer.sh --dry-run
# bash /boot/config/plugins/varaverk/Partnership/partnership_transfer.sh --status
# bash /boot/config/plugins/varaverk/Partnership/partnership_transfer.sh \
#     --confirm=i-understand-this-transfers-ownership

# gitea_ssh_setup.sh — generate Gitea SSH keypair and register it with Gitea via API
# Idempotent: skips generation if key exists, skips registration if same key already registered.
# Use --force to regenerate and re-register (e.g. after Gitea reinstall or key compromise).
# Called during onboard setup or when re-keying a server. Run after Gitea container is up.
#
# bash /boot/config/plugins/varaverk/Partnership/gitea_ssh_setup.sh --dry-run
# bash /boot/config/plugins/varaverk/Partnership/gitea_ssh_setup.sh --status
# bash /boot/config/plugins/varaverk/Partnership/gitea_ssh_setup.sh
# bash /boot/config/plugins/varaverk/Partnership/gitea_ssh_setup.sh --force


# ──────────────────────────────────────────────────────────────────────────────────────────────
# INITIAL RUN
# ──────────────────────────────────────────────────────────────────────────────────────────────
#
# Run ONCE when setting up a new server or joining a new partnership. Not scheduled.
# All initial-run scripts now live in Partnership/ alongside the lifecycle scripts they set up.

# ssh_setup.sh — generate the rsync SSH keypair and install it on the remote server
# Key named from hostname: unRAID-Gmer4Lfe → gmer4lfe_rsync_automation
# Idempotent — skips generation if key already exists. --force to regenerate and re-copy.
#
# --validate: health check mode. Called during partnership --check to detect broken SSH auth.
#   Remote Tailscale IP unreachable = network issue → NOT a strike.
#   Remote reachable but SSH auth fails = key issue → strike incremented.
#   At SSH_MAX_STRIKES (5): notify + return exit 2 for caller to escalate.
#   Strikes reset after SSH_STRIKE_RESET_HRS (24) of clean connectivity.
#
# bash /boot/config/plugins/varaverk/Partnership/ssh_setup.sh --dry-run
# bash /boot/config/plugins/varaverk/Partnership/ssh_setup.sh --status
# bash /boot/config/plugins/varaverk/Partnership/ssh_setup.sh
# bash /boot/config/plugins/varaverk/Partnership/ssh_setup.sh --force
# bash /boot/config/plugins/varaverk/Partnership/ssh_setup.sh --validate

# partnership_onboard.sh — full partnership setup orchestrator (run once to join a partner server)
# Orchestrates the complete setup sequence in order:
#   Step 1: ssh_setup.sh          generate keypair, install on remote, update conf
#   Step 2: partnership_manager.sh --onboard
#                                 verify both servers, reconfigure auth WebUIs → owner IP,
#                                 write ACTIVE state, create FolderView3 folder if enabled
#   Step 3: arr_sync.sh           bidirectional library bootstrap — both sides start with the
#                                 merged library from day one (non-fatal: skipped if arrs not live)
#
# Pre-requisites: both servers on same Tailscale tailnet, remote has password auth for root,
# PARTNERSHIP_OWNER_HOST correct in master.conf.
# ALWAYS --dry-run first.
#
# bash /boot/config/plugins/varaverk/Partnership/partnership_onboard.sh --dry-run
# bash /boot/config/plugins/varaverk/Partnership/partnership_onboard.sh
# bash /boot/config/plugins/varaverk/Partnership/partnership_onboard.sh --skip-ssh
# bash /boot/config/plugins/varaverk/Partnership/partnership_onboard.sh --skip-arr-sync


# ──────────────────────────────────────────────────────────────────────────────────────────────
# TOOLS
# ──────────────────────────────────────────────────────────────────────────────────────────────
#
# Situational utilities — run when something specific needs fixing, not on a schedule.
# Each exists because a specific situation arose that required bash commands to resolve.
# All support --dry-run and --status.

# bulk_permissions_repair.sh — apply nobody:users 755/664 to a specific path immediately
# Targeted fix — do not wait for next nightly media_shares_permissions.sh run.
# Use when: scp/cp left root:root files blocking arr operations right now.
# Accepts one or multiple paths on the same command line.
#
# bash /boot/config/plugins/varaverk/Tools/bulk_permissions_repair.sh /mnt/user/Movies --dry-run
# bash /boot/config/plugins/varaverk/Tools/bulk_permissions_repair.sh /mnt/user/Movies
# bash /boot/config/plugins/varaverk/Tools/bulk_permissions_repair.sh /mnt/user/Movies /mnt/user/Music

# container_data_export.sh — stop container, archive appdata, verify archive, restart
# Use before: major version updates with "no rollback" migrations, pool migrations,
# container removal, risky config changes that need a clean restore point.
# Sequence: space check → stop container → tar.gz archive → verify integrity → restart.
# Container is always restarted even if archiving fails — never leaves container stopped.
# Syntax: container_data_export.sh ContainerName /path/to/appdata /output/directory
#
# bash /boot/config/plugins/varaverk/Tools/container_data_export.sh \
#     Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/ --dry-run
# bash /boot/config/plugins/varaverk/Tools/container_data_export.sh \
#     Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/

# emby_database_repair.sh — SQLite integrity check on all Emby databases
# Use when: Emby crashes repeatedly with database errors, after power loss with Emby running.
# Stops Emby. Runs PRAGMA integrity_check on every .db file. Restarts Emby. Reports per database.
# Recovery guidance per database (this script does NOT auto-delete — human decision required):
#   library.db          → CORRUPT: safe to delete (Emby fully rebuilds from media files)
#   users.db            → CORRUPT: deleting resets ALL accounts + watch history (check backup)
#   authentication.db   → CORRUPT: safe to delete (API keys regenerated on restart)
#   activity.db         → CORRUPT: safe to delete (it's a log only)
# Config path detected automatically from Docker volume mounts — no configuration needed.
#
# bash /boot/config/plugins/varaverk/Tools/emby_database_repair.sh --status
# bash /boot/config/plugins/varaverk/Tools/emby_database_repair.sh --dry-run
# bash /boot/config/plugins/varaverk/Tools/emby_database_repair.sh

# zfs_pool_scrub.sh — trigger ZFS scrub on all pools, wait for completion
# ZFS checksums every block and verifies — catches silent data corruption before you
# discover it on reads. Safe to run while system is active (low I/O priority).
# Pools in ZFS_REPORT_IGNORE_POOLS excluded from default all-pool scrub.
# Specify a pool name to scrub it specifically (bypasses the ignore list).
# Notifies when complete with error summary.
#
# bash /boot/config/plugins/varaverk/Tools/zfs_pool_scrub.sh --dry-run
# bash /boot/config/plugins/varaverk/Tools/zfs_pool_scrub.sh
# bash /boot/config/plugins/varaverk/Tools/zfs_pool_scrub.sh poolname

# emby_to_lidarr_sync.sh — one-shot bootstrap: add all played Emby artists to Lidarr
# Finds artists played on Emby that are not tracked in Lidarr and adds them. No scoring.
# Run once after Lidarr is set up, or any time you suspect gaps between what you listen to
# and what Lidarr monitors. Use --days N to limit lookback window.
# bash /boot/config/plugins/varaverk/Tools/emby_to_lidarr_sync.sh --dry-run
# bash /boot/config/plugins/varaverk/Tools/emby_to_lidarr_sync.sh --status
# bash /boot/config/plugins/varaverk/Tools/emby_to_lidarr_sync.sh

# emby_to_radarr_sync.sh — one-shot bootstrap: add all played Emby movies to Radarr
# Same pattern as emby_to_lidarr_sync. Run once to ensure everything you've watched is monitored.
# bash /boot/config/plugins/varaverk/Tools/emby_to_radarr_sync.sh --dry-run
# bash /boot/config/plugins/varaverk/Tools/emby_to_radarr_sync.sh --status
# bash /boot/config/plugins/varaverk/Tools/emby_to_radarr_sync.sh

# emby_to_sonarr_sync.sh — one-shot bootstrap: add all played Emby series to Sonarr
# Same pattern as emby_to_lidarr_sync. Run once to ensure all watched shows are monitored.
# bash /boot/config/plugins/varaverk/Tools/emby_to_sonarr_sync.sh --dry-run
# bash /boot/config/plugins/varaverk/Tools/emby_to_sonarr_sync.sh --status
# bash /boot/config/plugins/varaverk/Tools/emby_to_sonarr_sync.sh

# recreate_shares.sh — create share directories on disks after fresh install or rebuild
# Use when: share .cfg files restored to fresh install but directories missing on disks.
# Reads /boot/config/shares/*.cfg, parses shareInclude list, creates /mnt/diskN/sharename.
# chown nobody:users on creation. Run once — then initial rsync populates the content.
# Without this: rsync aborts "remote share missing" even though the share shows in the UI.
#
# bash /boot/config/plugins/varaverk/Plugin/unraid/Tools/recreate_shares.sh --dry-run
# bash /boot/config/plugins/varaverk/Plugin/unraid/Tools/recreate_shares.sh --status
# bash /boot/config/plugins/varaverk/Plugin/unraid/Tools/recreate_shares.sh


# ==============================================================================================
# ██  QUICK REFERENCE — COMPLETE SCHEDULE  █████████████████████████████████████████████████████
# ==============================================================================================
#
# At Startup of Array:
#   Orchestrators/array_started.sh                             (single entry — handles everything)
#
# At Stopping of Array:
#   Orchestrators/array_stopping.sh                            (single entry — graceful shutdown)
#
# */15 * * * *   every 15 minutes:
#   Orchestrators/watchdog_orchestrator.sh
#     └─ resource_watchdog → docker_watchdog → system_watchdog (storage/webgui/network) → stability_watchdog
#
# */7 * * * *    every 7 minutes:
#   Orchestrators/transcode_management.sh
#
# */30 * * * *   every 30 minutes:
#   Orchestrators/critical_sync_maintenance.sh
#     └─ Critical-Data rsync → play_state_sync → partnership --check
#
# 0 */4 * * *    every 4 hours:
#   Orchestrators/intermediate_sync_maintenance.sh
#     └─ arr_sync → arr_cache_prefill + arrs_failed_stalled_recovery (INTERMEDIATE_MAINTENANCE_SCRIPTS)
#
# 0 */6 * * *    every 6 hours:
#   Monitors/system_tuning_monitor.sh
#
# 0 1 * * *      1am daily:
#   Orchestrators/daily_sync_maintenance.sh
#
# 0 0 15 * *     15th of each month at midnight:
#   Orchestrators/monthly_maintenance.sh
#
# 0 7 * * 0      Sunday 7am:
#   Orchestrators/sunday_morning_coffee_report.sh
#     └─ runs all monitors via COFFEE_REPORT_SCRIPTS (master.conf):
#          zfs_memory_snapshot / smart_health / cert_monitor / backup_verify
#          bandwidth_monitor / emby_session_report / weekly_health_digest
#
# 0 8 * * *      8am daily:
#   Monitors/weekly_health_digest.sh
#
# 30 2 * * 0     Sunday 2:30am:
#   Orchestrators/weekly_sync_maintenance.sh
#     └─ runs WEEKLY_MAINTENANCE_SCRIPTS after sync:
#          clear_logs / playback_aware_lidarr / playback_aware_radarr / playback_aware_sonarr
#
# ==============================================================================================