From f92ee4064b193b2d5a4abc9a71af0790d1ea5f41 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Fri, 26 Jun 2026 18:50:05 -0400 Subject: [PATCH] Add full banner headers to all scripts across the codebase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every script now has the established header format: PURPOSE with ─────── separator, OPERATIONAL MODEL, DESIGN PRINCIPLES, OPERATIONAL SAFEGUARDS, CONFIGURATION, and RUNTIME MODES — structured with full ====== banner sections throughout. Orchestrators converted from compact ── inline format to full banners. Stale emby-fallback and dirty sync references removed from Plugin/user_script_plug-in.sh. --- Docker_Essentials/docker_weekly_restart.sh | 20 +++ Media/arrs_failed_stalled_recovery.sh | 18 +++ Media/lidarr_cleanup.sh | 19 +++ Media/lidarr_missing_art.sh | 19 +++ Media/media_cleaner.sh | 18 +++ Media/media_shares_permissions.sh | 18 +++ Media/play_state_sync.sh | 32 ++++ Media/playback_aware_lidarr_discovery.sh | 53 ++++--- Media/playback_aware_radarr_discovery.sh | 57 +++++--- Media/playback_aware_sonarr_discovery.sh | 61 +++++--- Media/radarr_cleanup.sh | 19 +++ Media/radarr_tmdb_removed.sh | 19 +++ Media/sonarr_cleanup.sh | 19 +++ Media/sonarr_tvdb_removed.sh | 19 +++ Media/start_webhook_listener.sh | 61 ++++++++ Media/upgrade_webhook_handler.sh | 53 ++++++- Monitors/emby_session_report.sh | 19 +++ Monitors/mesh_monitor.sh | 30 ++++ Monitors/smart_health.sh | 19 +++ Monitors/system_tuning_monitor.sh | 18 +++ Monitors/weekly_health_digest.sh | 18 +++ Monitors/zfs_memory_snapshot.sh | 19 +++ Orchestrators/array_started.sh | 94 ++++++++---- Orchestrators/array_stopping.sh | 78 +++++++--- Orchestrators/critical_sync_maintenance.sh | 95 ++++++++---- Orchestrators/daily_sync_maintenance.sh | 137 +++++++++++------- .../intermediate_sync_maintenance.sh | 100 ++++++++----- Orchestrators/monthly_maintenance.sh | 91 +++++++++--- Orchestrators/sunday_morning_coffee_report.sh | 75 ++++++++-- Orchestrators/transcode_management.sh | 99 ++++++++----- Orchestrators/watchdog_orchestrator.sh | 98 +++++++++---- Orchestrators/weekly_sync_maintenance.sh | 111 ++++++++------ Plugin/build.sh | 11 +- Plugin/unraid/Partnership/containers.sh | 18 ++- .../System_Essentials/unraid_api_key_renew.sh | 23 +++ .../unraid/Tools/remote_arr_cache_writer.sh | 24 ++- Plugin/unraid/Tools/storage_migrate.sh | 23 +++ Plugin/unraid/adapter.sh | 72 +++++---- Plugin/unraid/run_job.sh | 78 ++++++++-- Plugin/unraid/user_script_plug-in.sh | 70 ++++----- System_Essentials/conf_cache_restore.sh | 24 +++ System_Essentials/conf_cache_save.sh | 22 +++ System_Essentials/conf_sync.sh | 23 +++ Tools/arr_profile_enforcer.sh | 39 ++++- Tools/claude_startup.sh | 26 ++++ Tools/docker_prune_images.sh | 19 +++ Tools/webhook_setup.sh | 48 +++++- Watchdogs/System/conf_cache_watchdog.sh | 25 ++++ Watchdogs/system_watchdog.sh | 66 +++++++-- git_pull_execute.sh | 97 +++++++++---- load_config.sh | 71 +++++---- 51 files changed, 1822 insertions(+), 563 deletions(-) diff --git a/Docker_Essentials/docker_weekly_restart.sh b/Docker_Essentials/docker_weekly_restart.sh index 9254775..a025f27 100755 --- a/Docker_Essentials/docker_weekly_restart.sh +++ b/Docker_Essentials/docker_weekly_restart.sh @@ -16,6 +16,26 @@ # stopped → leave, missing → skip. Container state is always respected. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Weekly Cadence +# Weekly restarts target services that accumulate state on a slower schedule +# than daily targets — less-critical containers that benefit from a periodic +# clean start but do not need nightly intervention. Daily restarts handle +# high-churn containers; weekly handles the longer-cycle ones. +# +# State Respect +# Running containers are restarted. Stopped containers are left stopped — they +# were intentionally halted and this script has no authority to override that +# decision. This rule is consistent across the entire ecosystem. +# +# Dependency-Safe Ordering +# Restarts follow the same dependency ordering used by docker_watchdog.sh. +# Services that other containers depend on restart first. A dependent is never +# restarted while its dependency is still coming up. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # diff --git a/Media/arrs_failed_stalled_recovery.sh b/Media/arrs_failed_stalled_recovery.sh index 449c0e8..aa286a3 100755 --- a/Media/arrs_failed_stalled_recovery.sh +++ b/Media/arrs_failed_stalled_recovery.sh @@ -28,6 +28,24 @@ # 3. Trigger new search — finds a different release automatically # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Hands-Free Recovery +# The script completes the full recovery cycle autonomously — blocklist, remove, +# re-search. No operator decision required. A failed import at midnight resolves +# itself before morning without any intervention. +# +# Age Gate Before Action +# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped. Arrs have their own +# retry logic — acting immediately would race against it. The age gate gives +# the arr time to self-resolve before this script escalates. +# +# Blocklist First +# The bad release is blocklisted before removal and re-search. Without this, +# the re-search can re-grab the same release that just failed. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # diff --git a/Media/lidarr_cleanup.sh b/Media/lidarr_cleanup.sh index 2d19e9c..d4776b7 100755 --- a/Media/lidarr_cleanup.sh +++ b/Media/lidarr_cleanup.sh @@ -27,6 +27,25 @@ # classification these would be deleted — breaking Lidarr and Emby display. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# API as Ground Truth +# What Lidarr tracks is authoritative. Files not in the API response are +# orphans — Lidarr has no record of them and they serve no purpose. +# The script never infers ownership from directory structure alone. +# +# Age Gate Before Deletion +# Files under LIDARR_ORPHAN_AGE are left alone regardless of tracked status. +# Lidarr's import pipeline writes files before registering them — acting +# immediately would delete files mid-import. +# +# Seven-Gate Safety Model +# Multiple independent sanity checks must all pass before any file is touched. +# No single check is trusted in isolation — a misconfigured path returning an +# empty API response must not result in a wiped library. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # diff --git a/Media/lidarr_missing_art.sh b/Media/lidarr_missing_art.sh index 8e72aa6..be53366 100755 --- a/Media/lidarr_missing_art.sh +++ b/Media/lidarr_missing_art.sh @@ -25,6 +25,25 @@ # album/artist directories. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Additive Only +# The script only adds missing files — it never overwrites existing artwork +# or touches audio files. Re-running after a partial fetch completes exactly +# where it left off with no side effects. +# +# Source Fallback Chain +# Multiple sources are tried in order of quality preference. fanart.tv is +# primary; fallbacks exist so partial coverage is better than none. A failed +# primary never blocks the fallback from running. +# +# External API Courtesy +# Rate limiting and parallel job caps prevent hammering fanart.tv and other +# external APIs. Burst behaviour during large initial runs would risk +# temporary blocks that break future scheduled fetches. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # diff --git a/Media/media_cleaner.sh b/Media/media_cleaner.sh index 4a9d1f2..1a386db 100755 --- a/Media/media_cleaner.sh +++ b/Media/media_cleaner.sh @@ -43,6 +43,24 @@ # Media profile also removes: *.iso *.lrc # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Pre-Scan Cleanup +# Runs before arr cleanup scripts so orphan detection only encounters actual +# media files. Scene debris and download artifacts would otherwise appear as +# untracked files and inflate false-positive orphan counts. +# +# Profile Separation +# Anime and media share different cleanup patterns because their content +# differs. *.lrc (lyrics) and *.iso belong in media cleanup but not anime. +# Separate profiles prevent cross-contamination of rules. +# +# Conservative by Default +# Only explicitly listed patterns are removed. The script never guesses +# at file intent — if a pattern is not in the list, the file is untouched. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # diff --git a/Media/media_shares_permissions.sh b/Media/media_shares_permissions.sh index edd67d5..4a0eca0 100755 --- a/Media/media_shares_permissions.sh +++ b/Media/media_shares_permissions.sh @@ -14,6 +14,24 @@ # or unRAID environment resets after updates. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Daily Failsafe, Not Enforcer +# Wrong ownership is a symptom of something else — a misconfigured container, +# a manual copy, an rsync without --chown. This script corrects the symptom +# daily rather than hunting the root cause. A persistent high correction count +# is the signal to investigate the source. +# +# Runs First in the Window +# Arr cleanup scripts depend on correct ownership to rename and delete files. +# Permissions must be correct before cleanup runs — ordering is not optional. +# +# Separate Passes for Directories and Files +# Directories need execute permission for traversal; files do not. Applying +# the same mode to both is a common mistake this script avoids by design. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # diff --git a/Media/play_state_sync.sh b/Media/play_state_sync.sh index 1b10101..073b852 100755 --- a/Media/play_state_sync.sh +++ b/Media/play_state_sync.sh @@ -33,6 +33,38 @@ # The higher tick count wins. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Newest Timestamp Wins +# No merge logic, no conflict resolution — the server with the most recent +# LastPlayedDate is simply authoritative. Simple rules produce predictable +# outcomes users can reason about. +# +# No Data Loss +# The sync only pushes state forward — it never clears a Played flag or +# resets a resume position to zero. A watch record on any server always +# propagates outward, never disappears. +# +# Provider ID Matching +# Items are matched by external IDs (IMDb, TVDB, MusicBrainz), not by +# title or file path. This makes matching robust across library reorganisation, +# renames, and multi-server path differences. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# PLAY_SYNC_ENABLED gate — exits cleanly when disabled; no partial runs +# PARTNERSHIP gate — skips remote sync when partnership is inactive +# Per-server reachability — unreachable servers are skipped individually; +# one offline server does not abort the entire sync +# User match required — a user missing from a server is skipped for that +# server; no cross-account state pollution +# acquire_lock — prevents concurrent runs from racing on the same +# items during the 30-minute critical window +# +# ============================================================================================== # CONFIGURATION (host*.conf, aliased by detect_hosts) # ============================================================================================== # diff --git a/Media/playback_aware_lidarr_discovery.sh b/Media/playback_aware_lidarr_discovery.sh index cce612f..b3fe62a 100755 --- a/Media/playback_aware_lidarr_discovery.sh +++ b/Media/playback_aware_lidarr_discovery.sh @@ -60,27 +60,23 @@ # Configure HOST*_LASTFM_API_KEY in host*.conf # # ============================================================================================== -# CONFIGURATION (master.conf) +# DESIGN PRINCIPLES # ============================================================================================== # -# LIDARR_DISCOVERY_THRESHOLD — minimum score for Stage 1 seeds and Stage 2 adds (default: 70) -# LIDARR_DISCOVERY_LOOKBACK_DAYS — Emby play history window in days (default: 7) -# LIDARR_DISCOVERY_MIN_PLAYS — min plays to be evaluated in Stage 1 (default: 3) -# LIDARR_DISCOVERY_MAX_ADDS — max seeds (Stage 1) and max adds (Stage 2) (default: 5) -# LIDARR_DISCOVERY_USER_CAP_PCT — max % any one user contributes to play weight (default: 35) -# LIDARR_DISCOVERY_REJECT_COOLDOWN — days before re-evaluating a Stage 2 reject (default: 30) -# LIDARR_DISCOVERY_HISTORY — history/state file path +# Playback as Intent Signal +# What users actually listen to is a stronger signal than what they follow or +# own. The scoring model weights demonstrated listening behaviour — recency, +# play count, user breadth — over passive library membership. # -# ============================================================================================== -# RUNTIME MODES -# ============================================================================================== +# Selective by Design +# 0–5 adds per week is the target, not bulk imports. A high score threshold +# combined with MAX_ADDS ensures only high-confidence recommendations are +# acted on. Volume is not the goal — meaningful discovery is. # -# playback_aware_lidarr_discovery.sh — normal run -# playback_aware_lidarr_discovery.sh --dry-run — score and rank, no Lidarr changes -# playback_aware_lidarr_discovery.sh --log — verbose output -# playback_aware_lidarr_discovery.sh --status — show config and exit -# -# Recommended schedule: weekly (WEEKLY_MAINTENANCE_SCRIPTS in master.conf) +# Two-Stage Filtering +# Stage 1 rejects weak seeds before they drive Stage 2. Low-quality seeds +# produce low-quality similar-artist recommendations. Filtering at the seed +# stage improves the entire output, not just the top of the list. # # ============================================================================================== # OPERATIONAL SAFEGUARDS @@ -110,6 +106,29 @@ # runs. Safe to delete — next run starts fresh with no memory. # # ============================================================================================== +# CONFIGURATION (master.conf) +# ============================================================================================== +# +# LIDARR_DISCOVERY_THRESHOLD — minimum score for Stage 1 seeds and Stage 2 adds (default: 70) +# LIDARR_DISCOVERY_LOOKBACK_DAYS — Emby play history window in days (default: 7) +# LIDARR_DISCOVERY_MIN_PLAYS — min plays to be evaluated in Stage 1 (default: 3) +# LIDARR_DISCOVERY_MAX_ADDS — max seeds (Stage 1) and max adds (Stage 2) (default: 5) +# LIDARR_DISCOVERY_USER_CAP_PCT — max % any one user contributes to play weight (default: 35) +# LIDARR_DISCOVERY_REJECT_COOLDOWN — days before re-evaluating a Stage 2 reject (default: 30) +# LIDARR_DISCOVERY_HISTORY — history/state file path +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# playback_aware_lidarr_discovery.sh — normal run +# playback_aware_lidarr_discovery.sh --dry-run — score and rank, no Lidarr changes +# playback_aware_lidarr_discovery.sh --log — verbose output +# playback_aware_lidarr_discovery.sh --status — show config and exit +# +# Recommended schedule: weekly (WEEKLY_MAINTENANCE_SCRIPTS in master.conf) +# +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Media/playback_aware_radarr_discovery.sh b/Media/playback_aware_radarr_discovery.sh index cff0934..2b7f4a0 100755 --- a/Media/playback_aware_radarr_discovery.sh +++ b/Media/playback_aware_radarr_discovery.sh @@ -59,29 +59,23 @@ # Free key at: https://www.themoviedb.org/settings/api # # ============================================================================================== -# CONFIGURATION (master.conf) +# DESIGN PRINCIPLES # ============================================================================================== # -# RADARR_DISCOVERY_THRESHOLD — minimum score to add a candidate (default: 52) -# RADARR_DISCOVERY_LOOKBACK_DAYS — Emby watch history window in days (default: 30) -# RADARR_DISCOVERY_MAX_SEEDS — max seed movies from Stage 1 (default: 5) -# RADARR_DISCOVERY_MAX_ADDS — max movies to add per run (default: 5) -# RADARR_DISCOVERY_MIN_VOTE_COUNT — min TMDB votes for a candidate (default: 100) -# RADARR_DISCOVERY_MIN_RATING — min TMDB vote_average × 10 (default: 60 = 6.0/10) -# RADARR_DISCOVERY_REJECT_COOLDOWN — days before re-evaluating a rejected movie (default: 60) -# RADARR_DISCOVERY_SEED_LIBRARIES — Emby library names to draw seeds from (default: ("Movies")) -# RADARR_DISCOVERY_HISTORY — history/state file path +# Playback as Intent Signal +# Recently watched movies are a stronger signal than what is in the library or +# on watchlists. The scoring model weights demonstrated viewing behaviour — +# recency, rating, vote confidence — over passive ownership. # -# ============================================================================================== -# RUNTIME MODES -# ============================================================================================== +# Selective by Design +# 0–5 adds per run is the target, not bulk imports. A score threshold combined +# with MAX_ADDS ensures only high-confidence recommendations are acted on. +# Volume is not the goal — meaningful discovery is. # -# playback_aware_radarr_discovery.sh — normal run -# playback_aware_radarr_discovery.sh --dry-run — score and rank, no Radarr changes -# playback_aware_radarr_discovery.sh --log — verbose output -# playback_aware_radarr_discovery.sh --status — show config and exit -# -# Recommended schedule: weekly (WEEKLY_MAINTENANCE_SCRIPTS in master.conf) +# Two-Stage Filtering +# Stage 1 rejects weak seeds before they drive Stage 2. Low-quality or +# low-confidence watched movies produce poor recommendations. Filtering at +# the seed stage improves the entire output. # # ============================================================================================== # OPERATIONAL SAFEGUARDS @@ -111,6 +105,31 @@ # runs. Safe to delete — next run starts fresh with no memory. # # ============================================================================================== +# CONFIGURATION (master.conf) +# ============================================================================================== +# +# RADARR_DISCOVERY_THRESHOLD — minimum score to add a candidate (default: 52) +# RADARR_DISCOVERY_LOOKBACK_DAYS — Emby watch history window in days (default: 30) +# RADARR_DISCOVERY_MAX_SEEDS — max seed movies from Stage 1 (default: 5) +# RADARR_DISCOVERY_MAX_ADDS — max movies to add per run (default: 5) +# RADARR_DISCOVERY_MIN_VOTE_COUNT — min TMDB votes for a candidate (default: 100) +# RADARR_DISCOVERY_MIN_RATING — min TMDB vote_average × 10 (default: 60 = 6.0/10) +# RADARR_DISCOVERY_REJECT_COOLDOWN — days before re-evaluating a rejected movie (default: 60) +# RADARR_DISCOVERY_SEED_LIBRARIES — Emby library names to draw seeds from (default: ("Movies")) +# RADARR_DISCOVERY_HISTORY — history/state file path +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# playback_aware_radarr_discovery.sh — normal run +# playback_aware_radarr_discovery.sh --dry-run — score and rank, no Radarr changes +# playback_aware_radarr_discovery.sh --log — verbose output +# playback_aware_radarr_discovery.sh --status — show config and exit +# +# Recommended schedule: weekly (WEEKLY_MAINTENANCE_SCRIPTS in master.conf) +# +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Media/playback_aware_sonarr_discovery.sh b/Media/playback_aware_sonarr_discovery.sh index 31f641a..efd9e15 100755 --- a/Media/playback_aware_sonarr_discovery.sh +++ b/Media/playback_aware_sonarr_discovery.sh @@ -66,31 +66,23 @@ # Free key at: https://www.themoviedb.org/settings/api # # ============================================================================================== -# CONFIGURATION (master.conf) +# DESIGN PRINCIPLES # ============================================================================================== # -# SONARR_DISCOVERY_THRESHOLD — minimum score to add a candidate (default: 52) -# SONARR_DISCOVERY_LOOKBACK_DAYS — Emby watch history window in days (default: 14) -# SONARR_DISCOVERY_MAX_SEEDS — max seed series from Stage 1 (default: 5) -# SONARR_DISCOVERY_MAX_ADDS — max shows to add per run (default: 3) -# SONARR_DISCOVERY_MIN_VOTE_COUNT — min TMDB votes for a candidate (default: 50) -# SONARR_DISCOVERY_MIN_RATING — min TMDB vote_average × 10 (default: 65 = 6.5/10) -# SONARR_DISCOVERY_REJECT_COOLDOWN — days before re-evaluating a rejected show (default: 60) -# SONARR_DISCOVERY_USER_EPISODE_CAP — max episodes per user in seed scoring (default: 8) -# SONARR_DISCOVERY_MONITOR_MODE — Sonarr monitor mode on add: "all" or "future" (default: "all") -# SONARR_EMBY_LIBRARIES — Emby library names to draw seeds from -# SONARR_DISCOVERY_HISTORY — history/state file path +# Playback as Intent Signal +# Recently watched episodes are a stronger signal than what is in the library. +# User diversity across a series is weighted above a single user binge — +# broad household interest is a better predictor of a good addition than +# one person's session. # -# ============================================================================================== -# RUNTIME MODES -# ============================================================================================== +# Selective by Design +# 0–3 adds per run is the target. TV is a larger commitment than movies — +# a lower MAX_ADDS cap reflects that. Volume is not the goal. # -# playback_aware_sonarr_discovery.sh — normal run -# playback_aware_sonarr_discovery.sh --dry-run — score and rank, no Sonarr changes -# playback_aware_sonarr_discovery.sh --log — verbose output -# playback_aware_sonarr_discovery.sh --status — show config and exit -# -# Recommended schedule: weekly (WEEKLY_MAINTENANCE_SCRIPTS in master.conf) +# Two-Stage Filtering +# Stage 1 rejects weak seeds before they drive Stage 2. A poorly-watched +# or niche series produces poor recommendations. Filtering at the seed +# stage improves the entire output. # # ============================================================================================== # OPERATIONAL SAFEGUARDS @@ -125,6 +117,33 @@ # runs. Safe to delete — next run starts fresh with no memory. # # ============================================================================================== +# CONFIGURATION (master.conf) +# ============================================================================================== +# +# SONARR_DISCOVERY_THRESHOLD — minimum score to add a candidate (default: 52) +# SONARR_DISCOVERY_LOOKBACK_DAYS — Emby watch history window in days (default: 14) +# SONARR_DISCOVERY_MAX_SEEDS — max seed series from Stage 1 (default: 5) +# SONARR_DISCOVERY_MAX_ADDS — max shows to add per run (default: 3) +# SONARR_DISCOVERY_MIN_VOTE_COUNT — min TMDB votes for a candidate (default: 50) +# SONARR_DISCOVERY_MIN_RATING — min TMDB vote_average × 10 (default: 65 = 6.5/10) +# SONARR_DISCOVERY_REJECT_COOLDOWN — days before re-evaluating a rejected show (default: 60) +# SONARR_DISCOVERY_USER_EPISODE_CAP — max episodes per user in seed scoring (default: 8) +# SONARR_DISCOVERY_MONITOR_MODE — Sonarr monitor mode on add: "all" or "future" (default: "all") +# SONARR_EMBY_LIBRARIES — Emby library names to draw seeds from +# SONARR_DISCOVERY_HISTORY — history/state file path +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# playback_aware_sonarr_discovery.sh — normal run +# playback_aware_sonarr_discovery.sh --dry-run — score and rank, no Sonarr changes +# playback_aware_sonarr_discovery.sh --log — verbose output +# playback_aware_sonarr_discovery.sh --status — show config and exit +# +# Recommended schedule: weekly (WEEKLY_MAINTENANCE_SCRIPTS in master.conf) +# +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Media/radarr_cleanup.sh b/Media/radarr_cleanup.sh index f8cbb48..2753cbc 100755 --- a/Media/radarr_cleanup.sh +++ b/Media/radarr_cleanup.sh @@ -31,6 +31,25 @@ # Emby removes ghost entries immediately — no user-facing file-not-found errors. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# API as Ground Truth +# What Radarr tracks is authoritative. Files not in the API response are +# orphans — Radarr has no record of them and they serve no purpose. +# The script never infers ownership from directory structure alone. +# +# Age Gate Before Deletion +# Files under RADARR_ORPHAN_AGE are left alone regardless of tracked status. +# Radarr's import pipeline writes files before registering them — acting +# immediately would delete files mid-import. +# +# Emby Cleanup Is Part of the Job +# Deleting a file without telling Emby leaves ghost entries that show as +# broken items. Triggering the Emby clean is not optional — it completes +# the deletion from the user's perspective. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # diff --git a/Media/radarr_tmdb_removed.sh b/Media/radarr_tmdb_removed.sh index f1a0fa2..ac38058 100755 --- a/Media/radarr_tmdb_removed.sh +++ b/Media/radarr_tmdb_removed.sh @@ -24,6 +24,25 @@ # Per-deletion output is always visible — deletions are never silently swallowed. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Health Error Hygiene +# status="deleted" entries can never be monitored or downloaded — they only +# generate persistent health errors. Removing them is maintenance, not +# data loss: the content never existed on disk for most of these entries. +# +# Conservative File Handling +# Files are not deleted by default because most TMDb-removed entries are +# announced-but-never-released films with no files. The --delete-files flag +# is an explicit opt-in, not the default path. +# +# Exclusion List Prevents Re-add +# Removed entries are added to Radarr's import exclusion list by default. +# Without this, the same deleted entry can be re-added by lists or searches +# and immediately generate the same health error again. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # diff --git a/Media/sonarr_cleanup.sh b/Media/sonarr_cleanup.sh index 103fca4..76bdc26 100755 --- a/Media/sonarr_cleanup.sh +++ b/Media/sonarr_cleanup.sh @@ -31,6 +31,25 @@ # Emby removes ghost entries immediately — no user-facing file-not-found errors. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# API as Ground Truth +# What Sonarr tracks is authoritative. Files not in the API response are +# orphans — Sonarr has no record of them and they serve no purpose. +# The script never infers ownership from directory structure alone. +# +# Age Gate Before Deletion +# Files under SONARR_ORPHAN_AGE are left alone regardless of tracked status. +# Sonarr's import pipeline writes files before registering them — acting +# immediately would delete files mid-import. +# +# Emby Cleanup Is Part of the Job +# Deleting a file without telling Emby leaves ghost entries that show as +# broken items. Triggering the Emby clean is not optional — it completes +# the deletion from the user's perspective. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # diff --git a/Media/sonarr_tvdb_removed.sh b/Media/sonarr_tvdb_removed.sh index 2969e55..701056e 100755 --- a/Media/sonarr_tvdb_removed.sh +++ b/Media/sonarr_tvdb_removed.sh @@ -23,6 +23,25 @@ # Per-deletion output is always visible — deletions are never silently swallowed. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Health Error Hygiene +# status="deleted" series can never be monitored or downloaded — they only +# generate persistent health errors. Removing them is maintenance, not +# data loss: most have no associated files. +# +# Conservative File Handling +# Files are not deleted by default. A TVDB-removed series may still have +# episodes on disk that the user wants to keep. The --delete-files flag +# is an explicit opt-in, not the default path. +# +# Exclusion List Prevents Re-add +# Removed entries are added to Sonarr's import exclusion list by default. +# Without this, the same deleted series can be re-added by lists or searches +# and immediately generate the same health error again. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # diff --git a/Media/start_webhook_listener.sh b/Media/start_webhook_listener.sh index b1294e8..a85f37c 100755 --- a/Media/start_webhook_listener.sh +++ b/Media/start_webhook_listener.sh @@ -23,6 +23,67 @@ # If WEBHOOK_PORT is 0: exits cleanly (disables the listener). # # ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# 1. Check WEBHOOK_PORT — exit cleanly if 0 (listener disabled) +# 2. Check WEBHOOK_SECRET — generate and persist one if empty +# 3. exec node webhook_listener.js — replaces this process; PID stays the same +# +# exec is intentional: array_started.sh tracks the PID of this script to check +# whether the listener is running. exec preserves that PID across the hand-off +# to node. +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Node.js Over php -S +# php -S on Unraid PHP 8.4 silently drops POST request bodies — webhooks arrive +# empty and the handler has no payload to act on. Node.js handles POST bodies +# correctly and has no equivalent silent-drop behaviour. +# +# Runs Outside nginx +# The listener binds directly to WEBHOOK_PORT — no nginx proxy, no session auth. +# The shared secret in the URL query string is the only gate. This keeps the +# webhook path independent of the auth stack. +# +# exec Preserves PID +# The script execs into node rather than forking it. array_started.sh stores the +# PID of this script to check liveness — exec ensures that PID continues to +# refer to the running node process after the hand-off. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# WEBHOOK_PORT=0 gate — exits cleanly before any setup if the listener is disabled +# Secret auto-generate — WEBHOOK_SECRET generated via openssl rand if empty; +# persisted to master.conf immediately so restarts reuse it +# Shared secret gate — webhook URL must include ?key=; +# requests without a valid key are rejected by the Node.js server +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# WEBHOOK_PORT — port the listener binds to; 0 = disabled +# WEBHOOK_SECRET — shared secret for URL auth; auto-generated if empty +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# start_webhook_listener.sh +# Started automatically at array start via ARRAY_START_SCRIPTS. +# Exits immediately if WEBHOOK_PORT=0. +# +# To stop: +# pkill -f webhook_listener.js +# +# ============================================================================================== set -uo pipefail diff --git a/Media/upgrade_webhook_handler.sh b/Media/upgrade_webhook_handler.sh index 241acf9..5f4a21c 100755 --- a/Media/upgrade_webhook_handler.sh +++ b/Media/upgrade_webhook_handler.sh @@ -15,7 +15,56 @@ # begin searching — a search it will never win because we already have it. # # ============================================================================================== -# USAGE +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Closes the Propagation Window +# arr_sync runs every 4 hours. Without this handler, a remote node that already +# has the old version sees the upgrade tagged in arr_sync but the new file not +# yet on disk, and initiates a redundant quality search — a search it will never +# win because this host already has the file. Immediate push eliminates that window. +# +# Rescan as Ground Truth +# Pushing the file is not enough — the remote arr must also be told the file +# exists. Triggering a rescan makes the remote accept the pushed file as the +# current version without starting a new search. +# +# Cache-First API Key Lookup +# Remote arr API keys are read from conf if cached; otherwise fetched via SSH +# from the remote's config.xml. This avoids storing secrets redundantly while +# keeping API calls fast on hosts where the key is already known. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Arg validation — exits with usage message if arr_type or item_path missing +# Path existence — exits if item_path is not a directory on disk +# Tailscale resolution — skips a node if its Tailscale IP cannot be resolved +# rsync exit check — rescan is only triggered if rsync succeeded; a failed +# transfer does not cause the remote arr to scan a partial file +# SSH fallback — if no cached API key, falls back to SSH to read config.xml +# on the remote rather than failing the rescan step +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# host*.conf +# +# HOST* — host names used to discover remote nodes +# HOST*_SONARR_API_KEY — cached API key for direct HTTP rescan (optional) +# HOST*_RADARR_API_KEY — cached API key for direct HTTP rescan (optional) +# HOST*_LIDARR_API_KEY — cached API key for direct HTTP rescan (optional) +# +# master.conf +# +# SSH_KEY — SSH key path for rsync and SSH fallback +# ARR_SYNC_CONNECT_TIMEOUT — SSH connect timeout in seconds +# DOCKER_APPDATA_BASE — base path for reading arr config.xml on remote +# +# ============================================================================================== +# RUNTIME MODES # ============================================================================================== # # upgrade_webhook_handler.sh @@ -25,7 +74,7 @@ # (series.path from Sonarr, movie.folderPath from Radarr, # artist.path from Lidarr) # -# Called by webhook.php — not intended for direct invocation outside testing. +# Called by webhook_listener.js — not intended for direct invocation outside testing. # # ============================================================================================== diff --git a/Monitors/emby_session_report.sh b/Monitors/emby_session_report.sh index 9550f26..e4995e8 100755 --- a/Monitors/emby_session_report.sh +++ b/Monitors/emby_session_report.sh @@ -18,6 +18,25 @@ # configuration issue. Silent on clean runs. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# No Persistent State +# Every report is generated fresh from the Emby API. No local database, no +# incremental tracking. A missed run leaves no gap — the next run simply +# covers its own window. +# +# Silent When Healthy +# The report goes to Discord/notification as a summary. Transcode alerts are +# the only proactive notification — high transcode ratios may indicate a +# misconfigured client that needs attention before it becomes a performance issue. +# +# Section Independence +# Each report section (sessions, library, activity, top content) guards its own +# API calls. A failure in one section does not abort the others — the report +# produces partial output rather than nothing. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # diff --git a/Monitors/mesh_monitor.sh b/Monitors/mesh_monitor.sh index 78b1d87..9551f2e 100755 --- a/Monitors/mesh_monitor.sh +++ b/Monitors/mesh_monitor.sh @@ -24,6 +24,36 @@ # Safe to share with mesh members — contains no API keys, passwords, or SSH keys. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Read-Only, No Network Calls +# All data comes from conf files — no SSH, no API calls, no pings. The output +# is always instant and never fails due to a node being unreachable. This makes +# it safe to run at any time without side effects. +# +# Scales Automatically +# Iterates all defined HOST* vars rather than a hardcoded list. Adding a new +# node to master.conf/host*.conf makes it appear in the output immediately. +# +# Safe to Share +# Output contains only identity and coverage configuration — no API keys, +# no passwords, no SSH keys. The report can be shared with other mesh members +# without exposing secrets. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# No External Dependencies +# Reads only from already-sourced conf files. No curl, no ssh, no docker — +# nothing that can fail, hang, or require credentials. +# +# Empty Mesh Guard +# collect_hosts() populates ALL_HOST_IDS — if no HOST* vars are defined the +# output sections iterate over an empty array and exit cleanly. +# +# ============================================================================================== # CONFIGURATION # ============================================================================================== # diff --git a/Monitors/smart_health.sh b/Monitors/smart_health.sh index 1904bf6..d79afaa 100755 --- a/Monitors/smart_health.sh +++ b/Monitors/smart_health.sh @@ -19,6 +19,25 @@ # from master.conf if dynamix.cfg is not found. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Live Queries, No Persistent State +# Every run queries smartctl directly — no cached attribute history, no trend +# tracking. Each report is a snapshot of current drive health. This keeps the +# script simple and the output always current. +# +# Threshold Parity With unRAID Dashboard +# Temperature thresholds are read from dynamix.cfg — the same values unRAID +# uses on its own dashboard. A consistent threshold means no conflicting alerts +# between this script and the built-in unRAID warnings. +# +# Silent When Healthy +# No output, no notification on a clean run. The absence of a report is the +# confirmation that all drives passed. Noise from weekly healthy runs would +# erode attention to the reports that matter. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # diff --git a/Monitors/system_tuning_monitor.sh b/Monitors/system_tuning_monitor.sh index 0d4d6e5..4228ebd 100755 --- a/Monitors/system_tuning_monitor.sh +++ b/Monitors/system_tuning_monitor.sh @@ -18,6 +18,24 @@ # WebGUI slowdowns or timeouts under load. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Snapshot for Trend, Not Just Alert +# Exhaustion events are rarely instant — they build over hours or days. +# Logging every 6 hours builds a trend that weekly_health_digest.sh can +# surface as a warning count, catching gradual pressure before it becomes +# an outage. +# +# Bounded Log Size +# Log entries are trimmed to TUNING_LOG_RETENTION days on every write. +# The log never grows unbounded regardless of how long the server runs. +# +# Silent When Healthy +# No output, no notification on a clean run. Threshold breach is the only +# signal — routine snapshots below the threshold produce nothing. +# +# ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # diff --git a/Monitors/weekly_health_digest.sh b/Monitors/weekly_health_digest.sh index e58e020..d5e3a24 100755 --- a/Monitors/weekly_health_digest.sh +++ b/Monitors/weekly_health_digest.sh @@ -14,6 +14,24 @@ # into a single digest. Reads only — writes nothing, changes nothing. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Aggregator, Not Generator +# This script reads state files that other scripts maintain. It never produces +# health data itself — it only presents what is already there. Each source +# script remains responsible for its own state; this script is the envelope. +# +# Profile-Driven Notification +# The cron schedule never changes. The DIGEST_PROFILE in master.conf controls +# when notifications actually send — switching from daily noise to weekly +# summaries is a one-line conf change, not a cron edit. +# +# Read-Only, No Side Effects +# Writes nothing, changes nothing, triggers nothing. Safe to run at any time +# for a health snapshot without affecting any running service or state file. +# +# ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # diff --git a/Monitors/zfs_memory_snapshot.sh b/Monitors/zfs_memory_snapshot.sh index 63d4af7..e5b5054 100755 --- a/Monitors/zfs_memory_snapshot.sh +++ b/Monitors/zfs_memory_snapshot.sh @@ -15,6 +15,25 @@ # comparison. In --dry-run mode, console only — nothing written to the log. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Informational, Not Interventional +# This script reports — it does not act. system_watchdog.sh handles +# threshold-based intervention. Keeping the roles separate means the report +# is never suppressed by the same logic that triggers remediation. +# +# Week-Over-Week Comparison +# Output is written to ZFS_REPORT_LOG so the same snapshot can be reviewed +# across weeks. Memory pressure and ARC creep are slow — a single run is +# rarely conclusive; the trend across weeks is what matters. +# +# Section Independence +# Each of the five report sections guards its own data source. ZFS not +# available, Docker not responding — those sections skip, the rest still run. +# A partial report is more useful than no report. +# +# ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # diff --git a/Orchestrators/array_started.sh b/Orchestrators/array_started.sh index 205f29f..32646f4 100755 --- a/Orchestrators/array_started.sh +++ b/Orchestrators/array_started.sh @@ -2,61 +2,91 @@ # ============================================================================================== # ================================= Array Start Orchestrator =================================== # ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── # Single entry point for array start — fired by the Varaverk plugin's # disks_mounted event hook (Plugin/unraid/event/disks_mounted/array_start_jobs). # Launches everything configured in ARRAY_START_SCRIPTS in master.conf. # This script exits after launching all scripts — the event hook sees it complete normally. # -# ── WHAT IT LAUNCHES ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# # Configured in master.conf ARRAY_START_SCRIPTS — no changes to this script ever needed. # Current order (order matters — see below): # # ONE-SHOT (run and exit naturally): -# System_Essentials/unraid_api_key_renew.sh — re-register Varaverk API key at boot -# System_Essentials/inotify_tuning.sh — raise inotify limits before containers start -# System_Essentials/docker_syslog_filter.sh — suppress veth log noise before logs fill -# System_Essentials/php_fpm_max_children.sh — WebGUI performance tuning -# Transcodes/ramdisk_setup.sh — create tmpfs + symlink before Emby starts +# System_Essentials/unraid_api_key_renew.sh — re-register Varaverk API key at boot +# System_Essentials/inotify_tuning.sh — raise inotify limits before containers start +# System_Essentials/docker_syslog_filter.sh — suppress veth log noise before logs fill +# System_Essentials/php_fpm_max_children.sh — WebGUI performance tuning +# Transcodes/ramdisk_setup.sh — create tmpfs + symlink before Emby starts # Docker_Essentials/docker_network_connect.sh — ensure networks + container connections # # CONTINUOUS (run until array stops): -# Fallback/fallback.sh — mutual fallback monitor +# Fallback/fallback.sh — mutual fallback monitor # # NOTE: watchdogs (docker, system, stability) are NOT launched here. # They run via watchdog_orchestrator.sh every 15 min (cron), not as daemons. # -# ── WHY ORDER MATTERS ───────────────────────────────────────────────────────────────────────── -# unraid_api_key_renew.sh — before anything else — self-heals API registry at boot -# inotify_tuning.sh — must run BEFORE Code-Server and other containers start -# containers that start with low inotify limits keep them ✅ -# docker_syslog_filter — must run BEFORE any container starts creating veth interfaces -# ramdisk_setup.sh — must run BEFORE Emby starts transcoding -# docker_network_connect — must run BEFORE watchdogs check container states -# fallback.sh — last — needs everything else stable to make decisions -# -# ── ONE-SHOT vs CONTINUOUS DETECTION ───────────────────────────────────────────────────────── # Script is launched in background with bash script.sh & # After 1 second: if PID still alive → continuous (running in background) -# if PID dead + exit 0 → one-shot completed successfully -# if PID dead + exit N → failure +# if PID dead + exit 0 → one-shot completed successfully +# if PID dead + exit N → failure # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# Root check — all launched scripts require root -# acquire_lock — prevents duplicate array start launches -# detect_hosts() — MY_ID in notifications +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Order Is Load-Bearing +# inotify limits must be raised before containers start — containers that +# start with low limits keep them. Ramdisk must exist before Emby starts. +# Docker networks must be connected before watchdogs check container states. +# fallback.sh goes last — it needs everything else stable to make decisions. +# +# Configuration Owns the List +# ARRAY_START_SCRIPTS in master.conf is the only place scripts are added or +# removed. This orchestrator never needs to be edited to change what runs — +# one-shot vs continuous behaviour is auto-detected from the PID after launch. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root check — all launched scripts require root +# acquire_lock — prevents duplicate array start launches +# detect_hosts() — MY_ID in notifications # platform_require_cmd — notify validated before use -# chmod +x auto-fix — non-executable scripts fixed before launch -# Full path on failure — shows exact path for debugging -# notify on failures — alert if any script fails to launch +# chmod +x auto-fix — non-executable scripts fixed before launch +# Full path on failure — shows exact path for debugging +# notify on failures — alert if any script fails to launch +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── # ARRAY_START_SCRIPTS — ordered list of scripts to launch at array start # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# array_started.sh — normal launch (called by Varaverk disks_mounted event hook) -# array_started.sh --dry-run — show what would be launched without launching -# array_started.sh --status — show configured scripts and their current state -# array_started.sh --log — verbose output per script +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# array_started.sh +# Normal launch — called by Varaverk disks_mounted event hook. +# +# array_started.sh --dry-run +# Show what would be launched without launching. +# +# array_started.sh --status +# Show configured scripts and their current state. +# +# array_started.sh --log +# Verbose output per script. +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Orchestrators/array_stopping.sh b/Orchestrators/array_stopping.sh index 2b37d56..a892234 100755 --- a/Orchestrators/array_stopping.sh +++ b/Orchestrators/array_stopping.sh @@ -2,40 +2,74 @@ # ============================================================================================== # ================================= Array Stop Orchestrator ==================================== # ============================================================================================== -# Planned shutdown orchestrator — stops all active processes cleanly before array maintenance. -# Runs ARRAY_STOP_SCRIPTS from master.conf sequentially, each confirmed complete before next. # -# ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Planned shutdown orchestrator — stops all active processes cleanly before +# array maintenance. Runs ARRAY_STOP_SCRIPTS from master.conf sequentially, +# each confirmed complete before the next starts. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# # 1. user_scripts_stop.sh — kill background user scripts (prevents new operations) # 2. rsync_stop.sh --rsync-only — kill rsync; skip container recovery (handled in step 4) # 3. mover_stop.sh — stop mover after rsync (both write to same paths) # 4. docker_container_stop.sh — stop all containers one-by-one with verification # -# ── WHY THIS ORDER ──────────────────────────────────────────────────────────────────────────── -# User scripts stopped first — they can spawn new rsync/docker operations mid-shutdown. -# Rsync before mover — both write to the same paths; running together risks corruption. -# Containers last — apps should stay available as long as possible during shutdown prep. +# Unlike array_started.sh, all scripts run in the foreground. Each must complete +# (pass or fail) before the next starts — a failed stop is noted but does not +# prevent remaining steps from running. # -# ── SEQUENTIAL vs BACKGROUND ───────────────────────────────────────────────────────────────── -# Unlike array_started.sh, all scripts run in the foreground. Each must complete (pass or fail) -# before the next starts — a failed stop is noted but does not prevent remaining steps. +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# Root check — all stop scripts require root -# acquire_lock — prevents concurrent array stop runs -# detect_hosts() — MY_ID in notifications and logs +# Order Is Load-Bearing +# User scripts are stopped first — they can spawn new rsync or docker operations +# mid-shutdown. Rsync stops before mover — both write to the same paths and +# running together risks corruption. Containers stop last — apps should stay +# available as long as possible during shutdown prep. +# +# Non-Fatal Steps +# A failed stop step is logged and notified but does not abort the sequence. +# Remaining scripts still run — a partial stop is better than a halted one. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root check — all stop scripts require root +# acquire_lock — prevents concurrent array stop runs +# detect_hosts() — MY_ID in notifications and logs # platform_require_cmd — notify validated before use -# Non-fatal steps — a failed step is logged but remaining steps still run -# notify on failures — alert if any stop script fails +# notify on failures — alert if any stop script fails +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── # ARRAY_STOP_SCRIPTS — ordered list of stop scripts to run # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# array_stopping.sh — run full stop sequence -# array_stopping.sh --dry-run — preview without stopping anything -# array_stopping.sh --status — show configured scripts and exit -# array_stopping.sh --log — verbose output +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# array_stopping.sh +# Run full stop sequence. +# +# array_stopping.sh --dry-run +# Preview without stopping anything. +# +# array_stopping.sh --status +# Show configured scripts and exit. +# +# array_stopping.sh --log +# Verbose output. +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Orchestrators/critical_sync_maintenance.sh b/Orchestrators/critical_sync_maintenance.sh index 101d7ab..68b3833 100755 --- a/Orchestrators/critical_sync_maintenance.sh +++ b/Orchestrators/critical_sync_maintenance.sh @@ -2,46 +2,81 @@ # ============================================================================================== # ============================= Critical Sync Maintenance ====================================== # ============================================================================================== -# Orchestrator for time-sensitive syncs that run every 30 minutes. -# Keeps the mirror current between the less frequent daily and weekly windows. +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Orchestrator for time-sensitive syncs running every 30 minutes. Keeps the +# mirror current between the less frequent daily and weekly windows. # Schedule: */30 * * * * (every 30 minutes via User Scripts plugin) # -# ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── -# 1. Critical-Data rsync — auth stack, NPM config, certs (containers stopped both sides) +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# 1. Critical-Data rsync — auth stack, NPM config, certs (containers stopped both sides) # 2. CRITICAL_MAINTENANCE_SCRIPTS — play_state_sync + any other per-window scripts -# 3. partnership --check — read both state files, detect changes, act accordingly +# 3. partnership --check — read both state files, detect changes, act accordingly # -# ── WHY EVERY 30 MINUTES ────────────────────────────────────────────────────────────────────── -# Auth stack changes (new users, proxy rules, certs) propagate within 30min ✅ -# Emby watch states stay in sync — mirror users see correct playback position ✅ -# Partnership state changes detected and acted on quickly ✅ -# Lock prevents: daily rsync doing Critical-Data mid-critical window ✅ -# -# ── RSYNC GATE ──────────────────────────────────────────────────────────────────────────────── +# RSYNC GATE # RSYNC_ENABLED=false → skips all syncs (global gate) # CRITICAL_RSYNC_ENABLED=false → skips critical syncs only (per-orchestrator gate) -# partnership --check always runs regardless — state check doesn't need rsync +# partnership --check always runs regardless — state check doesn't need rsync. # -# ── LOCK BEHAVIOUR ──────────────────────────────────────────────────────────────────────────── -# acquire_lock "strict" — if previous 30min run still going, skip this cycle entirely -# Critical-Data taking > 30min is a problem worth knowing about -# Strict mode prevents pile-up without waiting — log and move on ✅ +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== # -# ── SILENT WHEN HEALTHY ─────────────────────────────────────────────────────────────────────── -# Runs 48 times per day — clean runs must produce zero output ✅ -# Only failures and notable events produce visible output +# Silent When Healthy +# Runs 48 times per day — clean runs must produce zero output. Only failures +# and notable events produce visible output. # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# CRITICAL_RSYNC_ENABLED — enable/disable rsync section -# CRITICAL_SYNC_SHARES — shares synced every 30min (HOST*_CRITICAL_SYNC_SHARES) -# CRITICAL_MAINTENANCE_SCRIPTS — scripts run in critical window (optional) -# PARTNERSHIP_ENABLED — enable/disable partnership check +# Auth-First Window +# Auth stack changes (new users, proxy rules, certs) propagate within 30min. +# Emby watch states stay in sync — mirror users see correct playback position. +# Partnership state changes detected and acted on quickly. +# +# Strict Lock, Never Queue +# acquire_lock "strict" — if the previous 30-min run is still going, skip +# this cycle entirely. Critical-Data taking > 30min is a problem worth +# knowing about. Strict mode prevents pile-up without waiting. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root check — rsync and container stop/start require root +# acquire_lock "strict" — no pile-up; skip cycle if prior run still active +# detect_hosts() — MY_ID and REMOTE_ID for routing and logs +# resolve_remote_ip — confirms remote reachability before any transfer +# RSYNC_ENABLED gate — global kill switch respected before any rsync call +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# CRITICAL_RSYNC_ENABLED — enable/disable rsync section +# CRITICAL_SYNC_SHARES — shares synced every 30min (HOST*_CRITICAL_SYNC_SHARES) +# CRITICAL_MAINTENANCE_SCRIPTS — scripts run in critical window (optional) +# PARTNERSHIP_ENABLED — enable/disable partnership check +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# critical_sync_maintenance.sh +# Normal run. +# +# critical_sync_maintenance.sh --dry-run +# Preview syncs without transferring. +# +# critical_sync_maintenance.sh --log +# Verbose per-share output. +# +# critical_sync_maintenance.sh --status +# Show configuration and exit. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# critical_sync_maintenance.sh — normal run -# critical_sync_maintenance.sh --dry-run — preview syncs without transferring -# critical_sync_maintenance.sh --log — verbose per-share output -# critical_sync_maintenance.sh --status — show configuration and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Orchestrators/daily_sync_maintenance.sh b/Orchestrators/daily_sync_maintenance.sh index df7369b..8897373 100755 --- a/Orchestrators/daily_sync_maintenance.sh +++ b/Orchestrators/daily_sync_maintenance.sh @@ -2,68 +2,101 @@ # ============================================================================================== # ============================= Daily Sync Maintenance ========================================= # ============================================================================================== -# Daily orchestrator — runs the full daily maintenance window in the correct order. -# Schedule: 0 1 * * * (1am daily via User Scripts plugin) # -# ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── -# Pre-sync: -# git_pull_execute.sh — pull latest scripts first, always +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Daily maintenance window orchestrator — runs the full daily sequence in the +# correct order. Schedule: 0 1 * * * (1am daily via User Scripts plugin) # -# Arr Sync (arr_sync.sh): -# Syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally. -# All nodes agree on tracked library before any files are transferred. -# Remote nodes that don't have an arr running are skipped gracefully. +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# Rsync window (DAILY_SYNC_SHARES per host): -# HOST*_DAILY_SYNC_SHARES — media shares spread to all nodes (no --delete) -# HOST*_PERSONAL_SHARES — encrypted personal shares +# Pre-sync: +# git_pull_execute.sh — pull latest scripts first, always # -# Post-sync maintenance (DAILY_MAINTENANCE_SCRIPTS): -# media_shares_permissions.sh — fix ownership before arr cleanup -# media_cleaner.sh anime — remove junk from anime shares -# media_cleaner.sh media — remove junk from media shares -# lidarr_cleanup.sh — remove orphaned music files (local arr = truth) -# sonarr_cleanup.sh — remove orphaned TV files (local arr = truth) -# radarr_cleanup.sh — remove orphaned movie files (local arr = truth) -# docker_daily_restart.sh — restart containers needing daily restart +# Arr Sync (arr_sync.sh): +# Syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally. +# All nodes agree on tracked library before any files are transferred. +# Remote nodes that don't have an arr running are skipped gracefully. # -# ── WHY ORDER MATTERS ───────────────────────────────────────────────────────────────────────── -# git pull first — maintenance runs on latest code, not yesterday's -# arr sync before rsync — all nodes track the same library before files are spread; -# prevents remote arrs from searching for content already owned -# rsync before cleanup — cleanup sees fully spread state, rsync has no --delete -# permissions before arr cleanup — arrs need correct ownership to delete/rename -# arr cleanup after permissions — clean ownership = successful orphan deletion -# docker restart last — containers already processed by cleanup +# Rsync window (DAILY_SYNC_SHARES per host): +# HOST*_DAILY_SYNC_SHARES — media shares spread to all nodes (no --delete) +# HOST*_PERSONAL_SHARES — encrypted personal shares # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# Bidirectional — same script runs on both servers, correct direction automatic. -# detect_hosts() aliases DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars. -# No manual HOST1/HOST2 comparisons — MY_ID routes correctly on any server. +# Post-sync maintenance (DAILY_MAINTENANCE_SCRIPTS): +# media_shares_permissions.sh — fix ownership before arr cleanup +# media_cleaner.sh anime/media — remove junk from media and anime shares +# lidarr/sonarr/radarr_cleanup.sh — remove orphaned files (local arr = truth) +# docker_daily_restart.sh — restart containers needing daily restart # -# ── DRIVE TEMP HANDLING ─────────────────────────────────────────────────────────────────────── -# rsync.sh returns exit codes for temperature issues: -# exit 1 = temp WARN — skip this share, continue to next -# exit 2 = temp CRITICAL — abort ALL remaining syncs in this window -# All other failures — skip share, continue to next +# DRIVE TEMP HANDLING (rsync.sh exit codes): +# exit 1 = temp WARN → skip this share, continue to next +# exit 2 = temp CRITICAL → abort ALL remaining syncs in this window # -# ── SILENT WHEN HEALTHY ─────────────────────────────────────────────────────────────────────── -# Runs daily at 1am — clean run should produce minimal output. -# Each job reports log() on success (silent), warn()/error() on failure (visible). -# Summary always shown — gives window timing and share/job counts. -# Notify only on failure — successful daily maintenance doesn't need notification. +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== # -# ── CONFIGURATION (master.conf + host*.conf) ─────────────────────────────────────────── -# HOST*_DAILY_SYNC_SHARES — shares pushed to mirror each day -# HOST*_PERSONAL_SHARES — encrypted personal shares -# DAILY_MAINTENANCE_SCRIPTS — maintenance jobs (permissions, cleanup, restart) -# DAILY_RSYNC_ENABLED — enable/disable rsync section +# Order Is Load-Bearing +# git pull first — maintenance runs on latest code, not yesterday's. Arr sync +# before rsync — all nodes track the same library before files are spread, +# preventing remote arrs from searching for content already owned. Rsync before +# cleanup — cleanup sees fully spread state. Permissions before arr cleanup — +# arrs need correct ownership to delete/rename. Docker restart last — containers +# already processed by cleanup. +# +# Host-Aware Routing +# Bidirectional — same script runs on both servers, correct direction automatic. +# detect_hosts() aliases DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars. +# No manual HOST1/HOST2 comparisons needed. +# +# Silent When Healthy +# Runs daily at 1am — clean runs produce minimal output. Each job logs silently +# on success; failures surface to warn()/error(). Notify only on failure. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root check — rsync and docker operations require root +# acquire_lock — prevents concurrent daily windows +# detect_hosts() — aliases correct per-host share lists +# check_connectivity — verified before any rsync +# check_remote_rootfs — aborts rsync if remote rootfs nearly full +# Non-fatal jobs — a failed job logs and continues; remaining jobs still run +# notify on failure — successful daily run produces no notification +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# host*.conf +# +# HOST*_DAILY_SYNC_SHARES — shares pushed to mirror each day +# HOST*_PERSONAL_SHARES — encrypted personal shares +# +# master.conf +# +# DAILY_MAINTENANCE_SCRIPTS — maintenance jobs (permissions, cleanup, restart) +# DAILY_RSYNC_ENABLED — enable/disable rsync section +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# daily_sync_maintenance.sh +# Normal run. +# +# daily_sync_maintenance.sh --dry-run +# Preview without syncing or changing. +# +# daily_sync_maintenance.sh --log +# Verbose per-share/per-job output. +# +# daily_sync_maintenance.sh --status +# Show configured shares and jobs. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# daily_sync_maintenance.sh — normal run -# daily_sync_maintenance.sh --dry-run — preview without syncing or changing -# daily_sync_maintenance.sh --log — verbose per-share/per-job output -# daily_sync_maintenance.sh --status — show configured shares and jobs # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Orchestrators/intermediate_sync_maintenance.sh b/Orchestrators/intermediate_sync_maintenance.sh index 8936923..406c904 100755 --- a/Orchestrators/intermediate_sync_maintenance.sh +++ b/Orchestrators/intermediate_sync_maintenance.sh @@ -2,56 +2,86 @@ # ============================================================================================== # =========================== Intermediate Sync Maintenance ==================================== # ============================================================================================== -# 4-hour orchestrator — arr library reconciliation, artwork fetching, and optional rsync. -# Schedule: 0 */4 * * * (every 4 hours) # -# ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── -# 1. conf_sync.sh --pull-only — refresh partner conf cache in RAM (/tmp/.cache/vv/d/) -# 2. arr_sync.sh — sync Lidarr/Sonarr/Radarr libraries across all nodes -# 3. Rsync window (optional) — INTERMEDIATE_SYNC_SHARES, if any configured -# 4. INTERMEDIATE_MAINTENANCE_SCRIPTS — artwork fetch and any future 4-hour jobs +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# 4-hour orchestrator — arr library reconciliation, artwork fetching, and +# optional rsync. Schedule: 0 */4 * * * (every 4 hours) # -# ── WHY A SEPARATE ORCHESTRATOR ─────────────────────────────────────────────────────────────── -# arr libraries need to converge more frequently than once a day. If a remote node adds -# something at 2am, the next daily window is 23 hours away — remote arrs search for content -# they don't know is already owned. Running every 4 hours closes that gap. +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# lidarr_missing_art.sh is idempotent — skips existing files, runs fast after initial fill. -# Pairing it here means artwork catches up within 4 hours of a new album landing. +# 1. conf_sync.sh --pull-only — refresh partner conf cache in RAM (/tmp/.cache/vv/d/) +# 2. arr_sync.sh — sync Lidarr/Sonarr/Radarr libraries across all nodes +# 3. Rsync window (optional) — INTERMEDIATE_SYNC_SHARES, if any configured +# 4. INTERMEDIATE_MAINTENANCE_SCRIPTS — artwork fetch and any future 4-hour jobs # -# Rsync is optional — INTERMEDIATE_SYNC_SHARES empty by default. Add shares to the config -# if a subset of data needs mid-day propagation (e.g. watch state, metadata). Full media -# share sync stays in the daily window. +# DRIVE TEMP HANDLING (rsync.sh exit codes): +# exit 1 = temp WARN → skip this share, continue to next +# exit 2 = temp CRITICAL → abort ALL remaining syncs in this window # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID and aliases HOST*_INTERMEDIATE_SYNC_SHARES → INTERMEDIATE_SYNC_SHARES. -# Each server can have a different set of mid-day shares — configure in host*.conf. -# Each script in INTERMEDIATE_MAINTENANCE_SCRIPTS handles its own host logic. +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== # -# ── DRIVE TEMP HANDLING ─────────────────────────────────────────────────────────────────────── -# Same as daily_sync_maintenance.sh: -# exit 1 = temp WARN — skip this share, continue to next -# exit 2 = temp CRITICAL — abort ALL remaining syncs in this window +# Closes the Library Gap +# Arr libraries need to converge more frequently than once a day. If a remote +# node adds something at 2am, the next daily window is 23 hours away — remote +# arrs search for content they don't know is already owned. Running every 4 +# hours closes that gap. +# +# Idempotent Artwork +# lidarr_missing_art.sh skips existing files and runs fast after initial fill. +# Pairing it here means artwork catches up within 4 hours of a new album landing. +# +# Optional Rsync Layer +# INTERMEDIATE_SYNC_SHARES is empty by default — the rsync step is skipped +# entirely when nothing is configured. Add shares only if a subset of data +# needs mid-day propagation. Full media share sync stays in the daily window. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── # Root check — scripts called here require root # acquire_lock — prevents concurrent intermediate windows +# detect_hosts() — aliases correct per-host share lists # check_connectivity — verified before any rsync (skipped if no shares) # check_remote_rootfs — aborts rsync if remote rootfs nearly full # Non-fatal jobs — a failed arr_sync warns but does not block rsync or artwork fetch # Silent on success — runs 4x/day, only failures warrant notification # -# ── CONFIGURATION ───────────────────────────────────────────────────────────────────────────── -# host*.conf: HOST*_INTERMEDIATE_SYNC_SHARES — shares synced mid-day (empty = rsync skipped) -# master.conf: INTERMEDIATE_RSYNC_ENABLED — enable/disable rsync section (default: true) -# master.conf: INTERMEDIATE_MAINTENANCE_SCRIPTS — jobs run after rsync -# master.conf: ARR_SYNC_ENABLED — toggle inside arr_sync.sh +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# host*.conf +# +# HOST*_INTERMEDIATE_SYNC_SHARES — shares synced mid-day (empty = rsync skipped) +# +# master.conf +# +# INTERMEDIATE_RSYNC_ENABLED — enable/disable rsync section (default: true) +# INTERMEDIATE_MAINTENANCE_SCRIPTS — jobs run after rsync +# ARR_SYNC_ENABLED — toggle inside arr_sync.sh +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# intermediate_sync_maintenance.sh +# Normal run. +# +# intermediate_sync_maintenance.sh --dry-run +# Preview without changes. +# +# intermediate_sync_maintenance.sh --log +# Verbose per-job output. +# +# intermediate_sync_maintenance.sh --status +# Show configured shares/jobs and exit. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# intermediate_sync_maintenance.sh — normal run -# intermediate_sync_maintenance.sh --dry-run — preview without changes -# intermediate_sync_maintenance.sh --log — verbose per-job output -# intermediate_sync_maintenance.sh --status — show configured shares/jobs and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Orchestrators/monthly_maintenance.sh b/Orchestrators/monthly_maintenance.sh index cdf362d..ed15f5d 100755 --- a/Orchestrators/monthly_maintenance.sh +++ b/Orchestrators/monthly_maintenance.sh @@ -2,39 +2,86 @@ # ============================================================================================== # ========================= Monthly Maintenance Orchestrator =================================== # ============================================================================================== -# Uptime-triggered monthly maintenance — runs heavy tasks that need a stable, settled system. +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Uptime-triggered monthly maintenance — runs heavy tasks that need a stable, +# settled system. Schedule: 0 0 15 * * (15th of each month at midnight) # Fires only when BOTH gates pass: # 1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days # 2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run) # -# ── WHY UPTIME-GATED ───────────────────────────────────────────────────────────────────────── -# A scheduled reboot resets uptime. Monthly tasks (ZFS scrub, SMART long test) need a -# stable, settled system — not one that just rebooted. Uptime-gating ensures maintenance -# only runs after the server has been healthy for a full month, never immediately post-boot. -# If uptime or interval gate is not met on the 15th, the run is skipped until next month. +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── HOW TO CALL ────────────────────────────────────────────────────────────────────────────── -# Schedule: 0 0 15 * * (15th of each month at midnight) -# Silent exit 0 when either gate is not met. Only outputs when maintenance actually fires. +# Runs MONTHLY_MAINTENANCE_SCRIPTS sequentially when both gates pass. +# Silent exit 0 when either gate is not met — only outputs when maintenance fires. +# If uptime or interval gate is not met on the 15th, the run is skipped until +# next month. # -# ── STATE FILE ──────────────────────────────────────────────────────────────────────────────── -# MONTHLY_LAST_RUN_FILE — /boot/config — survives reboots, available before array starts. -# Written after each run (pass or partial fail). Format: Unix timestamp. -# A reboot does NOT reset the last-run state — the interval gate survives independently -# of the uptime gate. Both must pass before maintenance fires again. +# STATE FILE +# MONTHLY_LAST_RUN_FILE lives on /boot/config — survives reboots, available +# before the array starts. Written after each run (pass or partial fail). +# Format: Unix timestamp. A reboot does NOT reset the last-run state — the +# interval gate survives independently of the uptime gate. +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Uptime Gate Ensures Stability +# A scheduled reboot resets uptime. Monthly tasks (ZFS scrub, SMART long test) +# need a stable, settled system — not one that just rebooted. Both gates must +# pass before maintenance fires, ensuring the server has been healthy for a +# full month. +# +# State Survives Reboots +# MONTHLY_LAST_RUN_FILE is on /boot/config (USB flash), not on the array. +# It is always available regardless of array state, so the interval gate is +# never lost to a reboot. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root check — ZFS scrub, SMART tests require root +# acquire_lock — prevents concurrent monthly runs +# detect_hosts() — MY_ID in notifications and logs +# Uptime gate — MONTHLY_UPTIME_THRESHOLD_DAYS must be met +# Interval gate — MONTHLY_RUN_INTERVAL_DAYS since last run must be met +# --force flag — bypasses both gates for manual override +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── # MONTHLY_MAINTENANCE_SCRIPTS — ordered list of scripts to run # MONTHLY_UPTIME_THRESHOLD_DAYS — minimum uptime in days before maintenance fires # MONTHLY_RUN_INTERVAL_DAYS — minimum days since last run before running again -# MONTHLY_LAST_RUN_FILE — state file path — /boot/config, survives reboots +# MONTHLY_LAST_RUN_FILE — state file path (/boot/config — survives reboots) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# monthly_maintenance.sh +# Normal run — uptime + interval gates enforced. +# +# monthly_maintenance.sh --dry-run +# Preview gate state and scripts without running. +# +# monthly_maintenance.sh --status +# Show gate state, last run, and configured scripts. +# +# monthly_maintenance.sh --force +# Bypass uptime + interval gates (manual override). +# +# monthly_maintenance.sh --log +# Verbose output. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# monthly_maintenance.sh — normal run (uptime + interval gates enforced) -# monthly_maintenance.sh --dry-run — preview gate state and scripts without running -# monthly_maintenance.sh --status — show gate state, last run, and configured scripts -# monthly_maintenance.sh --force — bypass uptime + interval gates (manual override) -# monthly_maintenance.sh --log — verbose output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Orchestrators/sunday_morning_coffee_report.sh b/Orchestrators/sunday_morning_coffee_report.sh index e331d1c..7a8b7c8 100755 --- a/Orchestrators/sunday_morning_coffee_report.sh +++ b/Orchestrators/sunday_morning_coffee_report.sh @@ -2,24 +2,71 @@ # ============================================================================================== # ============================= Sunday Morning Coffee Report =================================== # ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── # Weekly monitoring orchestrator — runs all Sunday monitor scripts in sequence. -# Designed to be read over coffee Sunday morning while the system is fully caught up -# from the 2:30am maintenance window. -# Schedule: 0 7 * * 0 (7am Sunday — after weekly_sync_maintenance.sh finishes at ~3am) +# Designed to be read over coffee while the system is fully caught up from the +# 2:30am maintenance window. +# Schedule: 0 7 * * 0 (7am Sunday — after weekly_sync_maintenance.sh finishes) # -# ── SCRIPTS (master.conf COFFEE_REPORT_SCRIPTS) ─────────────────────────────────────────────── -# Each script runs independently, logs to its own output, and notifies on findings. -# All scripts receive --dry-run and --log flags from this orchestrator when set. +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID — used in banner and summary. -# HOST1 primary: runs all scripts. HOST2: limited to host-aware scripts only. +# Runs COFFEE_REPORT_SCRIPTS from master.conf in order. Each script runs +# independently, produces its own output, and notifies on findings. +# All scripts receive --dry-run and --log flags from this orchestrator when set. +# +# HOST1 primary: runs all scripts. +# HOST2: limited to host-aware scripts only — each script handles its own host logic. +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Independent, Isolated Scripts +# Each monitor script is fully self-contained — a failure in one does not +# prevent the others from running. The orchestrator logs the failure and +# continues to the next script. +# +# Runs After the Full Weekly Window +# Scheduled 4+ hours after weekly_sync_maintenance.sh — the system is fully +# synced and containers are back up before any monitoring reads run. Reports +# reflect the settled post-maintenance state. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# detect_hosts() — MY_ID in banner and summary +# Non-fatal steps — a failed script is logged; remaining scripts still run +# Flag pass-through — --dry-run and --log forwarded to all child scripts +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# COFFEE_REPORT_SCRIPTS — ordered list of monitor scripts to run +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# sunday_morning_coffee_report.sh +# Normal run. +# +# sunday_morning_coffee_report.sh --dry-run +# Preview without any writes or notifications. +# +# sunday_morning_coffee_report.sh --log +# Verbose per-script output. +# +# sunday_morning_coffee_report.sh --status +# Show configured scripts and exit. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# sunday_morning_coffee_report.sh — normal run -# sunday_morning_coffee_report.sh --dry-run — preview without any writes or notifications -# sunday_morning_coffee_report.sh --log — verbose per-script output -# sunday_morning_coffee_report.sh --status — show configured scripts and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Orchestrators/transcode_management.sh b/Orchestrators/transcode_management.sh index cc3094a..dd7907e 100755 --- a/Orchestrators/transcode_management.sh +++ b/Orchestrators/transcode_management.sh @@ -2,55 +2,82 @@ # ============================================================================================== # ============================= Transcode Management =========================================== # ============================================================================================== -# Orchestrator — runs transcode_cleanup.sh then transcode_manager.sh in the correct order. -# Replace individual transcode_manager and transcode_cleanup cron entries with this. +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Runs transcode_cleanup.sh then transcode_manager.sh in the correct order. +# Replaces individual cron entries for each — this is the single cron entry. # Schedule: */7 * * * * (every 7 minutes via User Scripts plugin) # -# ── WHY CLEANUP BEFORE MANAGER ──────────────────────────────────────────────────────────────── -# Cleanup runs first — removes stale segment files from ended sessions. -# Manager runs after — threshold decisions based on real current usage post-cleanup. -# Without this order, stale files inflate the ramdisk usage reading and trigger -# unnecessary SSD flips even when active sessions would fit on the ramdisk. +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── WHAT EACH SCRIPT DOES ───────────────────────────────────────────────────────────────────── -# transcode_cleanup.sh — removes aged segment files not open by any process -# uses lsof for O(1) per-file active check (never per-file lsof) -# also triggers flip-back to ramdisk after cleanup if recovered ✅ +# transcode_cleanup.sh +# Removes aged segment files not open by any process. Uses lsof for O(1) +# per-file active check. Triggers flip-back to ramdisk after cleanup if +# the ramdisk usage has recovered. # -# transcode_manager.sh — checks ramdisk usage against thresholds -# flips symlink between ramdisk and SSD as needed -# writes one entry to TRANSCODE_DAILY_LOG after each run -# shows active Emby sessions with play method +# transcode_manager.sh +# Checks ramdisk usage against thresholds. Flips the symlink between ramdisk +# and SSD as needed. Writes one entry to TRANSCODE_DAILY_LOG after each run. +# Shows active Emby sessions with play method. # -# ── DAILY LOG ───────────────────────────────────────────────────────────────────────────────── -# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run: +# DAILY LOG (written by transcode_manager.sh, not this orchestrator): # Format: DATE|RAMDISK_USED_GB|FLIP_COUNT|RAM_SESSIONS|SSD_SESSIONS -# This orchestrator does NOT write its own log — manager handles it ✅ -# Log trimmed to TRANSCODE_LOG_RETENTION days by manager on each write. -# Read by sunday_morning_coffee_report.sh and weekly_health_digest.sh. +# Trimmed to TRANSCODE_LOG_RETENTION days on each write. +# Read by sunday_morning_coffee_report.sh and weekly_health_digest.sh. # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID and aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_WARN_GB etc. -# Each server manages its own transcode location independently. +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Cleanup First, Decide Later +# Stale segment files from ended sessions inflate the ramdisk usage reading +# and trigger unnecessary SSD flips even when active sessions would fit on +# the ramdisk. Cleanup runs first so the manager measures real current usage. +# +# Delegated Logging +# This orchestrator does not write its own log — transcode_manager.sh owns +# the TRANSCODE_DAILY_LOG write. One log writer, one format, no duplication. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── # Root check — mount and docker operations require root -# acquire_lock — prevents concurrent 3-minute cycles overlapping -# detect_hosts() — correct paths per host +# acquire_lock — prevents concurrent 7-minute cycles overlapping +# detect_hosts() — aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_WARN_GB per host # --dry-run — passed through to both child scripts -# Exit code — worst exit code of both scripts returned +# Exit code — worst exit code of both scripts returned to cron # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# TRANSCODE_DAILY_LOG — daily stats log (written by transcode_manager.sh) +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# TRANSCODE_DAILY_LOG — daily stats log (written by transcode_manager.sh) # TRANSCODE_LOG_RETENTION — days to keep (trimmed by manager) -# TRANSCODE_STATE_FILE — current state (ramdisk_used, flip_count etc.) -# All TRANSCODE_* threshold vars — see master.conf Transcode Manager section +# TRANSCODE_STATE_FILE — current state (ramdisk_used, flip_count, etc.) +# TRANSCODE_* threshold vars — see master.conf Transcode Manager section +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# transcode_management.sh +# Normal run (every 7 minutes via cron). +# +# transcode_management.sh --dry-run +# Preview without changes (passed to both child scripts). +# +# transcode_management.sh --status +# Show configuration and current state. +# +# transcode_management.sh --log +# Verbose output from both child scripts. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# transcode_management.sh — normal run (every 7 minutes via cron) -# transcode_management.sh --dry-run — preview without changes (passed to children) -# transcode_management.sh --status — show configuration and current state -# transcode_management.sh --log — verbose output from both child scripts # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Orchestrators/watchdog_orchestrator.sh b/Orchestrators/watchdog_orchestrator.sh index a326923..aa891b5 100755 --- a/Orchestrators/watchdog_orchestrator.sh +++ b/Orchestrators/watchdog_orchestrator.sh @@ -2,50 +2,86 @@ # ============================================================================================== # ============================ Watchdog Orchestrator =========================================== # ============================================================================================== -# Runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle. +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle. Replaces the +# continuous loops previously embedded in individual watchdog scripts — those +# are now single-pass; this orchestrator provides the cadence. # Schedule: */15 * * * * (every 15 minutes) # -# ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── -# Driven by WATCHDOG_ORCHESTRATOR_SCRIPTS in master.conf — add, remove, or reorder there. -# Default: resource_watchdog → docker_watchdog → system_watchdog → unraid_api_key_renew → stability_watchdog +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── WHY ORDER MATTERS ───────────────────────────────────────────────────────────────────────── -# Resource Watchdog first — frees RAM and CPU before healing attempts container restarts. -# Containers restarted into a resource-pressured system just fail again. -# Docker Watchdog second — restarts with pressure already reduced, more likely to stabilise. -# System Watchdog third — system component health after containers are healed. -# API key renew fourth — self-heals unraid-api registry loss; check-first, silent when valid. -# Stability Watchdog last — only reboots when all prior layers could not resolve the issue. +# Order driven by WATCHDOG_ORCHESTRATOR_SCRIPTS in master.conf. +# Default: resource_watchdog → docker_watchdog → system_watchdog → +# unraid_api_key_renew → stability_watchdog # -# ── ARRAY CHECK ─────────────────────────────────────────────────────────────────────────────── -# Exits immediately if /mnt/user is not mounted as shfs (array not started). -# Watchdogs check Docker containers and storage — meaningless without the array. -# Prevents false positives and unnecessary reboots when array is stopped or stopping. +# ARRAY CHECK +# Exits immediately if /mnt/user is not mounted as shfs. Watchdogs check +# Docker containers and storage — meaningless without the array. Prevents +# false positives and unnecessary reboots when array is stopped or stopping. # -# ── STARTUP GRACE ───────────────────────────────────────────────────────────────────────────── -# No action until system uptime >= WATCHDOG_STARTUP_GRACE seconds. -# Prevents false positives from containers still starting at array launch. -# Each sub-script enforces this independently — orchestrator exits early to avoid log noise. +# STARTUP GRACE +# No action until system uptime >= WATCHDOG_STARTUP_GRACE seconds. Prevents +# false positives from containers still starting at array launch. Each +# sub-script enforces this independently — orchestrator exits early to avoid +# log noise. # -# ── OVERLAP PROTECTION ──────────────────────────────────────────────────────────────────────── -# acquire_lock() — exits immediately if a prior cycle is still in progress. -# Prevents pile-up when a cycle runs long (daemon restart attempt = 30s, etc.). +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== # -# ── REPLACES ────────────────────────────────────────────────────────────────────────────────── -# Continuous loops previously in system_watchdog.sh and docker_watchdog.sh. -# Those scripts are now single-pass — this orchestrator provides the cadence. +# Pressure Before Healing +# Resource Watchdog runs first — it frees RAM and CPU before any container +# restart is attempted. Containers restarted into a resource-pressured system +# just fail again. Docker Watchdog restarts with pressure already reduced. +# System Watchdog checks component health after containers are healed. +# Stability Watchdog reboots only when all prior layers could not resolve the +# issue. API key renew is check-first and silent when valid. +# +# Never Queue +# acquire_lock exits immediately if a prior cycle is still running. Prevents +# pile-up when a cycle runs long (daemon restart attempt = 30s, etc.). +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root check — watchdog operations require root +# acquire_lock — strict; no pile-up if prior cycle still active +# detect_hosts() — MY_ID in logs and notifications +# Array check — exits early if /mnt/user is not shfs-mounted +# Startup grace — WATCHDOG_STARTUP_GRACE respected before any checks +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── # WATCHDOG_ORCHESTRATOR_SCRIPTS — watchdogs to run, in order # WATCHDOG_STARTUP_GRACE — seconds after boot before checks activate # WATCHDOG_ORCHESTRATOR_HEARTBEAT — periodic heartbeat log toggle # WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# watchdog_orchestrator.sh — normal run (called by cron every 15 minutes) -# watchdog_orchestrator.sh --dry-run — pass --dry-run to all sub-scripts -# watchdog_orchestrator.sh --status — show script paths and current grace state -# watchdog_orchestrator.sh --log — verbose output from all sub-scripts +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# watchdog_orchestrator.sh +# Normal run (called by cron every 15 minutes). +# +# watchdog_orchestrator.sh --dry-run +# Pass --dry-run to all sub-scripts. +# +# watchdog_orchestrator.sh --status +# Show script paths and current grace state. +# +# watchdog_orchestrator.sh --log +# Verbose output from all sub-scripts. +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Orchestrators/weekly_sync_maintenance.sh b/Orchestrators/weekly_sync_maintenance.sh index 4c7dff8..895c811 100755 --- a/Orchestrators/weekly_sync_maintenance.sh +++ b/Orchestrators/weekly_sync_maintenance.sh @@ -2,57 +2,82 @@ # ============================================================================================== # ============================= Weekly Sync Maintenance ======================================== # ============================================================================================== -# Weekly maintenance window orchestrator — clean sync, container updates, weekly restarts. -# Schedule: 30 2 * * 0 (Sunday 2:30am — before Sunday 7am coffee report) # -# ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── -# 1. Stop local containers — Emby + auth stack stopped locally -# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH -# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true (zero extra downtime) -# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true -# 5. rsync WEEKLY_SYNC_SHARES — full clean mirror, containers stopped both sides -# 6. Start remote containers — correct order, delayed start respected -# 7. Start local containers — rebuild if new image pulled, docker start otherwise -# 8. WEEKLY_MAINTENANCE_SCRIPTS — weekly restarts etc. (docker_weekly_restart.sh) +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Weekly maintenance window orchestrator — clean Emby sync, container updates, +# and weekly restarts. Schedule: 30 2 * * 0 (Sunday 2:30am) # -# ── WHY WEEKLY NOT NIGHTLY FOR EMBY ────────────────────────────────────────────────────────── -# Emby builds a warm image cache on HOST2 throughout the week. -# Syncing nightly resets cache — cold loads every morning for users. -# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep. -# play_state_sync covers watch/resume state every 30 min between weekly syncs. +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── CONTAINER UPDATES ───────────────────────────────────────────────────────────────────────── -# Containers already stopped for sync — updates pull at zero extra downtime. -# Both servers start on identical image versions after the window completes. -# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in master.conf +# 1. Stop local containers — Emby + auth stack stopped locally +# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH +# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true (zero extra downtime) +# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true +# 5. rsync WEEKLY_SYNC_SHARES — full clean mirror, containers stopped both sides +# 6. Start remote containers — correct order, delayed start respected +# 7. Start local containers — rebuild if new image pulled, docker start otherwise +# 8. WEEKLY_MAINTENANCE_SCRIPTS — weekly restarts etc. (docker_weekly_restart.sh) # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID — used in banner, summary, and notifications. -# WEEKLY_SYNC_SHARES and WEEKLY_MAINTENANCE_SCRIPTS configured in master.conf. -# Same script runs correctly on both servers. +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# Root check — stop/start containers, rsync require root -# acquire_lock — prevents concurrent weekly windows -# check_connectivity — verifies remote before any remote operations -# check_remote_rootfs — aborts if remote rootfs nearly full -# DOCKER_TIMEOUT — all docker calls protected -# SSH_TIMEOUT — all SSH calls protected +# Weekly Cadence Preserves Cache +# Emby builds a warm image cache on HOST2 throughout the week. Syncing nightly +# resets that cache — cold loads every morning for users. Weekly sync keeps +# the cache warm for 6 days, resets Sunday night while users sleep. +# play_state_sync covers watch/resume state every 30 min between weekly syncs. +# +# Zero-Downtime Updates +# Containers are already stopped for the sync window — image pulls happen at +# zero extra downtime. Both servers start on identical image versions after +# the window completes. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root check — stop/start containers and rsync require root +# acquire_lock — prevents concurrent weekly windows +# detect_hosts() — MY_ID in banner, summary, and notifications +# check_connectivity — verifies remote before any remote operations +# check_remote_rootfs — aborts rsync if remote rootfs nearly full +# DOCKER_TIMEOUT — all docker calls protected +# SSH_TIMEOUT — all SSH calls protected # platform_require_cmd — notify validated before use -# Silent on success — runs weekly, only failures warrant notification +# Silent on success — runs weekly; only failures warrant notification # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# WEEKLY_SYNC_SHARES — shares synced during window -# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync -# WEEKLY_SYNC_UPDATES — toggle local container updates -# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates -# WEEKLY_RSYNC_ENABLED — enable/disable rsync section +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# WEEKLY_SYNC_SHARES — shares synced during window +# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync +# WEEKLY_SYNC_UPDATES — toggle local container updates +# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates +# WEEKLY_RSYNC_ENABLED — enable/disable rsync section +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# weekly_sync_maintenance.sh +# Normal run. +# +# weekly_sync_maintenance.sh --dry-run +# Preview without stopping containers or syncing. +# +# weekly_sync_maintenance.sh --log +# Verbose per-share/per-job output. +# +# weekly_sync_maintenance.sh --status +# Show configuration and exit. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# weekly_sync_maintenance.sh — normal run -# weekly_sync_maintenance.sh --dry-run — preview without stopping containers or syncing -# weekly_sync_maintenance.sh --log — verbose per-share/per-job output -# weekly_sync_maintenance.sh --status — show configuration and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Plugin/build.sh b/Plugin/build.sh index d754d84..538e9ed 100755 --- a/Plugin/build.sh +++ b/Plugin/build.sh @@ -26,15 +26,20 @@ # rebuild cron when the array mounts. Scripts stay on appdata (git clone). # # ============================================================================================== -# USAGE +# RUNTIME MODES # ============================================================================================== -# ./build.sh # version = today's date (YYYY.MM.DD) -# ./build.sh 2026.09.01 # explicit version +# +# ./build.sh +# Build with today's date as version (YYYY.MM.DD). +# +# ./build.sh 2026.09.01 +# Build with an explicit version string. # # After building: commit Plugin/dist/ + the updated .plg, then attach the # .txz to a GitHub release tagged so the .plg URL resolves for # downloaders. (The .plg also works offline if the .txz is already cached on # flash with a matching SHA256.) +# # ============================================================================================== set -euo pipefail diff --git a/Plugin/unraid/Partnership/containers.sh b/Plugin/unraid/Partnership/containers.sh index a6d185e..4ac81d5 100755 --- a/Plugin/unraid/Partnership/containers.sh +++ b/Plugin/unraid/Partnership/containers.sh @@ -3,15 +3,25 @@ # ====================== Partnership — Unraid Container Adapter ================================ # ============================================================================================== # -# Sourced by partnership_onboard.sh and partnership_offboard.sh via: +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Platform adapter providing container deploy/cleanup functions for the Unraid +# partnership system. Sourced (not executed) by partnership_onboard.sh and +# partnership_offboard.sh via: # source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh" # -# Provides container deploy/cleanup functions specific to the Unraid platform: -# - Docker container deployment from Unraid CA XML templates +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# Functions use variables from the calling script's scope (sourced, not exec'd): +# Functions operate on variables from the calling script's scope: # MIRROR, MIRROR_IP, MIRROR_SSH_KEY, SSH_TIMEOUT, DRY_RUN, SCRIPTS_ROOT # +# Capabilities: +# - Docker container deployment from Unraid CA XML templates +# - Remote GPU type detection (cached per session — one SSH call per onboard) +# - Deployed stack tracking via _STACK_DEPLOYED / _STACK_FAILED counters +# # ============================================================================================== TEMPLATES_DIR="/boot/config/plugins/dockerMan/templates-user" diff --git a/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh b/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh index b928e9e..8544e19 100755 --- a/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh +++ b/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh @@ -14,6 +14,29 @@ # page always reflects the live key value. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Self-Healing at Boot +# The unraid-api registry is ephemeral — OS updates and service restarts clear +# it without warning. Running at every array start means the key is always +# present after boot without any manual intervention. +# +# Conf Stays Current +# HOST*_UNRAID_API_KEY in the local host conf is updated after every renewal. +# The partnership page reads the conf — it always reflects the live key value +# without a separate sync step. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# acquire_lock — prevents concurrent renewal attempts at boot +# detect_hosts() — sets MY_ID to derive the correct conf var name +# Conf file check — aborts before any writes if the host conf is missing +# dry-run mode — shows what would happen without touching anything +# +# ============================================================================================== # RUNTIME MODES # ============================================================================================== # diff --git a/Plugin/unraid/Tools/remote_arr_cache_writer.sh b/Plugin/unraid/Tools/remote_arr_cache_writer.sh index 8997af5..06a57d7 100755 --- a/Plugin/unraid/Tools/remote_arr_cache_writer.sh +++ b/Plugin/unraid/Tools/remote_arr_cache_writer.sh @@ -15,7 +15,29 @@ # Accepts --host=HOST2 to refresh a single host (used by the UI refresh button). # # ============================================================================================== -# USAGE +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Cache-First, Never Live on Page Load +# Remote arr APIs have non-trivial latency — calling them on every page view +# would make the arrs page slow and fragile. Writing to /tmp/vv_cache/ on a +# 2-hour schedule decouples page load time from network availability. +# +# Single-Host Refresh for UI +# The UI refresh button passes --host=HOSTN to update one host without waiting +# for the full 2-hour cycle. Keeps the cache fresh when a user requests it. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# acquire_lock — prevents concurrent cache write runs +# detect_hosts() — MY_ID and partner host list +# SSH reachability — skips a host cleanly if it cannot be reached +# /tmp/vv_cache/ — auto-created if missing; cleared on reboot (intentional) +# +# ============================================================================================== +# RUNTIME MODES # ============================================================================================== # # remote_arr_cache_writer.sh — refresh all remote hosts diff --git a/Plugin/unraid/Tools/storage_migrate.sh b/Plugin/unraid/Tools/storage_migrate.sh index 91ff5a2..ef53be2 100755 --- a/Plugin/unraid/Tools/storage_migrate.sh +++ b/Plugin/unraid/Tools/storage_migrate.sh @@ -24,6 +24,29 @@ # varaverk.cron rebuilt via PHP (job paths regenerated for new SCRIPTS_DIR) # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Atomic Path Transition +# All four files (varaverk.cfg, master.conf, host*.conf, varaverk.cron) are +# updated in a single pass. A partial migration would leave cron entries +# pointing at the wrong SCRIPTS_DIR — all or nothing. +# +# PHP Rebuilds Cron +# Job paths in varaverk.cron are derived from SCRIPTS_DIR. Rather than +# text-substituting the cron file, the script regenerates it via PHP using +# the new SCRIPTS_DIR as the source of truth. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# acquire_lock — prevents concurrent migration attempts +# dry-run mode — shows all changes without touching any file +# --status mode — reports current mode without requiring a target +# --to= required — refuses to run without an explicit target mode +# +# ============================================================================================== # RUNTIME MODES # ============================================================================================== # diff --git a/Plugin/unraid/adapter.sh b/Plugin/unraid/adapter.sh index 8e0fc15..8580ff5 100755 --- a/Plugin/unraid/adapter.sh +++ b/Plugin/unraid/adapter.sh @@ -2,42 +2,52 @@ # ============================================================================================== # ================================= Unraid Platform Adapter ==================================== # ============================================================================================== -# Sourced by load_config.sh when PLATFORM=unraid. -# Provides the platform_*() API — bash scripts call these instead of OS-specific commands. # -# ── API CONTRACT ────────────────────────────────────────────────────────────────────────────── -# Every function returns 0 on success / 1 on failure unless noted. -# Functions that produce output write to stdout; callers capture with $(). -# No function calls exit — callers decide what failure means for their flow. +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Platform abstraction layer for Unraid. Sourced by load_config.sh when +# PLATFORM=unraid. Scripts call platform_*() functions instead of OS-specific +# commands — the adapter isolates all OS-dependent logic in one place. # -# ── ADDING A PLATFORM ───────────────────────────────────────────────────────────────────────── -# Create Plugin/truenas/adapter.sh (or ubuntu/adapter.sh) implementing the same function names. -# load_config.sh sources Plugin/$PLATFORM/adapter.sh — no other changes needed. +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── FUNCTIONS ───────────────────────────────────────────────────────────────────────────────── -# platform_require_cmd — verify a platform command exists and is executable -# platform_storage_healthy — array mounted and shfs active on /mnt/user -# platform_disk_states_path — path to the platform disk state file -# platform_get_disk_states — raw disk state content from platform -# platform_get_temp_thresholds — disk warn/crit °C from platform config +# API CONTRACT +# Every function returns 0 on success / 1 on failure unless noted. +# Functions that produce output write to stdout; callers capture with $(). +# No function calls exit — callers decide what failure means for their flow. +# +# ADDING A PLATFORM +# Create Plugin/truenas/adapter.sh (or ubuntu/adapter.sh) implementing the +# same function names. load_config.sh sources Plugin/$PLATFORM/adapter.sh — +# no other changes needed anywhere in the codebase. +# +# FUNCTIONS +# platform_require_cmd — verify a platform command exists and is executable +# platform_storage_healthy — array mounted and shfs active on /mnt/user +# platform_disk_states_path — path to the platform disk state file +# platform_get_disk_states — raw disk state content from platform +# platform_get_temp_thresholds — disk warn/crit °C from platform config # platform_is_maintenance_running — parity check/sync in progress -# platform_is_service_enabled — docker or libvirt enabled in boot config -# platform_restart_service — restart a named service via rc.d -# platform_stop_service — stop a named service via rc.d -# platform_is_service_running — check if a named service process is alive -# platform_is_mover_running — unRAID mover process check -# platform_stop_user_scripts — kill all user.scripts background processes -# platform_send_os_notification — native unRAID notify (dynamix) -# platform_storage_path — root path for user shares/storage (e.g. /mnt/user) -# platform_webui_install_path — where the platform serves the WebGUI plugin files from +# platform_is_service_enabled — docker or libvirt enabled in boot config +# platform_restart_service — restart a named service via rc.d +# platform_stop_service — stop a named service via rc.d +# platform_is_service_running — check if a named service process is alive +# platform_is_mover_running — unRAID mover process check +# platform_stop_user_scripts — kill all user.scripts background processes +# platform_send_os_notification — native unRAID notify (dynamix) +# platform_storage_path — root path for user shares/storage (e.g. /mnt/user) +# platform_webui_install_path — where the platform serves the WebGUI plugin files from # platform_scripts_dir_probe_cmd — shell command to run on a remote to discover its SCRIPTS_DIR -# platform_get_templates_dir — path to Unraid CA docker templates-user directory -# platform_setup_db_path — path to the persistent Varaverk setup/wizard state database -# platform_get_os_version — local OS version string (e.g. "7.2.3") -# platform_os_version_probe_cmd — shell command to run on a remote to retrieve its OS version -# platform_rebuild_container — rebuild a container from its stored XML template -# platform_push_conf — push master.conf to all listed hosts via WebGUI PHP -# platform_push_setup_state — push wizard setup state to WebGUI PHP +# platform_get_templates_dir — path to Unraid CA docker templates-user directory +# platform_setup_db_path — path to the persistent Varaverk setup/wizard state database +# platform_get_os_version — local OS version string (e.g. "7.2.3") +# platform_os_version_probe_cmd — shell command to run on a remote to retrieve its OS version +# platform_rebuild_container — rebuild a container from its stored XML template +# platform_push_conf — push master.conf to all listed hosts via WebGUI PHP +# platform_push_setup_state — push wizard setup state to WebGUI PHP +# # ============================================================================================== # ────────────────────────────────────────────────────────────────────────────────────────────── diff --git a/Plugin/unraid/run_job.sh b/Plugin/unraid/run_job.sh index c7c3faa..51f3f9b 100755 --- a/Plugin/unraid/run_job.sh +++ b/Plugin/unraid/run_job.sh @@ -1,19 +1,77 @@ #!/bin/bash -# Varaverk job runner — wraps script execution with JSON status tracking. -# Called by /etc/cron.d/varaverk for every scheduled job. +# ============================================================================================== +# ============================= Job Runner ===================================================== +# ============================================================================================== # -# Usage: bash run_job.sh [flags...] +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Wraps every scheduled script execution with JSON status tracking and log +# management. Called by /etc/cron.d/varaverk for every scheduled job. The PHP +# dashboard polls the JSON files to show live job status without running scripts. # -# Flags consumed by run_job.sh (stripped before passing to script): -# --manual — marks a UI-triggered run; writes a sentinel on completion so -# the next cron fire is suppressed if it falls within the job's -# own cron interval (prevents double-firing after manual run). +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# Writes: /var/log/varaverk/.json — status, timestamps, exit code, pid -# /var/log/varaverk/.log — appended per run, trimmed to LOG_MAX_LINES -# /var/log/varaverk/.manual_ts — sentinel: epoch of last manual completion +# Invocation: bash run_job.sh [flags...] +# +# Writes three files per job to /var/log/varaverk/: +# .json — status, timestamps, exit code, pid (polled by WebGUI) +# .log — appended per run, trimmed to LOG_MAX_LINES lines +# .manual_ts — sentinel: epoch of last manual completion (interval suppression) # # Status values: running → ok (exit 0) | warn (exit 1) | error (exit 2+) +# +# MANUAL FLAG +# --manual marks a UI-triggered run. On completion, writes a manual_ts sentinel. +# The next cron fire reads the sentinel and suppresses itself if the elapsed time +# is within the job's own cron interval — prevents double-firing after a manual run. +# Static schedules (e.g. "30 2 * * 0") are never suppressed — only */N intervals. +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Suppress Double-Fire +# When a user triggers a job from the UI, the next cron fire within the job's +# own interval is skipped. A 30-minute cron job triggered at HH:14 won't fire +# again at HH:30 — it waits for HH:44. Static schedules are never suppressed. +# +# Status as Ground Truth +# The JSON file is overwritten atomically on every state change (start → end). +# The WebGUI polls it directly — no additional IPC or database needed. +# +# Log Trim on Every Write +# The log file is trimmed to LOG_MAX_LINES after every run. Never grows +# unbounded regardless of how long the server runs or how often the job fires. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# --manual stripped — flag consumed here, never passed to the wrapped script +# mkdir -p — log dir created if missing before any write +# Log trim — tail -n LOG_MAX_LINES via tmp file + mv (atomic) +# Sentinel cleanup — manual_ts removed after it's used or expired +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# LOG_DIR /var/log/varaverk (hardcoded — tmpfs on Unraid, cleared on reboot) +# LOG_MAX_LINES 1000 (hardcoded — trim threshold per job log) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# bash run_job.sh [script_flags...] +# Normal cron-triggered run. +# +# bash run_job.sh --manual [script_flags...] +# UI-triggered run. Suppresses next cron fire within the job's interval. +# +# ============================================================================================== JOB_ID="$1" SCRIPT="$2" diff --git a/Plugin/unraid/user_script_plug-in.sh b/Plugin/unraid/user_script_plug-in.sh index 45de12d..d3b54d3 100755 --- a/Plugin/unraid/user_script_plug-in.sh +++ b/Plugin/unraid/user_script_plug-in.sh @@ -3,30 +3,36 @@ # ============================= USER SCRIPTS MASTER TEMPLATE =================================== # ============================================================================================== # -# Paste this file into a User Script entry. Uncomment ONE script block and set the schedule. -# Every script in the ecosystem is listed here — from the orchestrators that run it all, -# down to the individual scripts you can run standalone for specific tasks. +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Paste this file into a User Script entry. Uncomment ONE script block and set +# the schedule. Every script in the ecosystem is listed here — from the +# orchestrators that run it all, down to the individual scripts you can run +# standalone for specific tasks. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── HOW THIS ECOSYSTEM WORKS ────────────────────────────────────────────────────────────────── # When used as intended, only a handful of orchestrators need to be scheduled. -# The orchestrators handle everything else in the correct order — they call child scripts, -# manage timing and dependencies, track pass/fail, and send one notification per window. +# The orchestrators handle everything else in the correct order — they call child +# scripts, manage timing and dependencies, track pass/fail, and send one +# notification per window. # # You do not need to schedule every script below. The orchestrators cover it all: # # array_started.sh at array start — launches ALL startup scripts in order -# array_stopping.sh at array stop — graceful shutdown sequence -# watchdog_orchestrator.sh every 15 min — resource + docker + system watchdog -# transcode_management.sh every 7 min — cleanup then manager (order critical) -# critical_sync_maintenance.sh every 30 min — auth + Emby dirty sync + partnership -# intermediate_sync_maintenance.sh every 4 hours — arr library sync + artwork fetch -# daily_sync_maintenance.sh 1am daily — git + rsync + media + restart -# rsync.sh --profile=emby-fallback every 30 min — Emby watch state dirty sync -# weekly_sync_maintenance.sh 2:30am Sunday — clean sync + image updates -# sunday_morning_coffee_report.sh 7am Sunday — full weekly digest -# weekly_health_digest.sh 8am daily — profile-controlled health notification -# system_tuning_monitor.sh every 6 hours — inotify + php-fpm trend tracking -# monthly_maintenance.sh 15th monthly — uptime-gated heavy tasks (ZFS scrub, SMART long test) +# array_stopping.sh at array stop — graceful shutdown sequence +# watchdog_orchestrator.sh every 15 min — resource + docker + system watchdog +# transcode_management.sh every 7 min — cleanup then manager (order critical) +# critical_sync_maintenance.sh every 30 min — auth + play_state_sync + partnership +# intermediate_sync_maintenance.sh every 4 hours — arr library sync + artwork fetch +# daily_sync_maintenance.sh 1am daily — git + rsync + media + restart +# weekly_sync_maintenance.sh 2:30am Sunday — clean sync + image updates +# sunday_morning_coffee_report.sh 7am Sunday — full weekly digest +# weekly_health_digest.sh 8am daily — profile-controlled health notification +# system_tuning_monitor.sh every 6 hours — inotify + php-fpm trend tracking +# monthly_maintenance.sh 15th monthly — uptime-gated heavy tasks (ZFS scrub, SMART long test) # # ── INDIVIDUAL SCRIPTS ──────────────────────────────────────────────────────────────────────── # Every child script is also listed below, individually. @@ -227,10 +233,9 @@ # Auth stack to HOST2: NPM proxy rules, TLS certs, LLDAP user accounts, # Authelia config and policies. HOST2 auth always within 30min of HOST1. # -# rsync Emby dirty sync: -# Watch states, user activity, library delta — Emby stays running on both sides. -# WAL/SHM files excluded (unsafe to copy mid-write). HOST2 Emby restarts after sync -# to pick up config changes. +# play_state_sync: +# API-based watch state and resume position sync. Runs as a CRITICAL_MAINTENANCE_SCRIPT +# in this window. No rsync — uses Emby API to sync per-user watch state directly. # # partnership --check: # Reads remote state file. Increments offline counter on failed sync. @@ -295,22 +300,6 @@ # bash /boot/config/plugins/varaverk/Orchestrators/daily_sync_maintenance.sh -# ── RSYNC EMBY FALLBACK ─────────────────────────────────────────────────────────────────────── -# Schedule: */30 * * * * (every 30 minutes) -# Background: YES -# -# Keeps HOST2 Emby within 30 minutes of HOST1 on watch states and library changes. -# Direct rsync.sh call (not an orchestrator). Emby stays running on both sides. -# -# Syncs: users.db, library.db, authentication.db, config/ -# Skips: *.wal *.shm (unsafe mid-write), transcodes/, logs/, cache/ (volatile/local only) -# After: HOST2 Emby restarts to pick up any config changes from the sync. -# Result: if HOST1 fails, users resume from at most 30 minutes stale. -# -# bash /boot/config/plugins/varaverk/Rsync/rsync.sh \ -# /mnt/user/Media_Server/Emby --profile=emby-fallback - - # ── WEEKLY SYNC MAINTENANCE ─────────────────────────────────────────────────────────────────── # Schedule: 30 2 * * 0 (Sunday 2:30am — 4.5 hours before coffee report) # Background: YES @@ -524,7 +513,6 @@ # important-data Postgres-NextCloud + delayed: NextCloud # arrs_stack Sonarr, Radarr, Lidarr, Prowlarr, Bazarr, Pinchflat # emby Emby both sides (weekly clean sync — both instances stopped) -# emby-fallback nothing stopped (Emby stays running — dirty sync, WAL/SHM excluded) # [no profile] no containers stopped (media shares, plain data) # # bash /boot/config/plugins/varaverk/Rsync/rsync.sh \ @@ -536,8 +524,6 @@ # bash /boot/config/plugins/varaverk/Rsync/rsync.sh \ # /mnt/user/Media_Server/Emby --profile=emby # bash /boot/config/plugins/varaverk/Rsync/rsync.sh \ -# /mnt/user/Media_Server/Emby --profile=emby-fallback -# bash /boot/config/plugins/varaverk/Rsync/rsync.sh \ # /mnt/user/appdata-Fallback/Gmer4Lfe # bash /boot/config/plugins/varaverk/Rsync/rsync.sh /mnt/user/Movies # bash /boot/config/plugins/varaverk/Rsync/rsync.sh /mnt/user/Tv_Shows @@ -1422,7 +1408,7 @@ # # */30 * * * * every 30 minutes: # Orchestrators/critical_sync_maintenance.sh -# Rsync/rsync.sh /mnt/user/Media_Server/Emby --profile=emby-fallback +# └─ Critical-Data rsync → play_state_sync → partnership --check # # 0 */4 * * * every 4 hours: # Orchestrators/intermediate_sync_maintenance.sh diff --git a/System_Essentials/conf_cache_restore.sh b/System_Essentials/conf_cache_restore.sh index 1a52010..11e904c 100755 --- a/System_Essentials/conf_cache_restore.sh +++ b/System_Essentials/conf_cache_restore.sh @@ -20,6 +20,30 @@ # RAM → backup removed → fallback.sh runs with last-known-good partner vars # # Own conf is never in the backup — it's always on disk. +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Remove After Use +# The persistent backup is always removed at the end of the run — whether it +# was used or not. Stale backups from a previous shutdown should never be left +# behind as a permanent fallback; the backup is a single-boot safety net, not +# a long-lived cache. +# +# Partner Only +# Own conf is on disk and is always available regardless of array or partner +# state. Only partner confs can be missing after a boot — only those are +# restored. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# require_partnership — exits early if PARTNERSHIP_ENABLED=false +# detect_hosts() — determines which conf files belong to partners vs self +# No-backup guard — exits cleanly if PERSISTENT_CONF_CACHE doesn't exist +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/System_Essentials/conf_cache_save.sh b/System_Essentials/conf_cache_save.sh index 8199650..bc18c4f 100755 --- a/System_Essentials/conf_cache_save.sh +++ b/System_Essentials/conf_cache_save.sh @@ -16,6 +16,28 @@ # # Only partner confs are saved — own conf is always on disk. # Path adapts to storage mode: $SCRIPTS_DIR/.cache/vv/d (internal or appdata). +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Save at Shutdown, Not at Boot +# The backup is created at array stop while the RAM cache is at its freshest. +# By the next boot the partner may be unreachable — but the conf is already +# preserved and available the moment conf_cache_restore.sh runs. +# +# Partner Only +# Own conf is on disk — always present, never needs saving. Only partner confs +# live in RAM and can be missing at the next boot. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# require_partnership — exits early if PARTNERSHIP_ENABLED=false +# detect_hosts() — determines which confs to save (partner confs only) +# No-cache guard — exits cleanly if RAM cache is empty or missing +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/System_Essentials/conf_sync.sh b/System_Essentials/conf_sync.sh index e2a3085..d94ebc5 100755 --- a/System_Essentials/conf_sync.sh +++ b/System_Essentials/conf_sync.sh @@ -25,6 +25,29 @@ # works whether the remote is in internal or appdata storage mode. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# RAM-Only Credentials +# Partner credentials and keys never touch disk on the receiving server. +# /tmp/.cache/vv/d is tmpfs — cleared every reboot. This is intentional: +# partner conf files are not discoverable on disk between boots. +# +# Pull Resolves Remote Path +# The pull step reads the remote's varaverk.cfg to discover their SCRIPTS_DIR +# before SSHing for the conf. Works whether the remote is in internal or +# appdata storage mode — no hardcoded path assumptions about the remote. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# require_partnership — exits early if PARTNERSHIP_ENABLED=false +# detect_hosts() — partner list for push/pull routing +# acquire_lock — prevents concurrent sync runs +# SSH reachability — partners that fail SSH are skipped, not fatal +# +# ============================================================================================== # RUNTIME MODES # ============================================================================================== # diff --git a/Tools/arr_profile_enforcer.sh b/Tools/arr_profile_enforcer.sh index 388b706..cf4f230 100755 --- a/Tools/arr_profile_enforcer.sh +++ b/Tools/arr_profile_enforcer.sh @@ -17,14 +17,39 @@ # Profile names are looked up by name from the API at runtime, so profile IDs # do not need to be hardcoded and work across hosts. # -# ── USAGE ──────────────────────────────────────────────────────────────────── -# arr_profile_enforcer.sh [--dry-run] [--sonarr-only] [--radarr-only] +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== # -# ── CONFIGURATION ──────────────────────────────────────────────────────────── -# master.conf -# ARR_KIDS_PROFILE_NAME — profile name for kids/anime (default: "Kids shows") -# ARR_SONARR_DEFAULT_PROFILE — default Sonarr profile name (default: "Any") -# ARR_RADARR_DEFAULT_PROFILE — default Radarr profile name (default: "Any (mine)") +# Idempotent — only touches items whose current profile is wrong +# API-only — no file system changes; profile ID looked up by name at runtime +# --dry-run mode — reports what would change without applying any updates +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# ARR_KIDS_PROFILE_NAME — profile name for kids/anime (default: "Kids shows") +# ARR_SONARR_DEFAULT_PROFILE — default Sonarr profile name (default: "Any") +# ARR_RADARR_DEFAULT_PROFILE — default Radarr profile name (default: "Any (mine)") +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# arr_profile_enforcer.sh +# Enforce profiles in both Sonarr and Radarr. +# +# arr_profile_enforcer.sh --dry-run +# Show which items would be updated without making changes. +# +# arr_profile_enforcer.sh --sonarr-only +# Run only Sonarr enforcement. +# +# arr_profile_enforcer.sh --radarr-only +# Run only Radarr enforcement. # # ============================================================================================== diff --git a/Tools/claude_startup.sh b/Tools/claude_startup.sh index 17c1d51..9786bcd 100755 --- a/Tools/claude_startup.sh +++ b/Tools/claude_startup.sh @@ -28,6 +28,32 @@ # terminal or from array_started.sh. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Mode-Aware Paths +# Storage mode is read from host*.conf before any symlink is created. Internal +# mode (boot) requires no array — symlinks resolve immediately. Appdata mode +# requires the array to be mounted before symlinks are useful. +# +# Migrate on First Run +# If live data already exists at /root/.claude and the persistent store is +# empty, the live data is moved to persistent storage on first run. Subsequent +# runs only re-create the symlinks — migration is one-time. +# +# No common.sh Dependency +# Runs before load_config.sh is available (early in ARRAY_START_SCRIPTS). +# All logic is self-contained — no ecosystem functions used. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Conf probe — reads storage mode from Configurations/host*.conf directly +# Migration guard — only migrates if persistent store is empty; never overwrites +# Symlink-safe — removes existing symlink before re-creating; won't error on re-run +# +# ============================================================================================== # RUNTIME MODES # ============================================================================================== # diff --git a/Tools/docker_prune_images.sh b/Tools/docker_prune_images.sh index 8865572..5374f4d 100755 --- a/Tools/docker_prune_images.sh +++ b/Tools/docker_prune_images.sh @@ -20,6 +20,25 @@ # know all stopped containers are safe to delete. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Safe Default, Explicit Escalation +# The default mode (dangling only) is always safe — running containers are +# never affected. The --all mode requires deliberate opt-in and carries an +# explicit caution in the description, because it removes stopped containers +# that may be intentionally paused. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root check — Docker prune operations require root +# acquire_lock — prevents concurrent prune runs +# --dry-run — shows what would be removed without taking any action +# --status — lists current dangling images and stopped containers; no changes +# +# ============================================================================================== # RUNTIME MODES # ============================================================================================== # diff --git a/Tools/webhook_setup.sh b/Tools/webhook_setup.sh index 41b71de..79543fe 100755 --- a/Tools/webhook_setup.sh +++ b/Tools/webhook_setup.sh @@ -12,15 +12,47 @@ # The listener (start_webhook_listener.sh) must be running before arrs will # actually deliver events, but this script can register the connection first. # -# ── WHAT IT DOES ───────────────────────────────────────────────────────────── -# 1. Generates WEBHOOK_SECRET in master.conf if empty -# 2. Registers webhook in each local arr (Sonarr / Radarr / Lidarr) -# 3. SSHes to remote host and runs itself there (unless --local-only) +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── USAGE ──────────────────────────────────────────────────────────────────── -# webhook_setup.sh — configure local + remote -# webhook_setup.sh --local-only — local arrs only (used internally for SSH) -# webhook_setup.sh --dry-run — show what would be registered +# 1. Generate WEBHOOK_SECRET in master.conf if empty +# 2. Register webhook in each local arr (Sonarr / Radarr / Lidarr) +# 3. SSH to remote host and run itself there (unless --local-only) +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Idempotent Registration +# Skips any arr that already has the webhook registered. Safe to re-run after +# adding a new arr or after a conf change without creating duplicate entries. +# +# Self-Propagating +# SSHes to the remote and runs itself with --local-only — one execution +# configures both servers without a separate remote step. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Idempotent — skips arrs that already have the webhook registered +# Secret auto-gen — WEBHOOK_SECRET generated if empty; never left blank +# --local-only — used internally for SSH; prevents infinite recursion +# --dry-run mode — shows what would be registered without making API calls +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# webhook_setup.sh +# Configure local arrs, then SSH to remote and configure remote arrs. +# +# webhook_setup.sh --local-only +# Local arrs only (used internally when called via SSH on the remote). +# +# webhook_setup.sh --dry-run +# Show what would be registered without making any changes. # # ============================================================================================== diff --git a/Watchdogs/System/conf_cache_watchdog.sh b/Watchdogs/System/conf_cache_watchdog.sh index ca2f1b5..50bb1e0 100755 --- a/Watchdogs/System/conf_cache_watchdog.sh +++ b/Watchdogs/System/conf_cache_watchdog.sh @@ -19,6 +19,31 @@ # # Silent when remote is online and no backup exists (normal state). # No-op when FALLBACK_ENABLED=false or CONF_SYNC_ENABLED=false. +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Online = No Backup Needed +# When the remote is reachable, conf_sync.sh will pull fresh on the next boot. +# The persistent backup is removed — a stale backup is worse than no backup +# because it can mask a connectivity problem that conf_sync.sh would catch. +# +# Offline = Stay Ready +# While the remote is down, the RAM cache is the best available copy of partner +# vars. Refreshing the persistent backup every 15 minutes ensures it reflects +# the last-known-good state, not an old copy from days earlier. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# require_partnership — exits early if PARTNERSHIP_ENABLED=false +# FALLBACK_ENABLED gate — exits if fallback is disabled +# CONF_SYNC_ENABLED gate — exits if conf sync is disabled +# REMOTE_ID presence check — exits if partner identity is unset +# --dry-run mode — shows what would happen without touching the backup +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Watchdogs/system_watchdog.sh b/Watchdogs/system_watchdog.sh index f559fb8..9b50025 100755 --- a/Watchdogs/system_watchdog.sh +++ b/Watchdogs/system_watchdog.sh @@ -2,32 +2,68 @@ # ============================================================================================== # ================================= System Watchdog ============================================ # ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── # Thin orchestrator — runs SYSTEM_WATCHDOG_SCRIPTS from master.conf sequentially. # Called by watchdog_orchestrator.sh each cycle. Covers system component health: # storage pool growth, runaway logs, WebGUI availability, and network connectivity. # -# ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── -# Driven by SYSTEM_WATCHDOG_SCRIPTS in master.conf — add, remove, or reorder there. -# Default: storage_watchdog → webgui_watchdog → network_watchdog +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Driven by SYSTEM_WATCHDOG_SCRIPTS in master.conf — add, remove, or reorder there. +# Default: storage_watchdog → webgui_watchdog → network_watchdog # -# ── SEQUENTIAL EXECUTION ───────────────────────────────────────────────────────────────────── # All scripts run in the foreground. Each must complete before the next starts. # A failed script is logged but does not prevent remaining scripts from running. # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# Root check — child scripts require root -# acquire_lock — prevents concurrent system watchdog runs -# detect_hosts() — MY_ID in notifications and logs -# Non-fatal steps — a failed step is logged; remaining steps still run +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Configuration Owns the List +# SYSTEM_WATCHDOG_SCRIPTS in master.conf is the only place scripts are added +# or removed. This orchestrator never needs to be edited to change what runs. +# +# Non-Fatal Steps +# A failed watchdog step is logged and noted in the summary, but the remaining +# steps still execute. Partial coverage is better than a halted watchdog chain. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root check — child scripts require root +# acquire_lock — prevents concurrent system watchdog runs +# detect_hosts() — MY_ID in notifications and logs +# Non-fatal steps — a failed step is logged; remaining steps still run +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── # SYSTEM_WATCHDOG_SCRIPTS — ordered list of system component watchdog scripts to run # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# system_watchdog.sh — run all system component watchdogs -# system_watchdog.sh --dry-run — preview without running anything -# system_watchdog.sh --status — show configured scripts and exit -# system_watchdog.sh --log — verbose output +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# system_watchdog.sh +# Run all system component watchdogs. +# +# system_watchdog.sh --dry-run +# Preview without running anything. +# +# system_watchdog.sh --status +# Show configured scripts and exit. +# +# system_watchdog.sh --log +# Verbose output. +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/git_pull_execute.sh b/git_pull_execute.sh index a03f68d..fe0d4ff 100755 --- a/git_pull_execute.sh +++ b/git_pull_execute.sh @@ -2,35 +2,63 @@ # ============================================================================================== # ================================= Git Pull & Execute ========================================= # ============================================================================================== -# Pulls the latest scripts from the Gitea repository via SSH. -# Lives at the repo root — sources load_config.sh from the same directory. # -# ── WHAT THIS SCRIPT DOES ───────────────────────────────────────────────────────────────────── -# 1. Detects which server it's running on via detect_hosts() (MY_ID) -# 2. Configures sparse checkout to exclude other servers' credential files -# Each server only pulls its own host*.conf — never sees peer credentials -# 3. Pulls or clones latest scripts from Gitea -# 4. Sets executable permissions on all .sh files +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Pulls the latest scripts from the Gitea repository via SSH. Lives at the repo +# root — sources load_config.sh from the same directory. # -# ── SPARSE CHECKOUT ─────────────────────────────────────────────────────────────────────────── -# Sparse checkout ensures each server only receives its own host conf: -# HOST1 pulls: master.conf + host1.conf + all scripts -# HOST1 skips: host2.conf, host3.conf etc. -# HOST2 pulls: master.conf + host2.conf + all scripts -# HOST2 skips: host1.conf, host3.conf etc. +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# Adding a new server: -# Create host3.conf in the repo -# All existing servers automatically exclude it on next pull -# New server gets only its own conf ✅ +# 1. detect_hosts() — identifies which server is running the pull (MY_ID) +# 2. Configure sparse checkout — exclude other servers' credential files +# 3. Pull or clone latest scripts from Gitea +# 4. Set executable permissions on all .sh files # -# ── GITEA LOCATION DETECTION ────────────────────────────────────────────────────────────────── -# Detects where Gitea is running at runtime — works through fallback: -# Gitea local → connects via local IP -# Gitea remote → connects via Tailscale IP -# Both fail → falls back to GITEA_DOMAIN if configured +# SPARSE CHECKOUT +# Each server only receives its own host conf — never peer credentials: +# HOST1 pulls: master.conf + host1.conf + all scripts +# HOST1 skips: host2.conf, host3.conf etc. +# Adding a new server: create host3.conf in the repo — all existing servers +# automatically exclude it on next pull; new server gets only its own conf. +# +# GITEA LOCATION DETECTION +# Detected at runtime — works through fallback: +# Gitea local → connects via local IP +# Gitea remote → connects via Tailscale IP +# Both fail → falls back to GITEA_DOMAIN if configured +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Credentials Stay Separated +# Sparse checkout is configured per-server on every pull, not just at clone +# time. Ensures newly added host confs are automatically excluded on all +# existing servers without any manual intervention. +# +# Runtime Location Detection +# Gitea's IP is never hardcoded — the script probes whether Gitea is local +# or remote on every run. Handles Gitea container restarts, migrations, and +# Tailscale address changes automatically. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root check — git operations on /boot require root +# acquire_lock — prevents concurrent pulls +# detect_hosts() — MY_ID required to build correct sparse checkout rules +# Docker check — Gitea container status probed before any SSH attempt +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── # GITEA_CONTAINER — Docker container name for Gitea # GITEA_REPO_PATH — repo path on Gitea (e.g. Varaverk/varaverk.git) # GITEA_DOMAIN — public domain fallback (optional) @@ -38,11 +66,22 @@ # GITEA_SSH_KEY — SSH key for Gitea authentication # SSH_PORT — Gitea SSH port (often 221 or 222) # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# git_pull_execute.sh — normal pull -# git_pull_execute.sh --dry-run — preview without making changes -# git_pull_execute.sh --log — verbose output -# git_pull_execute.sh --status — show config and exit +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# git_pull_execute.sh +# Normal pull. +# +# git_pull_execute.sh --dry-run +# Preview without making changes. +# +# git_pull_execute.sh --log +# Verbose output. +# +# git_pull_execute.sh --status +# Show config and exit. +# # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/load_config.sh b/load_config.sh index a0e0446..985f46d 100755 --- a/load_config.sh +++ b/load_config.sh @@ -2,51 +2,48 @@ # ============================================================================================== # ================================= CONFIGURATION LOADER ======================================= # ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── # Single entry point for all configuration sourcing across the ecosystem. -# Every script sources this file instead of sourcing master.conf files directly. +# Every script sources this file instead of sourcing conf files directly. # -# ── HOW IT WORKS ────────────────────────────────────────────────────────────────────────────── -# 1. Detects OS platform → PLATFORM=unraid|truenas|unknown; exports SCRIPTS_DIR -# 2. Sources master.conf (shared config — hostnames, thresholds, toggles, profiles, job lists) -# 3. Auto-discovers and sources all host*.conf files in the same directory -# Each host conf extends the shared profile arrays and adds host-specific credentials -# 4. Sources common.sh (shared functions — detect_hosts, logging, notifications etc.) -# 5. Sources Plugin//adapter.sh (platform_*() functions for OS-specific ops) +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── WHY THIS EXISTS ─────────────────────────────────────────────────────────────────────────── -# Without this loader every script had to explicitly source each conf file: -# source master.conf -# source host1.conf -# source host2.conf -# source common.sh +# Sourcing order (order is load-bearing): +# 1. Detects OS platform → PLATFORM=unraid|truenas|unknown; exports SCRIPTS_DIR +# 2. Sources master.conf — shared config: hostnames, thresholds, toggles, profiles, job lists +# 3. Auto-discovers and sources all host*.conf files present in Configurations/ +# Each host conf extends the shared profile arrays and adds host-specific credentials +# 4. Sources common.sh — shared functions: detect_hosts, logging, notifications, etc. +# 5. Sources Plugin//adapter.sh — platform_*() functions for OS-specific ops # -# Adding a new server meant updating every script. -# With this loader — add host3.conf to the git repo and every server -# auto-discovers it on next git pull. Zero script changes required. Ever. +# USAGE IN SCRIPTS +# Scripts in subdirectories (Rsync/, Docker_Essentials/ etc.): +# SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# source "$SCRIPT_DIR/../load_config.sh" # -# ── ADDING A NEW SERVER ─────────────────────────────────────────────────────────────────────── -# 1. Create Configurations/host3.conf following the same structure as HOST1/HOST2 -# 2. Commit and push to git repo -# 3. All servers pull it automatically — no other changes needed +# Scripts in the repo root: +# SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# source "$SCRIPT_DIR/load_config.sh" # -# ── USAGE IN SCRIPTS ────────────────────────────────────────────────────────────────────────── -# Replace the three source lines at the top of every script with: +# SPARSE CHECKOUT NOTE +# Sparse checkout controls which host*.conf files each server receives. +# HOST1 only pulls Configurations/host1.conf — never HOST2's credentials. +# HOST2 only pulls Configurations/host2.conf — never HOST1's credentials. +# This loader sources whatever conf files ARE present — sparse checkout handles the rest. # -# SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# source "$SCRIPT_DIR/../load_config.sh" +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== # -# Scripts in subdirectories (Rsync/, Docker_Essentials/ etc.) use ../ to reach root. -# Scripts in root directory use ./ instead: -# -# SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# source "$SCRIPT_DIR/load_config.sh" -# -# ── SPARSE CHECKOUT NOTE ────────────────────────────────────────────────────────────────────── -# Sparse checkout controls which host*.conf files each server receives. -# HOST1 only pulls Configurations/host1.conf — never HOST2's credentials. -# HOST2 only pulls Configurations/host2.conf — never HOST1's credentials. -# This loader sources whatever conf files ARE present — sparse checkout handles the rest. -# Both servers pull all non-credential conf files (Configurations/master.conf, common.sh, load_config.sh). +# Zero-Script Expansion +# Without this loader, adding a new server required updating every script to +# source the new host conf. With this loader: create host3.conf in the repo, +# commit and push — all servers auto-discover it on next git pull. No script +# changes required. # # ==============================================================================================