diff --git a/Docker_Essentials/Manual-Docker_Essentials.md b/Docker_Essentials/Manual-Docker_Essentials.md index ed3aad8..1e11232 100644 --- a/Docker_Essentials/Manual-Docker_Essentials.md +++ b/Docker_Essentials/Manual-Docker_Essentials.md @@ -10,7 +10,7 @@ For per-script detail see the script headers directly. ## ━━━ WATCHDOG CONFIGURATION ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -All watchdog configuration lives in `master_host*.conf` (per-container lists) and +All watchdog configuration lives in `host*.conf` (per-container lists) and `master.conf` (shared thresholds and toggles). `detect_hosts()` aliases all `HOST1_` / `HOST2_` prefixed vars to their unprefixed names at runtime — scripts always read the right values for the server they're running on. @@ -20,7 +20,7 @@ always read the right values for the server they're running on. ### ── Memory Hard Limits ──────────────────────────────────────────────────────── ```bash -# master_host1.conf +# host1.conf # Format: "ContainerName:LimitInMB" # Immediate restart when exceeded — no strike system. Memory leaks are not spikes. # @@ -71,7 +71,7 @@ CPU_FAIL_LIMIT=2 # consecutive strikes before restart ### ── HTTP Health Checks ──────────────────────────────────────────────────────── ```bash -# master_host1.conf +# host1.conf # Format: "ContainerName:http://host:port/optional-path" # Hits the actual service endpoint on every watchdog cycle. # "Container running" and "service responding" are not the same thing. @@ -98,7 +98,7 @@ RESP_FAIL_LIMIT=2 # consecutive failures before restart ### ── Required Containers ─────────────────────────────────────────────────────── ```bash -# master_host1.conf +# host1.conf # Containers that must always be running. # Found stopped → watchdog attempts restart every cycle until running or skip-listed. # Uses the STRIKE SYSTEM — one miss might be mid-restart. @@ -119,7 +119,7 @@ HOST1_WATCHDOG_REQUIRED_CONTAINERS=( ### ── Dependency Ordering ─────────────────────────────────────────────────────── ```bash -# master_host1.conf +# host1.conf # Format: "DependentContainer:dependency1 dependency2" # Multiple dependencies space-separated. All must be running before dependent restarts. # @@ -245,7 +245,7 @@ WATCHDOG_BATCH_NOTIFY=true ### ── Daily Restart List ─────────────────────────────────────────────────────── ```bash -# master_host1.conf +# host1.conf # Restarted every night at 1am via daily_sync_maintenance.sh. # # Good candidates: @@ -271,7 +271,7 @@ their images updated daily before the restart. Add a container once, it gets bot ### ── Weekly Restart List ────────────────────────────────────────────────────── ```bash -# master_host1.conf +# host1.conf # Restarted every Sunday at 2:30am via weekly_sync_maintenance.sh. # Runs AFTER the sync window's own restart of critical containers (Emby, auth stack). # @@ -291,7 +291,7 @@ HOST1_WEEKLY_RESTART_CONTAINERS=( ## ━━━ NETWORK CONFIGURATION ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ```bash -# master_host1.conf +# host1.conf # Networks to ensure exist + containers to connect to each network. # Many-to-many: every container connects to every network listed. # @@ -339,7 +339,7 @@ Everything gets updated at least once per week with no explicit configuration. ## ━━━ FULL CONFIGURATION REFERENCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -### ── master_host*.conf ──────────────────────────────────────────────────────── +### ── host*.conf ──────────────────────────────────────────────────────── ```bash # Per-host — varies between HOST1 and HOST2 @@ -425,11 +425,11 @@ SLEEP=5 # seconds between retry attempts ### ── Adding a Container to Monitoring ──────────────────────────────────────── -Adding a container is purely additive — add the relevant lines to `master_host*.conf`. +Adding a container is purely additive — add the relevant lines to `host*.conf`. No script changes. `detect_hosts()` picks up the new config on the next watchdog cycle. ```bash -# master_host1.conf — example: adding "MyApp" to full Tier 1 + daily restarts +# host1.conf — example: adding "MyApp" to full Tier 1 + daily restarts # 1. Memory hard limit — size at ~150-200% of normal peak (check: docker stats MyApp) HOST1_WATCHDOG_CONTAINERS=( diff --git a/Docker_Essentials/README-Docker_Essentials.md b/Docker_Essentials/README-Docker_Essentials.md index 2abee83..2acb058 100644 --- a/Docker_Essentials/README-Docker_Essentials.md +++ b/Docker_Essentials/README-Docker_Essentials.md @@ -275,7 +275,7 @@ Array starts reads system_watchdog state (RAM emergency deferral) │ │ (on skip list event → operator uses) - └── Tools/watchdog_skip_list_manager.sh + └── Watchdogs/watchdog_skip_list_manager.sh inspect state, clear after fixing root cause Daily maintenance window (1am): diff --git a/Docker_Essentials/docker_daily_restart.sh b/Docker_Essentials/docker_daily_restart.sh index 04fb85c..c133a73 100644 --- a/Docker_Essentials/docker_daily_restart.sh +++ b/Docker_Essentials/docker_daily_restart.sh @@ -60,7 +60,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_DAILY_RESTART_CONTAINERS # Containers restarted nightly. Also used by docker_update.sh normal mode @@ -126,7 +126,7 @@ detect_hosts if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to restart" - warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in master_host*.conf" + warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in host*.conf" exit 0 fi diff --git a/Docker_Essentials/docker_network_connect.sh b/Docker_Essentials/docker_network_connect.sh index 7736179..0e27bb6 100755 --- a/Docker_Essentials/docker_network_connect.sh +++ b/Docker_Essentials/docker_network_connect.sh @@ -78,7 +78,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_NETWORK_CONNECT_NETWORKS # Networks to ensure exist at array start. Aliased by detect_hosts() → @@ -144,13 +144,13 @@ fi # Empty array guards if [[ ${#NETWORK_CONNECT_NETWORKS[@]} -eq 0 ]]; then warn "NETWORK_CONNECT_NETWORKS is empty for $MY_ID — nothing to do" - warn "Check HOST*_NETWORK_CONNECT_NETWORKS in master_host*.conf" + warn "Check HOST*_NETWORK_CONNECT_NETWORKS in host*.conf" exit 0 fi if [[ ${#NETWORK_CONNECT_CONTAINERS[@]} -eq 0 ]]; then warn "NETWORK_CONNECT_CONTAINERS is empty for $MY_ID — no containers to connect" - warn "Check HOST*_NETWORK_CONNECT_CONTAINERS in master_host*.conf" + warn "Check HOST*_NETWORK_CONNECT_CONTAINERS in host*.conf" exit 0 fi diff --git a/Docker_Essentials/docker_update.sh b/Docker_Essentials/docker_update.sh index 6155681..8c6834a 100644 --- a/Docker_Essentials/docker_update.sh +++ b/Docker_Essentials/docker_update.sh @@ -95,7 +95,7 @@ # Container names for emby and critical-data profiles — excluded from # remainder mode (already updated by the weekly sync window) # -# master_host*.conf +# host*.conf # # HOST*_DAILY_RESTART_CONTAINERS # Containers updated in normal mode. Aliased by detect_hosts() → @@ -205,7 +205,7 @@ else if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to update" - warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in master_host*.conf" + warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in host*.conf" exit 0 fi diff --git a/Docker_Essentials/docker_update_remaining.sh b/Docker_Essentials/docker_update_remaining.sh index 2a90a77..2dcf43f 100644 --- a/Docker_Essentials/docker_update_remaining.sh +++ b/Docker_Essentials/docker_update_remaining.sh @@ -52,7 +52,7 @@ # Enable or disable this script. (default: true) # To disable without the toggle: remove from WEEKLY_MAINTENANCE_SCRIPTS. # -# master_host*.conf +# host*.conf # # HOST*_DAILY_RESTART_CONTAINERS # Excluded from this script — already updated daily. Aliased by diff --git a/Docker_Essentials/docker_weekly_restart.sh b/Docker_Essentials/docker_weekly_restart.sh index 8ce0c8f..4666d89 100644 --- a/Docker_Essentials/docker_weekly_restart.sh +++ b/Docker_Essentials/docker_weekly_restart.sh @@ -43,7 +43,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_WEEKLY_RESTART_CONTAINERS # Containers restarted weekly. Aliased by detect_hosts() → @@ -108,7 +108,7 @@ detect_hosts if [[ ${#WEEKLY_RESTART_CONTAINERS[@]} -eq 0 ]]; then warn "WEEKLY_RESTART_CONTAINERS is empty for $MY_ID — nothing to restart" - warn "Check HOST*_WEEKLY_RESTART_CONTAINERS in master_host*.conf" + warn "Check HOST*_WEEKLY_RESTART_CONTAINERS in host*.conf" exit 0 fi diff --git a/Docker_Essentials/downloaders_reset.sh b/Docker_Essentials/downloaders_reset.sh index 39f4bac..d832b71 100644 --- a/Docker_Essentials/downloaders_reset.sh +++ b/Docker_Essentials/downloaders_reset.sh @@ -83,7 +83,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_SLSKD_URL / HOST*_SLSKD_API_KEY / HOST*_SLSKD_FAILED_IMPORTS_DIR # slskd connection and failed imports path. Aliased by detect_hosts() @@ -567,7 +567,7 @@ if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then if [[ -z "$QBIT_COOKIE" ]]; then error "Failed to authenticate with qBittorrent — check QBIT_USERNAME/PASSWORD" - notify "qBittorrent auth failed on $(hostname) — check credentials in master_host*.conf" "Downloaders Reset" "warning" + notify "qBittorrent auth failed on $(hostname) — check credentials in host*.conf" "Downloaders Reset" "warning" ((TOTAL_FAIL++)) else TORRENTS=$(curl -sf --max-time 15 \ diff --git a/Fallback/Manual-Fallback.md b/Fallback/Manual-Fallback.md index 144bf68..14f58a0 100644 --- a/Fallback/Manual-Fallback.md +++ b/Fallback/Manual-Fallback.md @@ -80,7 +80,7 @@ container start time. At 3 strikes × 30s + ~2min rsync + ~1min container start: --- -## ━━━ CONFIGURATION — master_host*.conf ━━━ +## ━━━ CONFIGURATION — host*.conf ━━━ ``` HOST*_DDNS_CONTAINERS=("...") @@ -181,7 +181,7 @@ FALLBACK_TEST_HANDBACK_WAIT=300 --- -### master_host2.conf — HOST2 covering HOST1 +### host2.conf — HOST2 covering HOST1 ```bash HOST2_DDNS_CONTAINERS=("Gmer4Lfe.us-DDNS") @@ -259,7 +259,7 @@ FALLBACK_HOST1_WRITEBACK_TIER3=( --- -### master_host1.conf — HOST1 covering HOST2 +### host1.conf — HOST1 covering HOST2 ```bash HOST1_DDNS_CONTAINERS=("Gmer4Lfe.com-DDNS") @@ -486,7 +486,7 @@ Restart fallback.sh via User Scripts plugin. It will resume from NORMAL on its n 1. Create the container on the covering server (stopped), with volume mounts pointing at the mirrored share path (e.g. `/mnt/user/Movies` must exist on the covering server) -2. Add the container name to `FALLBACK_HOST*_COVERS_HOST*_TIER*` in master_host*.conf +2. Add the container name to `FALLBACK_HOST*_COVERS_HOST*_TIER*` in host*.conf in the appropriate tier position (dependency ordering — databases before apps) 3. Verify: `fallback.sh --status` shows the container in the expected tier list 4. Run `fallback_test.sh --dry-run` to confirm the full configuration is valid diff --git a/Fallback/README-Fallback.md b/Fallback/README-Fallback.md index 2dec0c2..36cb19a 100644 --- a/Fallback/README-Fallback.md +++ b/Fallback/README-Fallback.md @@ -219,8 +219,8 @@ determines which server is local and which is remote at runtime, then selects th container arrays and tier delays from config via MY_ID. ``` -HOST2 covers HOST1: FALLBACK_HOST2_COVERS_HOST1_TIER* (in master_host2.conf) -HOST1 covers HOST2: FALLBACK_HOST1_COVERS_HOST2_TIER* (in master_host1.conf) +HOST2 covers HOST1: FALLBACK_HOST2_COVERS_HOST1_TIER* (in host2.conf) +HOST1 covers HOST2: FALLBACK_HOST1_COVERS_HOST2_TIER* (in host1.conf) ``` Both servers run identical scripts. MY_ID selects the correct arrays. No hostname diff --git a/Fallback/fallback.sh b/Fallback/fallback.sh index 407c0f6..c936b38 100755 --- a/Fallback/fallback.sh +++ b/Fallback/fallback.sh @@ -132,7 +132,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_DDNS_CONTAINERS # DDNS containers this host manages — stopped on internet loss, started diff --git a/Fallback/fallback_test.sh b/Fallback/fallback_test.sh index 77a8d07..ba8678b 100644 --- a/Fallback/fallback_test.sh +++ b/Fallback/fallback_test.sh @@ -289,7 +289,7 @@ fi # Tier 1 containers configured if [[ ${#TIER1_CONTAINERS[@]} -eq 0 ]]; then error "No Tier 1 containers configured for $MY_ID → $REMOTE_ID" - error "Check FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER1 in master_host*.conf" + error "Check FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER1 in host*.conf" phase_fail "Pre-flight" exit 1 fi diff --git a/Media/Manual-Media.md b/Media/Manual-Media.md index 5b398b2..351fbb2 100644 --- a/Media/Manual-Media.md +++ b/Media/Manual-Media.md @@ -228,7 +228,7 @@ LIDARR_DISCOVERY_REJECT_COOLDOWN=30 # days before re-evaluating a Stage 2 reje LIDARR_DISCOVERY_HISTORY="$DATA_DIR/lidarr_discovery_history.db" ``` -Requires `HOST*_LASTFM_API_KEY` in `master_host*.conf`. +Requires `HOST*_LASTFM_API_KEY` in `host*.conf`. --- @@ -246,7 +246,7 @@ RADARR_DISCOVERY_SEED_LIBRARIES=("Movies") # Emby libraries to draw seeds from RADARR_DISCOVERY_HISTORY="$DATA_DIR/radarr_discovery_history.db" ``` -Requires `HOST*_TMDB_API_KEY` in `master_host*.conf`. +Requires `HOST*_TMDB_API_KEY` in `host*.conf`. --- @@ -266,7 +266,7 @@ SONARR_DISCOVERY_HISTORY="$DATA_DIR/sonarr_discovery_history.db" # SONARR_EMBY_LIBRARIES is shared with emby_to_sonarr_sync — see Orchestrator Job Order ``` -Requires `HOST*_TMDB_API_KEY` in `master_host*.conf`. +Requires `HOST*_TMDB_API_KEY` in `host*.conf`. > **MONITOR_MODE note:** Use `"all"` (default) to have Sonarr search all existing seasons > after adding a show. `"future"` only marks upcoming seasons as monitored — shows where @@ -289,9 +289,9 @@ MEDIA_MAINTENANCE_JOBS=( --- -## ━━━ CONFIGURATION — master_host*.conf ━━━ +## ━━━ CONFIGURATION — host*.conf ━━━ -### master_host1.conf +### host1.conf ```bash # Shares this server applies permissions to @@ -429,7 +429,7 @@ cp radarr_cleanup.sh readarr_cleanup.sh # 2. Replace RADARR_ prefix with READARR_ throughout # Update API endpoint, tracked file API path, extension list, protected patterns -# 3. Add to master_host*.conf +# 3. Add to host*.conf HOST1_READARR_URL="http://192.168.50.2:8787" HOST1_READARR_API_KEY="your-api-key" HOST1_READARR_BOOKS_ROOT="/mnt/user/Books" @@ -534,7 +534,7 @@ If ghosts persist: 1. Is Emby's API responding? curl -s "http://[emby-ip]:8096/System/Info/Public" - 2. Is EMBY_URL / EMBY_API_KEY correct in master_host*.conf? + 2. Is EMBY_URL / EMBY_API_KEY correct in host*.conf? Run: sonarr_cleanup.sh --status (shows Emby config) 3. Trigger manually in Emby: diff --git a/Media/arr_sync.sh b/Media/arr_sync.sh index 1e9f8b8..3bb8f90 100755 --- a/Media/arr_sync.sh +++ b/Media/arr_sync.sh @@ -78,7 +78,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_LIDARR_URL / HOST*_LIDARR_API_KEY — local Lidarr (aliased by detect_hosts) # HOST*_SONARR_URL / HOST*_SONARR_API_KEY — local Sonarr diff --git a/Media/arrs_failed_stalled_recovery.sh b/Media/arrs_failed_stalled_recovery.sh index 2791e6b..c86a598 100644 --- a/Media/arrs_failed_stalled_recovery.sh +++ b/Media/arrs_failed_stalled_recovery.sh @@ -54,7 +54,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_SONARR_URL / HOST*_SONARR_API_KEY / HOST*_SONARR_RECOVERY # HOST*_RADARR_URL / HOST*_RADARR_API_KEY / HOST*_RADARR_RECOVERY diff --git a/Media/lidarr_cleanup.sh b/Media/lidarr_cleanup.sh index 3bd1b2b..4fde635 100644 --- a/Media/lidarr_cleanup.sh +++ b/Media/lidarr_cleanup.sh @@ -58,7 +58,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST1_LIDARR_URL / HOST1_LIDARR_API_KEY / HOST1_LIDARR_MUSIC_ROOT # HOST1_LIDARR_PATH_MAP — container path → host path translation diff --git a/Media/lidarr_missing_art.sh b/Media/lidarr_missing_art.sh index 4578027..2f6fbde 100644 --- a/Media/lidarr_missing_art.sh +++ b/Media/lidarr_missing_art.sh @@ -42,7 +42,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST1_LIDARR_URL / HOST1_LIDARR_API_KEY # Aliased by detect_hosts() — script uses LIDARR_URL / LIDARR_API_KEY diff --git a/Media/media_cleaner.sh b/Media/media_cleaner.sh index 239b27a..87d7945 100644 --- a/Media/media_cleaner.sh +++ b/Media/media_cleaner.sh @@ -5,10 +5,10 @@ # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── -# Remove junk files from media shares using configurable file patterns. Two -# profiles — anime and media — each with their own folder list and patterns. -# Runs second in the daily maintenance window, after permissions and before -# arr cleanup, so orphan detection only encounters actual media files. +# Remove scene debris, tool artifacts, and unsafe files from media shares +# before the orphan scan — so arr cleanup only encounters actual media files. +# Runs second in the daily window, after permissions and before arr cleanup. +# Two profiles — anime and media — each with their own folder list and patterns. # # ============================================================================================== # OPERATIONAL MODEL @@ -18,15 +18,29 @@ # anime — cleans ANIME_CLEAN_FOLDERS using ANIME_FILE_PATTERNS # media — cleans MEDIA_CLEAN_FOLDERS using MEDIA_FILE_PATTERNS (adds *.iso *.lrc) # -# Removes junk left by download clients, scene releases, and tools: -# *.sfv *.md5 *.sha1 — checksum files — useless post-download -# *.nfo *.url *.lnk — scene info files — not needed in media library -# *.rar *.zip — archives — source files not needed after extraction -# *.sample* *.proof* — scene samples — never needed -# *sync-conflict* — Syncthing conflict files -# *.scr *.exe — executables — should never be in a media folder -# *.torrent — torrent files left by download clients -# *.log *.json — tool output files +# Scene debris — left by scene releases and download clients: +# *.sfv *.md5 *.sha1 — checksums — useless post-download +# *.nzb *.torrent — download files left by clients +# *.url *.lnk *.info *.diz — scene metadata +# *.sample* *.proof* — scene samples — never needed in library +# *sync-conflict* — Syncthing conflict copies +# *.rar *.zip *.7z *.ace — archives — source not needed after extraction +# *.r00-*.r09 *.srr — multi-part rar segments and repair files +# *.001 *.002 *.003 — split archive parts +# *.gz *.tar *.bz2 — linux archives +# +# Tool artifacts — incomplete or stale files from download clients: +# *.!ut *.!qB — uTorrent / qBittorrent incomplete markers +# *.crdownload *.opdownload — Chrome / Opera incomplete downloads +# *.part — partial download files +# +# Unsafe files — executables that should never appear in a media folder: +# *.exe *.scr *.com — Windows executables +# *.bat *.cmd *.vbs *.ps1 — Windows scripts +# *.msi *.dll *.sys — Windows system files +# *.sh — shell scripts in media folders = suspicious +# +# Media profile also removes: *.iso *.lrc # # ============================================================================================== # OPERATIONAL SAFEGUARDS @@ -37,13 +51,12 @@ # Empty array guards — warns and exits cleanly if no folders or patterns configured # Folder existence — skips missing folders with warning, continues others # validate_unraid_cmd — notify script validated before use -# Silent by default — only problems and removals produce output # # ============================================================================================== # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_ANIME_CLEAN_FOLDERS — folders cleaned by the anime profile on this host # HOST*_MEDIA_CLEAN_FOLDERS — folders cleaned by the media profile on this host @@ -128,7 +141,7 @@ esac # Empty array guards if [[ ${#CLEAN_FOLDERS[@]} -eq 0 ]]; then warn "No folders configured for profile '$PROFILE' on $MY_ID" - warn "Check HOST*_${PROFILE^^}_CLEAN_FOLDERS in master_host*.conf" + warn "Check HOST*_${PROFILE^^}_CLEAN_FOLDERS in host*.conf" exit 0 fi diff --git a/Media/media_shares_permissions.sh b/Media/media_shares_permissions.sh index 1792770..319b352 100644 --- a/Media/media_shares_permissions.sh +++ b/Media/media_shares_permissions.sh @@ -34,7 +34,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_MEDIA_PERMISSION_SHARES — shares this host applies permissions to # Aliased by detect_hosts() — script uses MEDIA_PERMISSION_SHARES @@ -83,7 +83,7 @@ detect_hosts # Empty array guard if [[ ${#MEDIA_PERMISSION_SHARES[@]} -eq 0 ]]; then warn "MEDIA_PERMISSION_SHARES is empty for $MY_ID — nothing to do" - warn "Check HOST*_MEDIA_PERMISSION_SHARES in master_host*.conf" + warn "Check HOST*_MEDIA_PERMISSION_SHARES in host*.conf" exit 0 fi diff --git a/Media/playback_aware_lidarr_discovery.sh b/Media/playback_aware_lidarr_discovery.sh index 33a5880..9c18854 100755 --- a/Media/playback_aware_lidarr_discovery.sh +++ b/Media/playback_aware_lidarr_discovery.sh @@ -57,7 +57,7 @@ # ============================================================================================== # # Last.fm API key — required for both stages -# Configure HOST*_LASTFM_API_KEY in master_host*.conf +# Configure HOST*_LASTFM_API_KEY in host*.conf # # ============================================================================================== # CONFIGURATION (master.conf) @@ -135,18 +135,18 @@ if ! command -v jq >/dev/null 2>&1; then fi if [[ -z "${EMBY_URL:-}" || -z "${EMBY_API_KEY:-}" ]]; then - error "EMBY_URL / EMBY_API_KEY not configured — check master_host*.conf" + error "EMBY_URL / EMBY_API_KEY not configured — check host*.conf" exit 1 fi if [[ -z "${LIDARR_URL:-}" || -z "${LIDARR_API_KEY:-}" ]]; then - error "LIDARR_URL / LIDARR_API_KEY not configured — check master_host*.conf" + error "LIDARR_URL / LIDARR_API_KEY not configured — check host*.conf" exit 1 fi if [[ -z "${LASTFM_API_KEY:-}" ]]; then error "LASTFM_API_KEY not configured — required for discovery" - error "Configure HOST*_LASTFM_API_KEY in master_host*.conf" + error "Configure HOST*_LASTFM_API_KEY in host*.conf" exit 1 fi diff --git a/Media/playback_aware_radarr_discovery.sh b/Media/playback_aware_radarr_discovery.sh index b998c72..3336c32 100755 --- a/Media/playback_aware_radarr_discovery.sh +++ b/Media/playback_aware_radarr_discovery.sh @@ -55,7 +55,7 @@ # ============================================================================================== # # TMDB API key — required for Stage 2 recommendations -# Configure HOST*_TMDB_API_KEY in master_host*.conf +# Configure HOST*_TMDB_API_KEY in host*.conf # Free key at: https://www.themoviedb.org/settings/api # # ============================================================================================== @@ -136,19 +136,19 @@ if ! command -v jq >/dev/null 2>&1; then fi if [[ -z "${EMBY_URL:-}" || -z "${EMBY_API_KEY:-}" ]]; then - error "EMBY_URL / EMBY_API_KEY not configured — check master_host*.conf" + error "EMBY_URL / EMBY_API_KEY not configured — check host*.conf" exit 1 fi if [[ -z "${RADARR_URL:-}" || -z "${RADARR_API_KEY:-}" ]]; then - error "RADARR_URL / RADARR_API_KEY not configured — check master_host*.conf" + error "RADARR_URL / RADARR_API_KEY not configured — check host*.conf" exit 1 fi if [[ -z "${TMDB_API_KEY:-}" ]]; then error "TMDB_API_KEY not configured — required for discovery" error "Get a free key at https://www.themoviedb.org/settings/api" - error "Configure HOST*_TMDB_API_KEY in master_host*.conf" + error "Configure HOST*_TMDB_API_KEY in host*.conf" exit 1 fi diff --git a/Media/playback_aware_sonarr_discovery.sh b/Media/playback_aware_sonarr_discovery.sh index 8654d03..50fa9b8 100755 --- a/Media/playback_aware_sonarr_discovery.sh +++ b/Media/playback_aware_sonarr_discovery.sh @@ -62,7 +62,7 @@ # ============================================================================================== # # TMDB API key — required for Stage 2 recommendations and external_ids lookup -# Configure HOST*_TMDB_API_KEY in master_host*.conf +# Configure HOST*_TMDB_API_KEY in host*.conf # Free key at: https://www.themoviedb.org/settings/api # # ============================================================================================== @@ -150,19 +150,19 @@ if ! command -v jq >/dev/null 2>&1; then fi if [[ -z "${EMBY_URL:-}" || -z "${EMBY_API_KEY:-}" ]]; then - error "EMBY_URL / EMBY_API_KEY not configured — check master_host*.conf" + error "EMBY_URL / EMBY_API_KEY not configured — check host*.conf" exit 1 fi if [[ -z "${SONARR_URL:-}" || -z "${SONARR_API_KEY:-}" ]]; then - error "SONARR_URL / SONARR_API_KEY not configured — check master_host*.conf" + error "SONARR_URL / SONARR_API_KEY not configured — check host*.conf" exit 1 fi if [[ -z "${TMDB_API_KEY:-}" ]]; then error "TMDB_API_KEY not configured — required for discovery" error "Get a free key at https://www.themoviedb.org/settings/api" - error "Configure HOST*_TMDB_API_KEY in master_host*.conf" + error "Configure HOST*_TMDB_API_KEY in host*.conf" exit 1 fi diff --git a/Media/radarr_cleanup.sh b/Media/radarr_cleanup.sh index 0925f72..55a2fac 100644 --- a/Media/radarr_cleanup.sh +++ b/Media/radarr_cleanup.sh @@ -53,7 +53,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_RADARR_URL / HOST*_RADARR_API_KEY / HOST*_RADARR_MOVIES_ROOT # HOST*_RADARR_PATH_MAP — container path → host path translation diff --git a/Media/radarr_tmdb_removed.sh b/Media/radarr_tmdb_removed.sh index e8dc9a1..d221d98 100644 --- a/Media/radarr_tmdb_removed.sh +++ b/Media/radarr_tmdb_removed.sh @@ -39,7 +39,7 @@ # # RADARR_DROPPED_ADD_EXCLUSION — add removed movies to import exclusion (default: true) # -# master_host*.conf +# host*.conf # # HOST1_RADARR_URL / HOST1_RADARR_API_KEY # Aliased by detect_hosts() — script uses RADARR_URL / RADARR_API_KEY diff --git a/Media/sonarr_cleanup.sh b/Media/sonarr_cleanup.sh index b36f6bf..c91ac40 100644 --- a/Media/sonarr_cleanup.sh +++ b/Media/sonarr_cleanup.sh @@ -53,7 +53,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_SONARR_URL / HOST*_SONARR_API_KEY / HOST*_SONARR_TV_ROOT # HOST*_SONARR_PATH_MAP — container path → host path translation diff --git a/Media/sonarr_tvdb_removed.sh b/Media/sonarr_tvdb_removed.sh index 6d10b5d..6d9dd48 100644 --- a/Media/sonarr_tvdb_removed.sh +++ b/Media/sonarr_tvdb_removed.sh @@ -38,7 +38,7 @@ # # SONARR_DROPPED_ADD_EXCLUSION — add removed series to import exclusion (default: true) # -# master_host*.conf +# host*.conf # # HOST1_SONARR_URL / HOST1_SONARR_API_KEY # Aliased by detect_hosts() — script uses SONARR_URL / SONARR_API_KEY diff --git a/Monitors/Manual-Monitors.md b/Monitors/Manual-Monitors.md index 7cdb7d2..77d0284 100644 --- a/Monitors/Manual-Monitors.md +++ b/Monitors/Manual-Monitors.md @@ -13,7 +13,7 @@ Each domain and subdomain has an independent TLS certificate and must be listed separately. `cert_monitor.sh` makes one openssl connection per entry. ```bash -# master_host1.conf +# host1.conf HOST1_CERT_MONITOR_DOMAINS=( "Gmer4Lfe.com" "Gmer4Lfe.us" @@ -21,7 +21,7 @@ HOST1_CERT_MONITOR_DOMAINS=( # "cloud.Gmer4Lfe.com" ) -# master_host2.conf +# host2.conf HOST2_CERT_MONITOR_DOMAINS=( "Jayred365.com" # "auth.Jayred365.com" @@ -63,12 +63,12 @@ CERT_TIMEOUT=10 # seconds per domain before declaring FAILED ### Drive Ignore List ```bash -# master_host1.conf +# host1.conf HOST1_SMART_IGNORE_DRIVES=( "sda" # boot USB flash drive — no meaningful SMART data ) -# master_host2.conf +# host2.conf HOST2_SMART_IGNORE_DRIVES=( "sda" # boot USB flash drive ) @@ -109,7 +109,7 @@ if `dynamix.cfg` is not found (e.g., running outside of unRAID). ### Share Configuration ```bash -# master_host1.conf +# host1.conf HOST1_BACKUP_VERIFY_SHARES=( # empty — uses HOST1_DAILY_SYNC_SHARES automatically # "/mnt/user/Movies" # override to check specific shares only @@ -230,9 +230,9 @@ data — it just doesn't trigger a notification for that condition. | State File | Source | What It Shows | |-----------|--------|---------------| | `FALLBACK_STATE_FILE` | `Fallback/fallback.sh` | Current fallback state (NORMAL/FALLBACK/etc.) | -| `SYS_WATCHDOG_FAILED_FILE` | `Docker_Essentials/docker_watchdog.sh` | Container skip list — needs human attention | -| `WATCHDOG_STATE_FILE` | `Docker_Essentials/docker_watchdog.sh` | Active container strike counts | -| `SYS_WATCHDOG_STATE_FILE` | `unRAID_Essentials/system_watchdog.sh` | Active system watchdog strikes | +| `SYS_WATCHDOG_FAILED_FILE` | `Watchdogs/docker_watchdog.sh` | Container skip list — needs human attention | +| `WATCHDOG_STATE_FILE` | `Watchdogs/docker_watchdog.sh` | Active container strike counts | +| `SYS_WATCHDOG_STATE_FILE` | `Watchdogs/system_watchdog.sh` | Active system watchdog strikes | | `BANDWIDTH_LOG` | `bandwidth_monitor.sh` | Yesterday's transfer history | | `TRANSCODE_DAILY_LOG` | `Transcodes/` | Weekly transcode statistics | | `CERT_MONITOR_DOMAINS` | live openssl check | Current cert status per domain | @@ -244,11 +244,11 @@ data — it just doesn't trigger a notification for that condition. ### Emby Credentials ```bash -# master_host1.conf +# host1.conf HOST1_EMBY_URL="http://192.168.50.2:8096" HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829" -# master_host2.conf +# host2.conf HOST2_EMBY_URL="http://192.168.50.3:8096" HOST2_EMBY_API_KEY="" ``` @@ -306,7 +306,7 @@ in `unRAID_Essentials/`. ### Pool Ignore List ```bash -# master_host1.conf +# host1.conf HOST1_ZFS_REPORT_IGNORE_POOLS=( "disk10" # JBOD member — high usage expected, exclude from report noise "disk9" @@ -315,7 +315,7 @@ HOST1_ZFS_REPORT_IGNORE_POOLS=( "disk5" ) -# master_host2.conf +# host2.conf HOST2_ZFS_REPORT_IGNORE_POOLS=( # list host2's JBOD members here ) @@ -392,7 +392,7 @@ ZFS_REPORT_AVAIL_WARN_GB=20 ZFS_REPORT_DOCKER_TOP=10 ``` -### master_host*.conf +### host*.conf ```bash # cert_monitor.sh @@ -485,7 +485,7 @@ emby_session_report.sh --dry-run # test connectivity, generate report, no noti ### --status Shows current configuration and exits without running checks. Use to verify -configuration is loaded correctly after editing `master.conf` or `master_host*.conf`. +configuration is loaded correctly after editing `master.conf` or `host*.conf`. ```bash cert_monitor.sh --status # domain list, CERT_WARN_DAYS, CERT_CRIT_DAYS, timeout diff --git a/Monitors/backup_verify.sh b/Monitors/backup_verify.sh index 07e0222..2b343d9 100644 --- a/Monitors/backup_verify.sh +++ b/Monitors/backup_verify.sh @@ -63,7 +63,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_BACKUP_VERIFY_SHARES # Shares to verify. Leave empty to use HOST*_DAILY_SYNC_SHARES automatically. @@ -140,7 +140,7 @@ fi if [[ ${#VERIFY_SHARES[@]} -eq 0 ]]; then warn "No shares configured for $MY_ID — nothing to verify" - warn "Check HOST*_BACKUP_VERIFY_SHARES or HOST*_DAILY_SYNC_SHARES in master_host*.conf" + warn "Check HOST*_BACKUP_VERIFY_SHARES or HOST*_DAILY_SYNC_SHARES in host*.conf" exit 0 fi diff --git a/Monitors/bandwidth_monitor.sh b/Monitors/bandwidth_monitor.sh index e49d189..892e83b 100644 --- a/Monitors/bandwidth_monitor.sh +++ b/Monitors/bandwidth_monitor.sh @@ -90,9 +90,6 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" -# Monitor script — output is the point -SILENT_MODE=false - parse_args "$@" # ── Parse mode from PARSED_ARGS ─────────────────────────────────────────────────────────────── diff --git a/Monitors/cert_monitor.sh b/Monitors/cert_monitor.sh index f0dc4a4..340cef3 100644 --- a/Monitors/cert_monitor.sh +++ b/Monitors/cert_monitor.sh @@ -51,7 +51,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_CERT_MONITOR_DOMAINS # Domains this host monitors. Each domain and subdomain is a separate entry — @@ -91,9 +91,6 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" -# Monitor script — output is the point -SILENT_MODE=false - parse_args "$@" # ============================================================================================== @@ -126,7 +123,7 @@ detect_hosts # Empty array guard if [[ ${#CERT_MONITOR_DOMAINS[@]} -eq 0 ]]; then warn "CERT_MONITOR_DOMAINS is empty for $MY_ID" - warn "Check HOST*_CERT_MONITOR_DOMAINS in master_host*.conf" + warn "Check HOST*_CERT_MONITOR_DOMAINS in host*.conf" exit 0 fi diff --git a/Monitors/emby_session_report.sh b/Monitors/emby_session_report.sh index 8437b3b..afc9d42 100644 --- a/Monitors/emby_session_report.sh +++ b/Monitors/emby_session_report.sh @@ -42,7 +42,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_EMBY_URL # Emby server URL for this host. Aliased by detect_hosts() → EMBY_URL. @@ -87,9 +87,6 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" -# Monitor/report script — output is the point -SILENT_MODE=false - parse_args "$@" # ============================================================================================== diff --git a/Monitors/smart_health.sh b/Monitors/smart_health.sh index 55b044c..21550e4 100644 --- a/Monitors/smart_health.sh +++ b/Monitors/smart_health.sh @@ -43,7 +43,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_SMART_IGNORE_DRIVES # Drives skipped in SMART monitoring. Aliased by detect_hosts() → @@ -80,9 +80,6 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" -# Monitor script — output is the point -SILENT_MODE=false - parse_args "$@" # ============================================================================================== diff --git a/Monitors/weekly_health_digest.sh b/Monitors/weekly_health_digest.sh index 4114212..0c7f4d7 100644 --- a/Monitors/weekly_health_digest.sh +++ b/Monitors/weekly_health_digest.sh @@ -116,9 +116,6 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" -# Report/monitor script — output is the point when sending -SILENT_MODE=false - parse_args "$@" # ============================================================================================== diff --git a/Monitors/zfs_memory_snapshot.sh b/Monitors/zfs_memory_snapshot.sh index 98b115d..95052ac 100644 --- a/Monitors/zfs_memory_snapshot.sh +++ b/Monitors/zfs_memory_snapshot.sh @@ -67,7 +67,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_ZFS_REPORT_IGNORE_POOLS # Pools excluded from health reporting. Single-disk JBOD members generate @@ -113,9 +113,6 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" -# Monitor/report script — output is the point -SILENT_MODE=false - parse_args "$@" DOCKER_TIMEOUT=15 diff --git a/Old_Arch_Still_Works/arr_cleanup.sh b/Old_Arch_Still_Works/arr_cleanup.sh index 9f69f43..e57c77f 100755 --- a/Old_Arch_Still_Works/arr_cleanup.sh +++ b/Old_Arch_Still_Works/arr_cleanup.sh @@ -17,7 +17,7 @@ # # ADDING A NEW ARR # ───────────────────────────────────────────────────────────────────────────── -# 1. Add HOST*__URL, API_KEY, MEDIA_ROOT, PATH_MAP to master_host*.conf +# 1. Add HOST*__URL, API_KEY, MEDIA_ROOT, PATH_MAP to host*.conf # 2. Add _ORPHAN_AGE, MAX_DELETE_GB, EXTENSIONS, etc. to master.conf # 3. Add a profile entry to ARR_PROFILES in arr_cleanup.py (6 values) # 4. Add an export block for the new arr below (copy Sonarr block, change prefix) diff --git a/Orchestrators/README-Orchestrators.md b/Orchestrators/README-Orchestrators.md index 5ce5e65..c740f17 100644 --- a/Orchestrators/README-Orchestrators.md +++ b/Orchestrators/README-Orchestrators.md @@ -214,10 +214,10 @@ ARRAY_START_SCRIPTS=( # watchdogs check container states # ── Continuous scripts — run until array stops ───────────────────────────── - "unRAID_Essentials/system_watchdog.sh" # system health BEFORE docker watchdog — + "Watchdogs/system_watchdog.sh" # system health BEFORE docker watchdog — # system watchdog writes state file that # docker watchdog reads every cycle - "Docker_Essentials/docker_watchdog.sh" # container health BEFORE failover — + "Watchdogs/docker_watchdog.sh" # container health BEFORE failover — # containers must be healthy for failover # to make reliable decisions "Failover/failover.sh" # failover LAST — needs everything else stable @@ -465,7 +465,7 @@ new search — hands-free recovery while you sleep. ### ── Host Awareness ─────────────────────────────────────────────────────────── ```bash -# master.conf + master_host*.conf +# master.conf + host*.conf # ───────────────────────────────────────────────────────────────────────────── # Each arr is independently toggled per host. # Lidarr only runs on HOST1 — exits cleanly on HOST2 with no action. @@ -580,7 +580,7 @@ DAILY_MAINTENANCE_SCRIPTS=( "Docker_Essentials/docker_daily_restart.sh" # POST-SYNC — restarts after everything ) -# master_host1.conf +# host1.conf HOST1_DAILY_SYNC_SHARES=( "/mnt/user/Movies" # HOST1 source of truth — push to HOST2 "/mnt/user/Tv_Shows" # HOST1 source of truth @@ -595,7 +595,7 @@ HOST1_PERSONAL_SHARES=( "/mnt/user/Personal" # encrypted personal share — appended after standard ) -# master_host2.conf +# host2.conf HOST2_DAILY_SYNC_SHARES=( "/mnt/user/Anime_Shows" # HOST2 source of truth — push to HOST1 "/mnt/user/Anime_Movies" # HOST2 source of truth diff --git a/Orchestrators/array_started.sh b/Orchestrators/array_started.sh index dd82acd..7c8fb0b 100644 --- a/Orchestrators/array_started.sh +++ b/Orchestrators/array_started.sh @@ -18,8 +18,8 @@ # Docker_Essentials/docker_network_connect.sh — ensure networks + container connections # # CONTINUOUS (run until array stops): -# unRAID_Essentials/system_watchdog.sh — system health monitor (last line of defense) -# Docker_Essentials/docker_watchdog.sh — container health monitor +# Watchdogs/system_watchdog.sh — system health monitor (last line of defense) +# Watchdogs/docker_watchdog.sh — container health monitor # Fallback/fallback.sh — mutual failover monitor # # ── WHY ORDER MATTERS ───────────────────────────────────────────────────────────────────────── diff --git a/Orchestrators/critical_sync_maintenance.sh b/Orchestrators/critical_sync_maintenance.sh index a53b8e4..8605455 100644 --- a/Orchestrators/critical_sync_maintenance.sh +++ b/Orchestrators/critical_sync_maintenance.sh @@ -123,7 +123,7 @@ if ! check_rsync_enabled "CRITICAL"; then echo "Critical rsync disabled — skipping sync, running partnership check only" elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then warn "CRITICAL_RSYNC_ENABLED=true but CRITICAL_SYNC_SHARES is empty for $MY_ID" - warn "Check HOST*_CRITICAL_SYNC_SHARES in master_host*.conf" + warn "Check HOST*_CRITICAL_SYNC_SHARES in host*.conf" else echo "Critical sync — $MY_ID → $REMOTE_ID — $(date '+%H:%M:%S')" diff --git a/Orchestrators/daily_sync_maintenance.sh b/Orchestrators/daily_sync_maintenance.sh index 107bb80..4d0669f 100644 --- a/Orchestrators/daily_sync_maintenance.sh +++ b/Orchestrators/daily_sync_maintenance.sh @@ -53,7 +53,7 @@ # Summary always shown — gives window timing and share/job counts. # Notify only on failure — successful daily maintenance doesn't need notification. # -# ── CONFIGURATION (master.conf + master_host*.conf) ─────────────────────────────────────────── +# ── CONFIGURATION (master.conf + host*.conf) ─────────────────────────────────────────── # HOST*_DAILY_SYNC_SHARES — shares pushed to mirror each day # HOST*_PERSONAL_SHARES — encrypted personal shares # DAILY_MAINTENANCE_SCRIPTS — maintenance jobs (permissions, cleanup, restart) @@ -244,7 +244,7 @@ if ! check_rsync_enabled "DAILY"; then warn "Daily rsync disabled — skipping all $SHARE_COUNT share syncs" warn "Proceeding to maintenance jobs..." elif [[ "$SHARE_COUNT" -eq 0 ]]; then - warn "No shares configured for $MY_ID — check HOST*_DAILY_SYNC_SHARES in master_host*.conf" + warn "No shares configured for $MY_ID — check HOST*_DAILY_SYNC_SHARES in host*.conf" else # Pre-flight — connectivity then remote rootfs check_connectivity diff --git a/Orchestrators/intermediate_sync_maintenance.sh b/Orchestrators/intermediate_sync_maintenance.sh index 2113af9..c34179c 100644 --- a/Orchestrators/intermediate_sync_maintenance.sh +++ b/Orchestrators/intermediate_sync_maintenance.sh @@ -24,7 +24,7 @@ # # ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── # detect_hosts() sets MY_ID and aliases HOST*_INTERMEDIATE_SYNC_SHARES → INTERMEDIATE_SYNC_SHARES. -# Each server can have a different set of mid-day shares — configure in master_host*.conf. +# Each server can have a different set of mid-day shares — configure in host*.conf. # Each script in INTERMEDIATE_MAINTENANCE_SCRIPTS handles its own host logic. # # ── DRIVE TEMP HANDLING ─────────────────────────────────────────────────────────────────────── @@ -41,7 +41,7 @@ # Silent on success — runs 4x/day, only failures warrant notification # # ── CONFIGURATION ───────────────────────────────────────────────────────────────────────────── -# master_host*.conf: HOST*_INTERMEDIATE_SYNC_SHARES — shares synced mid-day (empty = rsync skipped) +# host*.conf: HOST*_INTERMEDIATE_SYNC_SHARES — shares synced mid-day (empty = rsync skipped) # master.conf: INTERMEDIATE_RSYNC_ENABLED — enable/disable rsync section (default: true) # master.conf: INTERMEDIATE_MAINTENANCE_SCRIPTS — jobs run after rsync # master.conf: ARR_SYNC_ENABLED — toggle inside arr_sync.sh diff --git a/Orchestrators/sunday_morning_coffee_report.sh b/Orchestrators/sunday_morning_coffee_report.sh index d4f56e6..90b6b4a 100644 --- a/Orchestrators/sunday_morning_coffee_report.sh +++ b/Orchestrators/sunday_morning_coffee_report.sh @@ -154,6 +154,8 @@ line() { REPORT+=(" $1"); } issue() { ISSUES+=("$1"); REPORT+=(" ⚠️ $1"); } finding() { FINDINGS+=("$1"); REPORT+=(" ℹ️ $1"); } +get_array() { eval "echo \"\${${1}[*]}\""; } + format_bytes() { local bytes=$1 if (( bytes > 1073741824 )); then @@ -656,6 +658,90 @@ if command -v tailscale >/dev/null 2>&1; then fi fi +# ============================================================================================== +# ━━━ 🔗 MESH ━━━ +# ============================================================================================== +section "🔗 MESH" + +_ALL_HOST_IDS=() +for _h in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do + [[ -n "${!_h:-}" ]] && _ALL_HOST_IDS+=("$_h") +done + +# ── Members ── +if [[ ${#_ALL_HOST_IDS[@]} -eq 0 ]]; then + line "No hosts defined" +else + line "Members:" + for _h in "${_ALL_HOST_IDS[@]}"; do + _server="${!_h}" + _owner_var="${_h}_OWNER"; _owner="${!_owner_var:-unknown}" + _email_var="${_h}_OWNER_EMAIL"; _email="${!_email_var:-(not set)}" + line " $_h $_server / $_owner / $_email" + done +fi + +# ── Protected Services ── +_coverage_found=false +for _covered in "${_ALL_HOST_IDS[@]}"; do + _covered_owner_var="${_covered}_OWNER"; _covered_owner="${!_covered_owner_var:-$_covered}" + _covered_email_var="${_covered}_OWNER_EMAIL"; _covered_email="${!_covered_email_var:-}" + + declare -A _tier_containers=() + declare -A _tier_delays=() + _covered_by="" + _any_tiers=false + + for _covering in "${_ALL_HOST_IDS[@]}"; do + [[ "$_covering" == "$_covered" ]] && continue + for _tier in 1 2 3 4; do + _containers=$(get_array "FALLBACK_${_covering}_COVERS_${_covered}_TIER${_tier}") + [[ -z "$_containers" ]] && continue + _any_tiers=true + _tier_containers[$_tier]="$_containers" + _delay_var="${_covered}_TIER${_tier}_DELAY" + _tier_delays[$_tier]="${!_delay_var:-0}" + done + if [[ "$_any_tiers" == true ]]; then + _cov_owner_var="${_covering}_OWNER"; _cov_owner="${!_cov_owner_var:-$_covering}" + _covered_by="$_covering ($_cov_owner)" + fi + done + + [[ "$_any_tiers" == false ]] && { unset _tier_containers _tier_delays; declare -A _tier_containers=() _tier_delays=(); continue; } + + _coverage_found=true + _hdr="$_covered_owner" + [[ -n "$_covered_email" ]] && _hdr+=" — $_covered_email" + line "Protected: $_hdr" + for _tier in 1 2 3 4; do + [[ -z "${_tier_containers[$_tier]:-}" ]] && continue + _d="${_tier_delays[$_tier]:-0}" + if (( _d == 0 )); then _dlabel="immediate" + elif (( _d >= 1440 )); then _dlabel="$(( _d / 1440 ))d" + elif (( _d >= 60 )); then _dlabel="$(( _d / 60 ))hr" + else _dlabel="${_d}min" + fi + line " Tier $_tier (${_dlabel}): ${_tier_containers[$_tier]// /, }" + done + [[ -n "$_covered_by" ]] && line " Covered by: $_covered_by" + + unset _tier_containers _tier_delays + declare -A _tier_containers=() _tier_delays=() +done + +[[ "$_coverage_found" == false ]] && line "No fallback coverage configured" + +# ── Partnership ── +if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then + _po_host="${PARTNERSHIP_OWNER_HOST:-HOST1}" + _po_server="${!_po_host:-unknown}" + _po_name_var="${_po_host}_OWNER"; _po_name="${!_po_name_var:-unknown}" + line "Partnership: enabled — owner $_po_host ($_po_server / $_po_name), sync every ${PARTNERSHIP_SYNC_INTERVAL:-15}min" +else + line "Partnership: disabled" +fi + # ============================================================================================== # ━━━ 🔐 SECURITY ━━━ # ============================================================================================== diff --git a/Partnership/Manual-Partnership.md b/Partnership/Manual-Partnership.md index 9d58610..ffa762e 100644 --- a/Partnership/Manual-Partnership.md +++ b/Partnership/Manual-Partnership.md @@ -130,7 +130,7 @@ PARTNERSHIP_ONBOARD_VERIFY=true # curl-verify each WebUI after onboard PARTNERSHIP_ONBOARD_NOTIFY=true # notify both servers on successful onboard ``` -### master_host1.conf (owner side) +### host1.conf (owner side) ```bash # Containers whose WebUI URLs are redirected to owner's Tailscale IP on onboard. @@ -179,7 +179,7 @@ HOST1_PARTNERSHIP_OWN_CONTAINERS=( ) ``` -### master_host2.conf (mirror side) +### host2.conf (mirror side) ```bash # Containers whose WebUI URLs are redirected on onboard. @@ -342,9 +342,9 @@ Partnership/partnership_onboard.sh ### Adding a New Container to the Auth Stack 1. Create the XML template on HOST1 (`/boot/config/plugins/dockerMan/templates-user/my-NewContainer.xml`) -2. Add the XML filename to `HOST1_PARTNERSHIP_AUTH_STACK` in `master_host1.conf` +2. Add the XML filename to `HOST1_PARTNERSHIP_AUTH_STACK` in `host1.conf` - If it has a database dependency, put the dep earlier in the array -3. Add the container name to `HOST2_PARTNERSHIP_REPLACE_CONTAINERS` in `master_host2.conf` +3. Add the container name to `HOST2_PARTNERSHIP_REPLACE_CONTAINERS` in `host2.conf` 4. Re-run the auth stack portion: ```bash Partnership/partnership_onboard.sh --skip-ssh --skip-arr-stack --skip-arr-sync @@ -514,7 +514,7 @@ automatically, but if HOST2 had no pre-existing auth stack of its own, it needs docker ps # Verify own parked containers came back up: -# (listed in HOST2_PARTNERSHIP_OWN_CONTAINERS in master_host2.conf) +# (listed in HOST2_PARTNERSHIP_OWN_CONTAINERS in host2.conf) # If you need a fresh auth stack, deploy from HOST2's own XML templates: docker create ... && docker start NginxProxyManager # etc. diff --git a/Partnership/partnership_manager.sh b/Partnership/partnership_manager.sh index 9522e57..6cf8084 100644 --- a/Partnership/partnership_manager.sh +++ b/Partnership/partnership_manager.sh @@ -141,7 +141,7 @@ # TAILSCALE_API_KEY / TAILSCALE_TAILNET # Required when PARTNERSHIP_REMOVE_TAILSCALE=true # -# master_host*.conf +# host*.conf # # HOST*_PARTNERSHIP_AUTH_WEBUIS # Containers reconfigured on onboard/offboard. Format: "ContainerName|WebUIPort" @@ -253,7 +253,7 @@ OWNER="${!OWNER_ID}" # hostname string MIRROR="${!MIRROR_ID}" # SSH_KEY (set by detect_hosts) is this server's own private key. # The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh. -# With sparse checkout, each server only has its own master_host{N}.conf — the other server's +# With sparse checkout, each server only has its own host{N}.conf — the other server's # key path is never available here. Use SSH_KEY for all outbound SSH regardless of mode. MIRROR_SSH_KEY="$SSH_KEY" OWNER_SSH_KEY="$SSH_KEY" diff --git a/Partnership/partnership_offboard.sh b/Partnership/partnership_offboard.sh index 09bc1fc..7351aad 100755 --- a/Partnership/partnership_offboard.sh +++ b/Partnership/partnership_offboard.sh @@ -39,7 +39,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_PARTNERSHIP_AUTH_STACK # Auth container XMLs to push during onboard — used on offboard to identify what diff --git a/Partnership/partnership_onboard.sh b/Partnership/partnership_onboard.sh index 178a936..342001d 100755 --- a/Partnership/partnership_onboard.sh +++ b/Partnership/partnership_onboard.sh @@ -45,7 +45,7 @@ # # Dependency ordering in the auth stack is owner-enforced # PARTNERSHIP_AUTH_STACK order matters: Mariadb and Redis must come before Authelia. -# The array is ordered correctly in master_host1.conf. After each Mariadb/Redis deploy, +# The array is ordered correctly in host1.conf. After each Mariadb/Redis deploy, # the script waits for the container to be healthy before continuing. This is a remote # health check — the container must be running (or report healthy) before the next # dependent is deployed. @@ -73,7 +73,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_PARTNERSHIP_AUTH_STACK # XML filenames (from this server's templates-user/) to push and deploy on the @@ -82,7 +82,7 @@ # # HOST*_PARTNERSHIP_REPLACE_CONTAINERS # Containers to stop on the mirror before deploying the auth stack. -# Defined in the MIRROR's own conf (master_host*.conf on HOST2) — never in HOST1's conf. +# Defined in the MIRROR's own conf (host*.conf on HOST2) — never in HOST1's conf. # Read live from the mirror via SSH during Step 3 (sources mirror's load_config.sh at # the same $SCRIPTS_ROOT path — convention: both servers use the same repo location). # Leave empty on HOST2 if no conflicting containers exist (fresh mirror: nothing to stop). @@ -171,7 +171,7 @@ OWNER="${!OWNER_ID}" MIRROR="${!MIRROR_ID}" # SSH_KEY (set by detect_hosts) is this server's own private key. # The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh. -# HOST{N}_SSH_KEY lives in master_host{N}.conf — with sparse checkout, the other server's +# HOST{N}_SSH_KEY lives in host{N}.conf — with sparse checkout, the other server's # conf is never present here. Always use SSH_KEY (local private key) for outbound SSH. MIRROR_SSH_KEY="$SSH_KEY" @@ -359,7 +359,7 @@ wait_for_container_healthy() { # # SSHes to the mirror, sources its load_config.sh at the same $SCRIPTS_ROOT path (both servers # use the same convention), and reads the named config array from the mirror's own conf. -# HOST2's container list stays in HOST2's master_host2.conf — not duplicated in HOST1's conf. +# HOST2's container list stays in HOST2's host2.conf — not duplicated in HOST1's conf. # Fails gracefully if scripts aren't present yet or the array is empty (nothing to stop). # # deploy_container_from_xml() already stops/removes containers with the same name as what's @@ -547,7 +547,7 @@ if [[ "$SKIP_AUTH_STACK" == true ]]; then warn "Skipping (--skip-auth-stack)" elif [[ ${#PARTNERSHIP_AUTH_STACK[@]} -eq 0 ]]; then warn "PARTNERSHIP_AUTH_STACK not set in ${MY_ID} conf — skipping auth stack deploy" - warn "Add HOST${MY_ID: -1}_PARTNERSHIP_AUTH_STACK to master_host${MY_ID: -1}.conf" + warn "Add HOST${MY_ID: -1}_PARTNERSHIP_AUTH_STACK to host${MY_ID: -1}.conf" STEP_AUTH_OK=false else deploy_xml_stack PARTNERSHIP_AUTH_STACK diff --git a/Partnership/ssh_setup.sh b/Partnership/ssh_setup.sh index 134454a..4e07f86 100755 --- a/Partnership/ssh_setup.sh +++ b/Partnership/ssh_setup.sh @@ -7,7 +7,7 @@ # unRAID-Gmer4Lfe → gmer4lfe_rsync_automation # unRAID-Jayred365 → jayred365_rsync_automation # Idempotent — skips generation if key already exists (use --force to regenerate). -# Updates master_host*.conf with key path on success. +# Updates host*.conf with key path on success. # # ── MODES ───────────────────────────────────────────────────────────────────────────────────── # (default) — generate key if missing, copy to remote, update conf @@ -85,7 +85,7 @@ SSH_PUB_PATH="${SSH_KEY_PATH}.pub" # ── Host conf path ──────────────────────────────────────────────────────────────────────────── HOST_NUM="${MY_ID#HOST}" # "1" or "2" -HOST_CONF="$SCRIPTS_ROOT/master_host${HOST_NUM}.conf" +HOST_CONF="$SCRIPTS_ROOT/host${HOST_NUM}.conf" KEY_CONF_VAR="${MY_ID}_SSH_KEY" # ── Strike state file ───────────────────────────────────────────────────────────────────────── @@ -316,12 +316,12 @@ fi # ── Update conf ─────────────────────────────────────────────────────────────────────────────── echo "" -echo "━━━ $ICON_GEAR Update master_host${HOST_NUM}.conf ━━━" +echo "━━━ $ICON_GEAR Update host${HOST_NUM}.conf ━━━" if [[ "$DRY_RUN" == false ]]; then update_conf_key_path else - warn "DRY RUN — would set ${KEY_CONF_VAR}=\"${SSH_KEY_PATH}\" in master_host${HOST_NUM}.conf" + warn "DRY RUN — would set ${KEY_CONF_VAR}=\"${SSH_KEY_PATH}\" in host${HOST_NUM}.conf" fi # ── Copy to remote ──────────────────────────────────────────────────────────────────────────── @@ -373,7 +373,7 @@ echo "" echo "━━━━━ $ICON_SUMMARY SSH SETUP SUMMARY ━━━━━" echo " Key: $SSH_KEY_PATH" echo " Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)" -echo " Conf: ${KEY_CONF_VAR} in master_host${HOST_NUM}.conf" +echo " Conf: ${KEY_CONF_VAR} in host${HOST_NUM}.conf" echo "" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \ warn "$ICON_DONE DONE — SSH key ready for rsync automation ✅" diff --git a/README-User_Script_Plug-in.md b/README-User_Script_Plug-in.md index 22e68aa..1ade41d 100644 --- a/README-User_Script_Plug-in.md +++ b/README-User_Script_Plug-in.md @@ -893,7 +893,7 @@ Background: NO ```bash #!/bin/bash -/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_watchdog.sh --status +/mnt/user/appdata/unraid_scripts/Watchdogs/docker_watchdog.sh --status ``` **What it shows:** @@ -1275,7 +1275,7 @@ Background: NO ```bash #!/bin/bash -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/system_watchdog.sh --status +/mnt/user/appdata/unraid_scripts/Watchdogs/system_watchdog.sh --status ``` **What it shows:** diff --git a/README.md b/README.md index 717b072..f32d38e 100644 --- a/README.md +++ b/README.md @@ -462,8 +462,8 @@ Monitoring: Unraid_Scripts/ │ ├── master.conf ← All shared configuration — the only file you edit regularly -├── master_host1.conf ← HOST1-specific: share lists, container names, API keys -├── master_host2.conf ← HOST2-specific: same structure, different values +├── host1.conf ← HOST1-specific: share lists, container names, API keys +├── host2.conf ← HOST2-specific: same structure, different values ├── common.sh ← Shared library — all functions used by every script ├── load_config.sh ← Sources all conf files and common.sh │ @@ -509,11 +509,11 @@ Unraid_Scripts/ # definitions, watchdog settings, arr cleanup config, DDNS timing, etc. # Pushed to both servers via git. Never contains server-specific values. # -# master_host1.conf — sourced only on HOST1 +# host1.conf — sourced only on HOST1 # HOST1_* prefixed variables: share lists, container names, API keys, # ramdisk size, specific paths, per-server toggle overrides. # -# master_host2.conf — sourced only on HOST2 +# host2.conf — sourced only on HOST2 # HOST2_* prefixed variables: same structure, different values. # # detect_hosts() in common.sh: diff --git a/Rsync/Manual-Rsync.md b/Rsync/Manual-Rsync.md index af60693..ef75b46 100644 --- a/Rsync/Manual-Rsync.md +++ b/Rsync/Manual-Rsync.md @@ -130,7 +130,7 @@ access key (for script repository pulls). ### 3a — Rsync Automation Keys (ssh_setup.sh) Run on **each server**. `ssh_setup.sh` generates the key, copies it to the remote, -and updates `master_host*.conf` with the key path automatically. +and updates `host*.conf` with the key path automatically. ```bash # On HOST1 — after the repo is cloned: @@ -146,7 +146,7 @@ unRAID-Gmer4Lfe → /root/.ssh/gmer4lfe_rsync_automation unRAID-Jayred365 → /root/.ssh/jayred365_rsync_automation ``` -`master_host*.conf` is updated automatically with `HOST*_SSH_KEY` pointing to the +`host*.conf` is updated automatically with `HOST*_SSH_KEY` pointing to the generated key. Run `--status` to verify: ```bash @@ -188,8 +188,8 @@ find /mnt/user/appdata/unraid_scripts -name "*.sh" -exec chmod +x {} \; Expected structure after clone: ``` master.conf ← all shared configuration -master_host1.conf ← HOST1-specific configuration -master_host2.conf ← HOST2-specific configuration +host1.conf ← HOST1-specific configuration +host2.conf ← HOST2-specific configuration common.sh ← shared library load_config.sh ← config loader Orchestrators/ @@ -232,7 +232,7 @@ SSH_PORT=221 ### Daily Sync Shares ```bash -# master_host1.conf +# host1.conf # Shares this server downloads to — pushed to the other server nightly. # arr_sync.sh runs before rsync in the daily window, so the receiving # server's arrs already track incoming files when they arrive. @@ -247,7 +247,7 @@ HOST1_DAILY_SYNC_SHARES=( "/mnt/user/stand-up_comedy" ) -# master_host2.conf +# host2.conf HOST2_DAILY_SYNC_SHARES=( "/mnt/user/Anime_Shows" "/mnt/user/Anime_Movies" @@ -274,8 +274,8 @@ to their unprefixed names. Scripts only ever reference the unprefixed name — t work identically on both servers. ```bash -nano /mnt/user/appdata/unraid_scripts/master_host1.conf # on HOST1 -nano /mnt/user/appdata/unraid_scripts/master_host2.conf # on HOST2 +nano /mnt/user/appdata/unraid_scripts/host1.conf # on HOST1 +nano /mnt/user/appdata/unraid_scripts/host2.conf # on HOST2 ``` Every variable is documented in the conf files. Key values to set: @@ -413,7 +413,7 @@ zfs load-key poolname/Gmer4Lfe-Personal # prompts for passphrase zfs mount poolname/Gmer4Lfe-Personal ``` -### Add to master_host1.conf +### Add to host1.conf ```bash HOST1_PERSONAL_SHARES=( @@ -662,7 +662,7 @@ declare -A PROFILE_EXCLUDE_DIRS declare -A PROFILE_REMOTE_RESTART_CONTAINERS ``` -### master_host*.conf +### host*.conf ```bash # Daily sync shares — one list per server (mutually exclusive) diff --git a/Tools/Manual-Tools.md b/Tools/Manual-Tools.md index 20f4484..de3f71a 100644 --- a/Tools/Manual-Tools.md +++ b/Tools/Manual-Tools.md @@ -432,7 +432,7 @@ library.db-wal — write-ahead log (uncommitted transactions) the main library.db is also affected ``` -### Configuration (master_host*.conf) +### Configuration (host*.conf) ```bash HOST1_EMBY_CONTAINER="Emby" # aliased by detect_hosts() → EMBY_CONTAINER @@ -470,7 +470,7 @@ without triggering any error — until you try to read that specific file. By th Run monthly. Also run after any disk replacement or power event. Safe to run while the system is in use — scrub runs at low I/O priority. -### Configuration (master_host*.conf) +### Configuration (host*.conf) ```bash HOST1_ZFS_REPORT_IGNORE_POOLS=( diff --git a/Tools/emby_database_repair.sh b/Tools/emby_database_repair.sh index a0e8550..dd19a90 100644 --- a/Tools/emby_database_repair.sh +++ b/Tools/emby_database_repair.sh @@ -55,7 +55,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_EMBY_CONTAINER # Name of the Emby Docker container on this host. @@ -122,7 +122,7 @@ fi detect_hosts if [[ -z "${EMBY_CONTAINER:-}" ]]; then - error "EMBY_CONTAINER not set for $MY_ID — check HOST*_EMBY_CONTAINER in master_host*.conf" + error "EMBY_CONTAINER not set for $MY_ID — check HOST*_EMBY_CONTAINER in host*.conf" exit 1 fi @@ -135,7 +135,7 @@ EMBY_CONFIG_HOST=$(timeout "$DOCKER_TIMEOUT" docker inspect "$EMBY_CONTAINER" 2> if [[ -z "$EMBY_CONFIG_HOST" ]]; then error "Could not detect Emby config path from Docker mounts" - error "Is $EMBY_CONTAINER the correct container name? Check HOST*_EMBY_CONTAINER in master_host*.conf" + error "Is $EMBY_CONTAINER the correct container name? Check HOST*_EMBY_CONTAINER in host*.conf" exit 1 fi diff --git a/Tools/emby_to_lidarr_sync.sh b/Tools/emby_to_lidarr_sync.sh index f64378f..a09c601 100755 --- a/Tools/emby_to_lidarr_sync.sh +++ b/Tools/emby_to_lidarr_sync.sh @@ -36,7 +36,7 @@ # modification. Safe to run multiple times — the second run finds nothing to add. # # ============================================================================================== -# CONFIGURATION (master_host*.conf) +# CONFIGURATION (host*.conf) # ============================================================================================== # # HOST*_LIDARR_URL / HOST*_LIDARR_API_KEY — Lidarr connection (aliased by detect_hosts) @@ -73,12 +73,12 @@ if ! command -v jq >/dev/null 2>&1; then fi if [[ -z "${EMBY_URL:-}" || -z "${EMBY_API_KEY:-}" ]]; then - error "EMBY_URL / EMBY_API_KEY not configured — check master_host*.conf" + error "EMBY_URL / EMBY_API_KEY not configured — check host*.conf" exit 1 fi if [[ -z "${LIDARR_URL:-}" || -z "${LIDARR_API_KEY:-}" ]]; then - error "LIDARR_URL / LIDARR_API_KEY not configured — check master_host*.conf" + error "LIDARR_URL / LIDARR_API_KEY not configured — check host*.conf" exit 1 fi diff --git a/Tools/emby_to_radarr_sync.sh b/Tools/emby_to_radarr_sync.sh index ce0f1bc..38f845c 100644 --- a/Tools/emby_to_radarr_sync.sh +++ b/Tools/emby_to_radarr_sync.sh @@ -38,7 +38,7 @@ # without modification. Safe to run multiple times — the second run finds nothing to add. # # ============================================================================================== -# CONFIGURATION (master_host*.conf) +# CONFIGURATION (host*.conf) # ============================================================================================== # # HOST*_RADARR_URL / HOST*_RADARR_API_KEY — Radarr connection (aliased by detect_hosts) @@ -74,12 +74,12 @@ if ! command -v jq >/dev/null 2>&1; then fi if [[ -z "${EMBY_URL:-}" || -z "${EMBY_API_KEY:-}" ]]; then - error "EMBY_URL / EMBY_API_KEY not configured — check master_host*.conf" + error "EMBY_URL / EMBY_API_KEY not configured — check host*.conf" exit 1 fi if [[ -z "${RADARR_URL:-}" || -z "${RADARR_API_KEY:-}" ]]; then - error "RADARR_URL / RADARR_API_KEY not configured — check master_host*.conf" + error "RADARR_URL / RADARR_API_KEY not configured — check host*.conf" exit 1 fi diff --git a/Tools/emby_to_sonarr_sync.sh b/Tools/emby_to_sonarr_sync.sh index be497a7..0fef85a 100644 --- a/Tools/emby_to_sonarr_sync.sh +++ b/Tools/emby_to_sonarr_sync.sh @@ -38,7 +38,7 @@ # without modification. Safe to run multiple times — the second run finds nothing to add. # # ============================================================================================== -# CONFIGURATION (master.conf / master_host*.conf) +# CONFIGURATION (master.conf / host*.conf) # ============================================================================================== # # SONARR_EMBY_LIBRARIES — Emby library names to scan (master.conf) @@ -75,12 +75,12 @@ if ! command -v jq >/dev/null 2>&1; then fi if [[ -z "${EMBY_URL:-}" || -z "${EMBY_API_KEY:-}" ]]; then - error "EMBY_URL / EMBY_API_KEY not configured — check master_host*.conf" + error "EMBY_URL / EMBY_API_KEY not configured — check host*.conf" exit 1 fi if [[ -z "${SONARR_URL:-}" || -z "${SONARR_API_KEY:-}" ]]; then - error "SONARR_URL / SONARR_API_KEY not configured — check master_host*.conf" + error "SONARR_URL / SONARR_API_KEY not configured — check host*.conf" exit 1 fi diff --git a/Tools/zfs_pool_scrub.sh b/Tools/zfs_pool_scrub.sh index 1bf7b05..00126ad 100644 --- a/Tools/zfs_pool_scrub.sh +++ b/Tools/zfs_pool_scrub.sh @@ -50,7 +50,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_ZFS_REPORT_IGNORE_POOLS # Pools to exclude from automatic scrub. Typically single-disk VM pools diff --git a/Transcodes/Manual-Transcoding.md b/Transcodes/Manual-Transcoding.md index 5130190..975b681 100644 --- a/Transcodes/Manual-Transcoding.md +++ b/Transcodes/Manual-Transcoding.md @@ -216,7 +216,7 @@ TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd ### Threshold Sizing ```bash -# master_host*.conf +# host*.conf HOST1_RAMDISK_SIZE="10G" HOST1_RAMDISK_WARN_GB=8.8 # flip to SSD above this HOST1_RAMDISK_LOW_GB=6.5 # flip back below this @@ -331,7 +331,7 @@ transcode_cleanup.sh --log # verbose per-file output TRANSCODE_LINK="/mnt/ram-transcode" # symlink Emby points at TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" # SSD fallback -# ── Per-Host (master_host*.conf) ─────────────────────────────────────────────── +# ── Per-Host (host*.conf) ─────────────────────────────────────────────── HOST1_RAMDISK_PATH="/mnt/ramdisk_transcodes" # ramdisk mount point HOST1_RAMDISK_SIZE="10G" # tmpfs ceiling (not a reservation) HOST1_RAMDISK_WARN_GB=8.8 # flip to SSD above this @@ -420,7 +420,7 @@ docker inspect Emby | grep -A3 "Mounts" # Check peak usage from the weekly health digest: Transcodes → "Week peak: X.XGB" # # If peak is close to RAMDISK_WARN_GB → increase ramdisk size: -# master_host1.conf +# host1.conf HOST1_RAMDISK_SIZE="12G" # increase by 2G HOST1_RAMDISK_WARN_GB=10.5 # adjust thresholds accordingly HOST1_RAMDISK_LOW_GB=8.5 @@ -465,7 +465,7 @@ readlink /mnt/ram-transcode ### Increasing Ramdisk Size After Initial Setup ```bash -# 1. Set new size and thresholds in master_host*.conf +# 1. Set new size and thresholds in host*.conf # 2. Unmount the existing ramdisk (no sessions should be active): umount /mnt/ramdisk_transcodes diff --git a/Transcodes/ramdisk_setup.sh b/Transcodes/ramdisk_setup.sh index f4b4230..607eb97 100755 --- a/Transcodes/ramdisk_setup.sh +++ b/Transcodes/ramdisk_setup.sh @@ -53,7 +53,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_RAMDISK_SIZE # tmpfs ceiling (e.g. 10G). Must change together with WARN_GB and LOW_GB. diff --git a/Transcodes/transcode_cleanup.sh b/Transcodes/transcode_cleanup.sh index 4804db8..b699fd2 100644 --- a/Transcodes/transcode_cleanup.sh +++ b/Transcodes/transcode_cleanup.sh @@ -65,7 +65,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_RAMDISK_PATH / HOST*_TRANSCODE_SSD / HOST*_RAMDISK_LOW_GB # Aliased by detect_hosts() → RAMDISK_PATH / TRANSCODE_SSD / RAMDISK_LOW_GB. diff --git a/Transcodes/transcode_manager.sh b/Transcodes/transcode_manager.sh index a66eb48..b6d75b7 100644 --- a/Transcodes/transcode_manager.sh +++ b/Transcodes/transcode_manager.sh @@ -65,7 +65,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_RAMDISK_PATH / HOST*_TRANSCODE_SSD # Ramdisk mount point and SSD fallback path. diff --git a/Docker_Essentials/docker_watchdog.sh b/Watchdogs/docker_watchdog.sh similarity index 99% rename from Docker_Essentials/docker_watchdog.sh rename to Watchdogs/docker_watchdog.sh index 3fd61a9..dccfbca 100755 --- a/Docker_Essentials/docker_watchdog.sh +++ b/Watchdogs/docker_watchdog.sh @@ -19,7 +19,7 @@ # ============================================================================================== # # Tier 1 — Strict Per-Container Monitoring -# Applies only to containers explicitly configured in master_host*.conf. +# Applies only to containers explicitly configured in host*.conf. # # Memory hard limits — immediate restart if container exceeds MB ceiling # Memory soft threshold — warn at SOFT_MEM_THRESHOLD % of limit (no restart) @@ -145,7 +145,7 @@ # CONFIGURATION # ============================================================================================== # -# master_host*.conf +# host*.conf # # HOST*_WATCHDOG_CONTAINERS # Memory hard limits per container. Format: "ContainerName:LimitInMB" @@ -922,6 +922,7 @@ CYCLE_START=$(date +%s) fi # WATCHDOG_SCAN_ALL + # ── Send notifications ──────────────────────────────────────────────────────────────────── flush_notify diff --git a/unRAID_Essentials/resource_watchdog.sh b/Watchdogs/resource_watchdog.sh similarity index 99% rename from unRAID_Essentials/resource_watchdog.sh rename to Watchdogs/resource_watchdog.sh index 05927bb..4729c7b 100755 --- a/unRAID_Essentials/resource_watchdog.sh +++ b/Watchdogs/resource_watchdog.sh @@ -71,7 +71,7 @@ # RW_QBIT_ENABLED, RW_QBIT_DL_SOFT, RW_QBIT_DL_MEDIUM # RW_CRITICAL_CONTAINERS — never paused or stopped regardless of pressure # -# master_host*.conf (aliased by detect_hosts()) +# host*.conf (aliased by detect_hosts()) # HOST*_RW_PAUSE_CONTAINERS — docker pause at medium pressure # HOST*_RW_STOP_CONTAINERS — docker stop at hard pressure # HOST*_SABNZBD_URL, HOST*_SABNZBD_API_KEY diff --git a/Watchdogs/storage_watchdog.sh b/Watchdogs/storage_watchdog.sh new file mode 100755 index 0000000..01e85f9 --- /dev/null +++ b/Watchdogs/storage_watchdog.sh @@ -0,0 +1,428 @@ +#!/bin/bash +# ============================================================================================== +# ================================= Storage Watchdog =========================================== +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Pool and storage health monitoring — catches runaway data growth before it +# fills a pool. Runs as a single-pass script called by watchdog_orchestrator.sh +# every cycle. Sits between docker_watchdog.sh (container health) and +# system_watchdog.sh (last line of defense). Never reboots — detects, alerts, +# and optionally remediates. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Appdata Size Monitoring +# Two complementary checks run every cycle: +# +# Part 1 — Growth rate (zero-config catch-all): +# Reads per-container dir totals via du, compares to previous cycle baseline. +# Any container growing more than WATCHDOG_APPDATA_GROWTH_GB triggers a focused +# *.log scan inside that container's dir. No per-container config required — +# new containers are covered automatically. Baseline built on first cycle after +# boot; growth detection active from cycle 2. +# +# Part 2 — Absolute log size: +# Finds *.log / *.log.* files over WATCHDOG_APPDATA_LOG_MAX_GB anywhere in +# WATCHDOG_APPDATA_PATHS. Catches logs already large but no longer actively +# growing. Independent strike counter per file. +# +# Strike System +# Reuses the same strike pattern as CPU/HTTP checks in docker_watchdog.sh: +# +# Strike 1 — warn + notify: condition first detected this run +# Strike 2 — warn + escalated notify: still present next cycle +# Strike 3 (WATCHDOG_APPDATA_STRIKE_LIMIT) — action cycle: +# WATCHDOG_APPDATA_TRUNCATE_LOGS=true → truncate *.log in-place, clear strikes +# WATCHDOG_APPDATA_TRUNCATE_LOGS=false → critical notify, hold strikes until resolved +# Condition resolves (growth stops / log drops below threshold) → strikes auto-clear +# +# Suppress Ceiling (WATCHDOG_APPDATA_SIZES) +# Containers in HOST*_WATCHDOG_APPDATA_SIZES suppress growth warnings while +# under their configured ceiling MB. Use ONLY when a container legitimately +# holds large stable data and would otherwise false-alarm (e.g. Tdarr cache). +# Zero-config growth detection covers everything else automatically. +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Zero-config for new containers +# Growth rate detection requires no per-container configuration. Add a game +# server, spin up a new arr, install anything — it is monitored automatically +# from the second cycle after it appears. The suppress ceiling in conf is the +# exception, not the rule. +# +# Alert-only for data, truncate-only for logs +# Non-log growth (databases, game saves, caches) is detected and alerted but +# never touched. Only *.log / *.log.* files are candidates for truncation — +# and only when WATCHDOG_APPDATA_TRUNCATE_LOGS=true. Truncation zeroes the +# file in-place; the container keeps its open file handle, space is reclaimed +# immediately. Never deletes. +# +# Strike before acting +# One cycle of growth could be a legitimate library scan or game save burst. +# Three consecutive cycles of growth is a runaway. The strike system separates +# transient activity from sustained problems before any action fires. +# +# Silent when healthy +# Produces no output when all checks pass. Loud only when something needs +# attention. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# WATCHDOG_CHECK_APPDATA +# Master toggle for all appdata checks (default: true) +# +# WATCHDOG_APPDATA_PATHS +# Array of paths to scan (e.g. "/mnt/docker-unraid/appdata") +# +# WATCHDOG_APPDATA_GROWTH_GB +# Per-cycle growth threshold in GB — flag containers growing more than this (default: 2) +# +# WATCHDOG_APPDATA_LOG_MAX_GB +# Absolute *.log file size alert threshold in GB (default: 2) +# +# WATCHDOG_APPDATA_TRUNCATE_LOGS +# Auto-truncate oversized *.log files on action cycle (default: false) +# +# WATCHDOG_APPDATA_STRIKE_LIMIT +# Consecutive cycles before action fires (default: 3) +# +# WATCHDOG_APPDATA_GROWTH_FILE +# Per-container size baseline — /tmp resets on reboot (correct: stale baseline +# after reboot would give false growth readings on first cycle) +# +# STORAGE_WATCHDOG_STATE_FILE +# Strike counts for this script — /tmp resets on reboot +# +# host*.conf (aliased by detect_hosts()) +# +# HOST*_WATCHDOG_APPDATA_SIZES +# Per-container growth suppress ceilings in MB. Suppress growth alerts while +# a container's dir stays below this ceiling. Only needed when a container +# legitimately has large stable data. Growth detection covers everything else. +# +# ============================================================================================== +# STATE FILES +# ============================================================================================== +# +# STORAGE_WATCHDOG_STATE_FILE — strike counts (/tmp — resets on reboot ✅) +# WATCHDOG_APPDATA_GROWTH_FILE — per-container size baseline (/tmp — resets on reboot ✅) +# +# /tmp files reset on reboot — correct. Pre-reboot strikes and growth baselines are +# meaningless after a reboot. Both rebuild cleanly from cycle 1. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# storage_watchdog.sh +# Single-pass storage health check. Silent if all healthy. +# +# storage_watchdog.sh --dry-run +# Run all checks without truncating anything. Shows what would be actioned. +# +# storage_watchdog.sh --status +# Show configuration, active strikes, and growth baseline status. Then exit. +# +# storage_watchdog.sh --log +# Verbose output — every container checked, every size comparison, every decision. +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../load_config.sh" + +parse_args "$@" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +acquire_lock +detect_hosts + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be truncated" + +touch "$STORAGE_WATCHDOG_STATE_FILE" 2>/dev/null +touch "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STORAGE WATCHDOG STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_GEAR Appdata check: ${WATCHDOG_CHECK_APPDATA:-true}" + echo "$ICON_GEAR Paths: ${WATCHDOG_APPDATA_PATHS[*]:-none}" + echo "$ICON_GEAR Growth thresh: ${WATCHDOG_APPDATA_GROWTH_GB:-2}GB/cycle" + echo "$ICON_GEAR Log max: ${WATCHDOG_APPDATA_LOG_MAX_GB:-2}GB" + echo "$ICON_GEAR Truncate logs: ${WATCHDOG_APPDATA_TRUNCATE_LOGS:-false}" + echo "$ICON_GEAR Strike limit: ${WATCHDOG_APPDATA_STRIKE_LIMIT:-3}" + echo "" + echo "── Active Strikes ──" + if [[ -s "$STORAGE_WATCHDOG_STATE_FILE" ]]; then + while IFS=':' read -r _sk _sv; do + _sv_clean="${_sv//[^0-9]/}" + [[ "${_sv_clean:-0}" -gt 0 ]] && echo " $_sk → $_sv_clean strikes" + done < "$STORAGE_WATCHDOG_STATE_FILE" + else + echo " none" + fi + echo "" + echo "── Growth Baseline ──" + if [[ -s "$WATCHDOG_APPDATA_GROWTH_FILE" ]]; then + _entries=$(wc -l < "$WATCHDOG_APPDATA_GROWTH_FILE") + _age=$(( $(date +%s) - $(stat -c %Y "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null || echo 0) )) + echo " Entries: $_entries containers Age: $(( _age / 60 ))m ago" + else + echo " No baseline yet (builds on first cycle after boot)" + fi + echo "" + echo "── Suppress Ceilings (this host) ──" + if [[ ${#WATCHDOG_APPDATA_SIZES[@]} -gt 0 ]]; then + for _c in "${!WATCHDOG_APPDATA_SIZES[@]}"; do + _ceil_gb=$(awk "BEGIN {printf \"%.0f\", ${WATCHDOG_APPDATA_SIZES[$_c]} / 1024}") + echo " $_c → ${_ceil_gb}GB" + done + else + echo " none configured" + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Appdata Size Monitoring ━━━ +# ============================================================================================== +[[ "$WATCHDOG_CHECK_APPDATA" != "true" ]] && exit 0 + +log "$ICON_GEAR Storage watchdog — $MY_ID — $(date '+%H:%M:%S')" + +WARNINGS=0 + +_STRIKE_LIMIT=${WATCHDOG_APPDATA_STRIKE_LIMIT:-3} +_GROWTH_MB=$(( ${WATCHDOG_APPDATA_GROWTH_GB:-2} * 1024 )) +_LOG_KB=$(( ${WATCHDOG_APPDATA_LOG_MAX_GB:-2} * 1024 * 1024 )) + +# Tracks files handled by Part 1 to prevent duplicate alerts in Part 2 +declare -A _HANDLED=() + +for _appdata_path in "${WATCHDOG_APPDATA_PATHS[@]:-}"; do + [[ -z "$_appdata_path" || ! -d "$_appdata_path" ]] && continue + + # ── Part 1: Growth rate scan ────────────────────────────────────────────────────────────── + declare -A _PREV=() + if [[ -f "$WATCHDOG_APPDATA_GROWTH_FILE" ]]; then + while IFS='|' read -r _cn _cs _; do + [[ -n "$_cn" ]] && _PREV["$_cn"]="$_cs" + done < "$WATCHDOG_APPDATA_GROWTH_FILE" + fi + + _growth_tmp=$(mktemp 2>/dev/null) || _growth_tmp="" + _now=$(date +%s) + + while IFS= read -r _du_line; do + _curr_mb=$(echo "$_du_line" | awk '{print $1}') + _cdir=$(echo "$_du_line" | awk '{print $2}') + _cname=$(basename "$_cdir") + [[ -z "$_cname" || "$_cname" == "*" ]] && continue + + [[ -n "$_growth_tmp" ]] && echo "${_cname}|${_curr_mb}|${_now}" >> "$_growth_tmp" + + _prev_mb="${_PREV[$_cname]:-}" + [[ -z "$_prev_mb" ]] && continue # First run after boot — building baseline + + _growth_mb=$(( _curr_mb - _prev_mb )) + _safe=$(echo "$_cname" | tr -cd '[:alnum:]_') + + # Condition resolved — growth stopped, clear strikes + if [[ "$_growth_mb" -le 0 ]]; then + _existing=$(get_strikes "appdata_growth_${_safe}" "$STORAGE_WATCHDOG_STATE_FILE") + _existing="${_existing//[^0-9]/}" + [[ "${_existing:-0}" -gt 0 ]] && \ + set_strikes "appdata_growth_${_safe}" 0 "$STORAGE_WATCHDOG_STATE_FILE" + continue + fi + + # Check suppress ceiling + _ceiling="${WATCHDOG_APPDATA_SIZES[$_cname]:-}" + if [[ -n "$_ceiling" && "$_curr_mb" -lt "$_ceiling" ]]; then + log "$_cname — growth suppressed (${_curr_mb}MB < ${_ceiling}MB ceiling)" + continue + fi + + # Growth exceeds threshold — strike logic + if [[ "$_growth_mb" -ge "$_GROWTH_MB" ]]; then + _growth_gb=$(awk "BEGIN {printf \"%.1f\", $_growth_mb / 1024}") + _curr_gb=$(awk "BEGIN {printf \"%.1f\", $_curr_mb / 1024}") + + _strikes=$(get_strikes "appdata_growth_${_safe}" "$STORAGE_WATCHDOG_STATE_FILE") + _strikes="${_strikes//[^0-9]/}"; _strikes="${_strikes:-0}" + + [[ "$_strikes" -lt "$_STRIKE_LIMIT" ]] && { + _strikes=$(( _strikes + 1 )) + set_strikes "appdata_growth_${_safe}" "$_strikes" "$STORAGE_WATCHDOG_STATE_FILE" + } + + warn "$_cname — grew ${_growth_gb}GB this cycle (total: ${_curr_gb}GB) [strike ${_strikes}/${_STRIKE_LIMIT}]" + (( WARNINGS++ )) + + # Focused *.log scan inside the growing container's dir + _found_logs=() + while IFS= read -r _lf; do + [[ -n "$_lf" ]] && _found_logs+=("$_lf") + done < <(find "$_cdir" -maxdepth 3 -type f \ + \( -name "*.log" -o -name "*.log.*" \) \ + -size +${_LOG_KB}k -printf "%s %p\n" 2>/dev/null) + + _log_summary="" + [[ ${#_found_logs[@]} -gt 0 ]] && _log_summary=$(printf '%s\n' "${_found_logs[@]}" | \ + awk '{printf "%.1fGB %s | ", $1/1073741824, $2}' | head -c 200) + + if [[ "$_strikes" -lt "$_STRIKE_LIMIT" ]]; then + if [[ -n "$_log_summary" ]]; then + notify "$_cname grew ${_growth_gb}GB on $(hostname) [strike ${_strikes}/${_STRIKE_LIMIT}] — logs: ${_log_summary}" \ + "Storage Watchdog" "warning" + else + notify "$_cname grew ${_growth_gb}GB on $(hostname) [strike ${_strikes}/${_STRIKE_LIMIT}] — data growth, no log files" \ + "Storage Watchdog" "warning" + fi + else + # Action cycle + error "$_cname — growth strike limit reached (${_STRIKE_LIMIT} consecutive cycles, ${_growth_gb}GB this cycle)" + if [[ "$WATCHDOG_APPDATA_TRUNCATE_LOGS" == "true" && ${#_found_logs[@]} -gt 0 ]]; then + for _lf_entry in "${_found_logs[@]}"; do + _lf_path=$(echo "$_lf_entry" | cut -d' ' -f2-) + _lf_gb=$(echo "$_lf_entry" | awk '{printf "%.1f", $1/1073741824}') + _HANDLED["$_lf_path"]=1 + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would truncate $_lf_path (${_lf_gb}GB)" + else + if truncate -s 0 "$_lf_path" 2>/dev/null; then + success "Truncated runaway log: $_lf_path (was ${_lf_gb}GB)" + notify "Truncated runaway log on $(hostname): $_lf_path (was ${_lf_gb}GB)" \ + "Storage Watchdog" "warning" + else + warn "Failed to truncate $_lf_path" + fi + fi + done + set_strikes "appdata_growth_${_safe}" 0 "$STORAGE_WATCHDOG_STATE_FILE" + else + if [[ -n "$_log_summary" ]]; then + notify "$_cname appdata runaway on $(hostname) — ${_growth_gb}GB growth for ${_STRIKE_LIMIT} cycles — logs: ${_log_summary}" \ + "Storage Watchdog" "critical" + else + notify "$_cname appdata runaway on $(hostname) — ${_growth_gb}GB growth for ${_STRIKE_LIMIT} cycles — data growth, manual investigation needed" \ + "Storage Watchdog" "critical" + fi + fi + fi + + # Mark found logs as handled to suppress Part 2 duplicates this cycle + for _lf_entry in "${_found_logs[@]}"; do + _HANDLED["$(echo "$_lf_entry" | cut -d' ' -f2-)"]=1 + done + fi + done < <(du -sm "$_appdata_path"/*/ 2>/dev/null) + + # Atomically update growth baseline + [[ -n "$_growth_tmp" ]] && mv "$_growth_tmp" "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null + + # ── Part 2: Absolute log size scan ──────────────────────────────────────────────────────── + # Catches *.log files already large but no longer actively growing this cycle. + # Same strike logic. Skips files already handled by Part 1 above. + declare -A _LOG_SEEN=() + + while IFS= read -r _hit; do + [[ -z "$_hit" ]] && continue + _fpath=$(echo "$_hit" | cut -d' ' -f2-) + [[ -n "${_HANDLED[$_fpath]:-}" ]] && continue + + _fsize_bytes=$(echo "$_hit" | awk '{print $1}') + _fsize_gb=$(awk "BEGIN {printf \"%.1f\", $_fsize_bytes / 1073741824}") + _safe_fkey=$(echo "$_fpath" | tr -cd '[:alnum:]_' | cut -c1-120) + _LOG_SEEN["appdata_log_${_safe_fkey}"]=1 + + _strikes=$(get_strikes "appdata_log_${_safe_fkey}" "$STORAGE_WATCHDOG_STATE_FILE") + _strikes="${_strikes//[^0-9]/}"; _strikes="${_strikes:-0}" + + [[ "$_strikes" -lt "$_STRIKE_LIMIT" ]] && { + _strikes=$(( _strikes + 1 )) + set_strikes "appdata_log_${_safe_fkey}" "$_strikes" "$STORAGE_WATCHDOG_STATE_FILE" + } + + warn "Oversized log: $_fpath (${_fsize_gb}GB) [strike ${_strikes}/${_STRIKE_LIMIT}]" + (( WARNINGS++ )) + + if [[ "$_strikes" -lt "$_STRIKE_LIMIT" ]]; then + notify "Oversized log on $(hostname): $_fpath (${_fsize_gb}GB) [strike ${_strikes}/${_STRIKE_LIMIT}]" \ + "Storage Watchdog" "warning" + else + error "Oversized log persists for ${_STRIKE_LIMIT} cycles: $_fpath (${_fsize_gb}GB)" + if [[ "$WATCHDOG_APPDATA_TRUNCATE_LOGS" == "true" ]]; then + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would truncate $_fpath (${_fsize_gb}GB)" + else + if truncate -s 0 "$_fpath" 2>/dev/null; then + success "Truncated oversized log: $_fpath (was ${_fsize_gb}GB)" + notify "Truncated oversized log on $(hostname): $_fpath (was ${_fsize_gb}GB)" \ + "Storage Watchdog" "warning" + set_strikes "appdata_log_${_safe_fkey}" 0 "$STORAGE_WATCHDOG_STATE_FILE" + unset "_LOG_SEEN[appdata_log_${_safe_fkey}]" + else + warn "Failed to truncate $_fpath" + notify "Oversized log on $(hostname): $_fpath (${_fsize_gb}GB) — ${_STRIKE_LIMIT} cycles, truncate failed" \ + "Storage Watchdog" "critical" + fi + fi + else + notify "Oversized log on $(hostname): $_fpath (${_fsize_gb}GB) — ${_STRIKE_LIMIT} cycles, manual intervention needed" \ + "Storage Watchdog" "critical" + fi + fi + done < <(find "$_appdata_path" -maxdepth 4 -type f \ + \( -name "*.log" -o -name "*.log.*" \) \ + -size +${_LOG_KB}k -printf "%s %p\n" 2>/dev/null) + + # Auto-clear strikes for log files no longer oversized this cycle + while IFS=':' read -r _sk _sv; do + [[ "$_sk" != appdata_log_* ]] && continue + _sv_clean="${_sv//[^0-9]/}" + [[ "${_sv_clean:-0}" -eq 0 ]] && continue + [[ -n "${_LOG_SEEN[$_sk]:-}" ]] && continue + set_strikes "$_sk" 0 "$STORAGE_WATCHDOG_STATE_FILE" + done < "$STORAGE_WATCHDOG_STATE_FILE" 2>/dev/null + + unset _LOG_SEEN + +done + +unset _PREV _HANDLED + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== +if [[ "$WARNINGS" -gt 0 ]]; then + echo "" + echo "━━━ $ICON_GEAR Storage Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + echo "$ICON_WATCHDOG Warnings: $WARNINGS" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +else + log "Storage healthy ✅ ($(date '+%H:%M:%S'))" +fi diff --git a/unRAID_Essentials/system_watchdog.sh b/Watchdogs/system_watchdog.sh similarity index 100% rename from unRAID_Essentials/system_watchdog.sh rename to Watchdogs/system_watchdog.sh diff --git a/Orchestrators/watchdog_orchestrator.sh b/Watchdogs/watchdog_orchestrator.sh similarity index 100% rename from Orchestrators/watchdog_orchestrator.sh rename to Watchdogs/watchdog_orchestrator.sh diff --git a/Tools/watchdog_skip_list_manager.sh b/Watchdogs/watchdog_skip_list_manager.sh similarity index 100% rename from Tools/watchdog_skip_list_manager.sh rename to Watchdogs/watchdog_skip_list_manager.sh diff --git a/common.sh b/common.sh index 72773af..af6da45 100644 --- a/common.sh +++ b/common.sh @@ -55,11 +55,11 @@ # check_local_disk_temps() — pre-rsync temp check with exit codes 0/1/2 # stop/start local containers added alongside existing remote variants # -# v3.1 Three-file config split — master.conf + master_host1.conf + master_host2.conf -# load_config.sh introduced — auto-discovers all master_host*.conf files +# v3.1 Three-file config split — master.conf + host1.conf + host2.conf +# load_config.sh introduced — auto-discovers all host*.conf files # detect_hosts() rewritten — sets MY_ID/REMOTE_ID and aliases all HOST* vars # All scripts now source load_config.sh instead of conf files directly -# Adding a new server = add master_host*.conf, zero script changes required +# Adding a new server = add host*.conf, zero script changes required # # v3.2 check_remote_disks() rewritten — three-tier detection: # Tier 1: array disk paths (/mnt/disk*/sharename) @@ -78,10 +78,6 @@ # Emby immediately removes ghost entries — no user-facing file-not-found errors # Called by lidarr/sonarr/radarr_cleanup.sh when files are deleted # -# v3.4 Silent-by-default output model -# info() and success() now gated by SILENT_MODE — only warn/error always visible -# Exception: monitor scripts designed to produce output stay verbose -# Reduces notification spam — ecosystem only speaks when something is wrong # # Three new safety functions added: # check_unraid_version_parity() — refuses remote ops on version mismatch @@ -190,27 +186,16 @@ ICON_SUCCESS="✅" # ============================================================================================== # Standardised output functions used across all scripts. # -# Silent-by-default model: -# SILENT_MODE=true (default) — only warn() and error() produce output -# SILENT_MODE=false — all functions produce output -# --log flag — enables ENABLE_LOGGING (detailed [LOG] lines) -# -# Rules: -# error() — always visible — something broke -# warn() — always visible — something needs attention -# info() — silent by default — operational detail, visible when SILENT_MODE=false -# success() — silent by default — confirmation, visible when SILENT_MODE=false -# log() — debug detail — only when ENABLE_LOGGING=true -# -# Exception — monitor scripts are designed to produce output and set SILENT_MODE=false -# at the top of the script. All other scripts use the silent default. +# Two-tier model: +# echo — always visible — summaries, status conclusions, section headers +# warn() — always visible — state transitions, warnings, important events +# error() — always visible — something broke +# log() — only with --log flag — per-item detail, internal checks # # All output goes to stdout — callers can redirect as needed. -info() { [[ "${SILENT_MODE:-true}" == false ]] && echo "$ICON_INFO [INFO] $*"; return 0; } warn() { echo "$ICON_WARN [WARN] $*"; } error() { echo "$ICON_ERROR [ERROR] $*"; } -success() { [[ "${SILENT_MODE:-true}" == false ]] && echo "$ICON_SUCCESS [OK] $*"; return 0; } log() { [[ "${ENABLE_LOGGING:-false}" == true ]] && echo "[LOG] $*" @@ -223,7 +208,7 @@ log() { # Sends a notification via unRAID native system and/or Discord webhook. # Both channels are optional and independently controlled: # NOTIFY_UNRAID — shared toggle in master.conf -# MY_DISCORD_WEBHOOK — per-host in master_host*.conf, set by detect_hosts() +# MY_DISCORD_WEBHOOK — per-host in host*.conf, set by detect_hosts() # # Severity levels: normal, warning, alert # Usage: notify "message" "subject" "severity" @@ -254,7 +239,7 @@ notify() { -d "$payload" "$MY_DISCORD_WEBHOOK" >/dev/null 2>&1; then log "$ICON_NOTIFY Discord notification sent" else - warn "Discord notification failed — check HOST*_DISCORD_WEBHOOK in master_host*.conf" + warn "Discord notification failed — check HOST*_DISCORD_WEBHOOK in host*.conf" fi fi } @@ -363,7 +348,7 @@ validate_int() { # ── HOST DETECTION ──────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Determines which server is local and which is remote by comparing hostname against -# all HOST* values discovered from master_host*.conf files. +# all HOST* values discovered from host*.conf files. # # Sets: # MY_ID — "HOST1" or "HOST2" (the role key, not the hostname) @@ -384,6 +369,7 @@ validate_int() { # WATCHDOG_REQUIRED_CONTAINERS ← HOST*_WATCHDOG_REQUIRED_CONTAINERS # WATCHDOG_SCAN_IGNORE ← HOST*_WATCHDOG_SCAN_IGNORE # WATCHDOG_DEPENDENCIES ← HOST*_WATCHDOG_DEPENDENCIES (associative) +# WATCHDOG_APPDATA_SIZES ← HOST*_WATCHDOG_APPDATA_SIZES (associative) # NETWORK_CONNECT_CONTAINERS ← HOST*_NETWORK_CONNECT_CONTAINERS # NETWORK_CONNECT_NETWORKS ← HOST*_NETWORK_CONNECT_NETWORKS # MEDIA_PERMISSION_SHARES ← HOST*_MEDIA_PERMISSION_SHARES @@ -423,7 +409,7 @@ detect_hosts() { local host_var host_val # Scan all HOST* vars that hold a hostname value - # HOST1, HOST2, HOST3 etc. — all defined in master_host*.conf files + # HOST1, HOST2, HOST3 etc. — all defined in host*.conf files for host_var in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do host_val="${!host_var:-}" [[ -z "$host_val" ]] && continue @@ -435,7 +421,7 @@ detect_hosts() { if [[ -z "$MY_ID" ]]; then error "Unknown host: $local_hostname" - error "Hostname must match a HOST* value in master_host*.conf" + error "Hostname must match a HOST* value in host*.conf" error "Available: $(for h in HOST1 HOST2 HOST3 HOST4; do [[ -n "${!h:-}" ]] && echo -n "${!h} "; done)" exit 1 @@ -459,7 +445,7 @@ detect_hosts() { local ssh_key_var="${MY_ID}_SSH_KEY" SSH_KEY="${!ssh_key_var:-}" if [[ -z "$SSH_KEY" ]]; then - error "Missing SSH key: ${ssh_key_var} not set in master_host*.conf" + error "Missing SSH key: ${ssh_key_var} not set in host*.conf" exit 1 fi @@ -467,6 +453,11 @@ detect_hosts() { MY_DISCORD_WEBHOOK_VAR="${MY_ID}_DISCORD_WEBHOOK" MY_DISCORD_WEBHOOK="${!MY_DISCORD_WEBHOOK_VAR:-}" + OWNER_NAME_VAR="${MY_ID}_OWNER" + OWNER_NAME="${!OWNER_NAME_VAR:-}" + OWNER_EMAIL_VAR="${MY_ID}_OWNER_EMAIL" + OWNER_EMAIL="${!OWNER_EMAIL_VAR:-}" + EMBY_CONTAINER_VAR="${MY_ID}_EMBY_CONTAINER" EMBY_CONTAINER="${!EMBY_CONTAINER_VAR:-}" EMBY_URL_VAR="${MY_ID}_EMBY_URL" @@ -598,6 +589,7 @@ detect_hosts() { _alias_assoc "WATCHDOG_CONTAINERS" _alias_assoc "WATCHDOG_CONTAINER_URLS" _alias_assoc "WATCHDOG_DEPENDENCIES" + _alias_assoc "WATCHDOG_APPDATA_SIZES" # ── Output ──────────────────────────────────────────────────────────────── info "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" @@ -1779,7 +1771,7 @@ show_status() { echo "Remote ID: ${REMOTE_ID:-not set}" echo "unRAID ver: $local_ver" echo "Profile: ${PROFILE_NAME:-n/a}" - echo "Silent mode: ${SILENT_MODE:-true}" + echo "DryRun: ${DRY_RUN:-false}" echo "Logging: ${ENABLE_LOGGING:-false}" echo "SSH Key: ${SSH_KEY:-not set}" diff --git a/git_pull_execute.sh b/git_pull_execute.sh index b72bab6..6cb5a43 100755 --- a/git_pull_execute.sh +++ b/git_pull_execute.sh @@ -8,19 +8,19 @@ # ── WHAT THIS SCRIPT DOES ───────────────────────────────────────────────────────────────────── # 1. Detects which server it's running on via detect_hosts() (MY_ID) # 2. Configures sparse checkout to exclude other servers' credential files -# Each server only pulls its own master_host*.conf — never sees peer credentials +# Each server only pulls its own host*.conf — never sees peer credentials # 3. Pulls or clones latest scripts from Gitea # 4. Sets executable permissions on all .sh files # # ── SPARSE CHECKOUT ─────────────────────────────────────────────────────────────────────────── # Sparse checkout ensures each server only receives its own host conf: -# HOST1 pulls: master.conf + master_host1.conf + all scripts -# HOST1 skips: master_host2.conf, master_host3.conf etc. -# HOST2 pulls: master.conf + master_host2.conf + all scripts -# HOST2 skips: master_host1.conf, master_host3.conf etc. +# HOST1 pulls: master.conf + host1.conf + all scripts +# HOST1 skips: host2.conf, host3.conf etc. +# HOST2 pulls: master.conf + host2.conf + all scripts +# HOST2 skips: host1.conf, host3.conf etc. # # Adding a new server: -# Create master_host3.conf in the repo +# Create host3.conf in the repo # All existing servers automatically exclude it on next pull # New server gets only its own conf ✅ # @@ -125,7 +125,7 @@ fi # ============================================================================================== # ━━━ Sparse Checkout Configuration ━━━ # ============================================================================================== -# Build the list of master_host*.conf files that belong to OTHER servers. +# Build the list of host*.conf files that belong to OTHER servers. # This server pulls everything EXCEPT those files. # MY_ID is set by detect_hosts() — e.g. "HOST1" @@ -137,7 +137,7 @@ configure_sparse_checkout() { # Enable sparse checkout git -C "$repo_dir" config core.sparseCheckout true 2>/dev/null - # Build exclusion list — all master_host*.conf files except MY_ID's + # Build exclusion list — all host*.conf files except MY_ID's local sparse_file="$repo_dir/.git/info/sparse-checkout" mkdir -p "$(dirname "$sparse_file")" @@ -145,9 +145,9 @@ configure_sparse_checkout() { echo "/*" > "$sparse_file" # Exclude each other server's conf file - # Find all master_host*.conf files present in the repo + # Find all host*.conf files present in the repo local excluded=0 - for conf_file in "$repo_dir"/master_host*.conf; do + for conf_file in "$repo_dir"/host*.conf; do [[ -f "$conf_file" ]] || continue local conf_name conf_name=$(basename "$conf_file") @@ -193,7 +193,7 @@ SYNC_SUCCESS=false if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would sync $REPO_SSH → $TARGET_DIR" warn "DRY RUN — would configure sparse checkout for $MY_ID" - warn "DRY RUN — would exclude peer master_host*.conf files" + warn "DRY RUN — would exclude peer host*.conf files" SYNC_SUCCESS=true else mkdir -p "$TARGET_DIR" @@ -233,7 +233,7 @@ else echo " Clone successful" # Configure sparse checkout after clone - # Now all master_host*.conf files are present — can detect exclusions + # Now all host*.conf files are present — can detect exclusions configure_sparse_checkout "$TARGET_DIR" # Apply sparse checkout — removes excluded files from working tree @@ -267,7 +267,7 @@ echo "━━━━━ $ICON_SUMMARY GIT SYNC SUMMARY ━━━━━" echo "$ICON_NET Repo: $REPO_SSH" echo "$ICON_GEAR Target: $TARGET_DIR" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" -echo "$ICON_LOCK Excluded: peer master_host*.conf files" +echo "$ICON_LOCK Excluded: peer host*.conf files" echo "$ICON_TIME Duration: $(format_duration $((END - START)))" if [[ "$DRY_RUN" == true ]]; then echo "$ICON_WARN Status: DRY RUN — no changes made" diff --git a/master_host1.conf b/host1.conf similarity index 96% rename from master_host1.conf rename to host1.conf index 49533ac..c7efac2 100644 --- a/master_host1.conf +++ b/host1.conf @@ -10,7 +10,7 @@ # HOST2 never sees HOST1 credentials — clean separation at the file level. # # DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf. -# DO NOT put HOST2 variables here — they belong in master_host2.conf. +# DO NOT put HOST2 variables here — they belong in host2.conf. # # ── INDEX ───────────────────────────────────────────────────────────────────────────────────── # @@ -59,6 +59,12 @@ # RADARR URL, API key, path map # ARR RECOVERY per-arr recovery toggles # +# ── SYSTEM WATCHDOG ──────────────────────────────────────────────────────────────────────── +# SYSTEM WATCHDOG per-host check toggles and NIC configuration +# +# ── RESOURCE MANAGER ─────────────────────────────────────────────────────────────────────── +# RESOURCE MANAGER containers paused/stopped under memory pressure +# # ============================================================================================== # ============================================================================================== @@ -70,6 +76,8 @@ # SSH key used for all server-to-server operations — rsync, failover container commands. # Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys. HOST1_SSH_KEY="/root/.ssh/gmer4lfe_rsync_automation" + HOST1_OWNER="gmer4lfe" + HOST1_OWNER_EMAIL="gmer4lfe@gmail.com" # ━━━ Emby ━━━ # Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh, @@ -319,6 +327,17 @@ ["NextCloud"]="Postgres-NextCloud" ) +# Per-container appdata growth suppress ceilings in MB. +# ONLY needed in specific cases — growth rate detection covers all containers automatically. +# Use this when a container legitimately has large stable data and you want to guarantee +# it never triggers a false-positive growth alert. Growth warnings are suppressed while the +# container's dir stays below this ceiling; above it, warnings resume as normal. +# 50GB=51200 25GB=25600 20GB=20480 15GB=15360 10GB=10240 5GB=5120 + declare -A HOST1_WATCHDOG_APPDATA_SIZES=( + ["Tdarr"]="25600" # 25GB — transcode cache grows legitimately during active jobs + ["7dtd"]="20480" # 20GB — game server world data, expected to be large + ) + # ━━━ Docker Network Connect ━━━ # Containers connected to custom networks at array start by docker_network_connect.sh. # Networks created if they don't exist — idempotent, safe to re-run. @@ -354,7 +373,7 @@ # ━━━ Fallback Tiers — HOST1 Runs for HOST2 ━━━ # Containers HOST1 starts when HOST2 goes down. # Tier 1 is always immediate — vital services cannot wait. -# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in master_host2.conf). +# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in host2.conf). FALLBACK_HOST1_COVERS_HOST2_TIER1=( "Gmer4Lfe.us" "VaultWarden-Jayred365" diff --git a/master_host2.conf b/host2.conf similarity index 93% rename from master_host2.conf rename to host2.conf index 620381d..8c7f572 100644 --- a/master_host2.conf +++ b/host2.conf @@ -10,7 +10,7 @@ # HOST1 never sees HOST2 credentials — clean separation at the file level. # # DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf. -# DO NOT put HOST1 variables here — they belong in master_host1.conf. +# DO NOT put HOST1 variables here — they belong in host1.conf. # # ── STATUS ──────────────────────────────────────────────────────────────────────────────────── # HOST2 is currently being rebuilt — most sections scaffolded, fill in when back online. @@ -61,6 +61,12 @@ # RADARR URL, API key, path map # ARR RECOVERY per-arr recovery toggles (no Lidarr on HOST2) # +# ── SYSTEM WATCHDOG ──────────────────────────────────────────────────────────────────────── +# SYSTEM WATCHDOG per-host check toggles and NIC configuration +# +# ── RESOURCE MANAGER ─────────────────────────────────────────────────────────────────────── +# RESOURCE MANAGER containers paused/stopped under memory pressure +# # ============================================================================================== # ============================================================================================== @@ -72,6 +78,8 @@ # SSH key used for all server-to-server operations — rsync, failover container commands. # Must be in /root/.ssh/ and authorised in HOST1's /root/.ssh/authorized_keys. HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" + HOST2_OWNER="jayred365" + HOST2_OWNER_EMAIL="" # fill in when HOST2 is back online # ━━━ Emby ━━━ # Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh, @@ -284,6 +292,14 @@ # ["Authelia"]="Mariadb-Authelia Redis-Authelia" ) +# Per-container appdata growth suppress ceilings in MB. +# ONLY needed in specific cases — growth rate detection covers all containers automatically. +# Use when a container legitimately has large stable data and you want to suppress false-positive +# growth alerts. Add entries here only when a container triggers warnings it shouldn't. + declare -A HOST2_WATCHDOG_APPDATA_SIZES=( + # add HOST2 suppress entries here only as needed + ) + # ━━━ Docker Network Connect ━━━ # Containers connected to custom networks at array start by docker_network_connect.sh. # Networks created if they don't exist — idempotent, safe to re-run. @@ -318,7 +334,7 @@ # ━━━ Fallback Tiers — HOST2 Runs for HOST1 ━━━ # Containers HOST2 starts when HOST1 goes down. # Tier 1 is always immediate — vital services cannot wait. -# Higher tiers activate after HOST1_TIER*_DELAY minutes (set in master_host1.conf). +# Higher tiers activate after HOST1_TIER*_DELAY minutes (set in host1.conf). FALLBACK_HOST2_COVERS_HOST1_TIER1=( "Gmer4Lfe.com" "Emby" @@ -501,10 +517,6 @@ HOST2_RADARR_RECOVERY=true # HOST2_LIDARR_RECOVERY not set — Lidarr does not run on HOST2 -# ============================================================================================== -# ──────────────────────── End Of HOST2 Variables ────────────────────────────────────────────── -# ============================================================================================== - # ============================================================================================== # ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -597,6 +609,24 @@ # DISABLED — rebuild workloads may legitimately peg CPU. Enable after rebuild. HOST2_SYS_WATCHDOG_CHECK_RUNAWAY=false +# ============================================================================================== +# ── RESOURCE MANAGER ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Containers to manage under pressure — see master.conf RW_CRITICAL_CONTAINERS for exclusions. + +# docker pause at medium pressure (RAM < RW_RAM_MEDIUM_GB or load > medium threshold) +# Suspended in-place — instant to pause/unpause, no state lost, no restart delay. + HOST2_RW_PAUSE_CONTAINERS=( + # fill in when HOST2 is back online + ) + +# docker stop at hard pressure (RAM < RW_RAM_HARD_GB) +# Full stop — these are optional/heavy services that free significant RAM when stopped. +# resource_watchdog.sh restarts them when pressure fully clears (RAM >= RW_RAM_RECOVER_GB). + HOST2_RW_STOP_CONTAINERS=( + # fill in when HOST2 is back online + ) + # ============================================================================================== # ──────────────────────── End Of HOST2 Variables ────────────────────────────────────────────── # ============================================================================================== \ No newline at end of file diff --git a/load_config.sh b/load_config.sh index 2dfb53e..cfa2191 100644 --- a/load_config.sh +++ b/load_config.sh @@ -7,23 +7,23 @@ # # ── HOW IT WORKS ────────────────────────────────────────────────────────────────────────────── # 1. Sources master.conf (shared config — hostnames, thresholds, toggles, profiles, job lists) -# 2. Auto-discovers and sources all master_host*.conf files in the same directory +# 2. Auto-discovers and sources all host*.conf files in the same directory # Each host conf extends the shared profile arrays and adds host-specific credentials # 3. Sources common.sh (shared functions — detect_hosts, logging, notifications etc.) # # ── WHY THIS EXISTS ─────────────────────────────────────────────────────────────────────────── # Without this loader every script had to explicitly source each conf file: # source master.conf -# source master_host1.conf -# source master_host2.conf +# source host1.conf +# source host2.conf # source common.sh # # Adding a new server meant updating every script. -# With this loader — add master_host3.conf to the git repo and every server +# With this loader — add host3.conf to the git repo and every server # auto-discovers it on next git pull. Zero script changes required. Ever. # # ── ADDING A NEW SERVER ─────────────────────────────────────────────────────────────────────── -# 1. Create master_host3.conf following the same structure as HOST1/HOST2 +# 1. Create host3.conf following the same structure as HOST1/HOST2 # 2. Commit and push to git repo # 3. All servers pull it automatically — no other changes needed # @@ -40,9 +40,9 @@ # source "$SCRIPT_DIR/load_config.sh" # # ── SPARSE CHECKOUT NOTE ────────────────────────────────────────────────────────────────────── -# Sparse checkout controls which master_host*.conf files each server receives. -# HOST1 only pulls master_host1.conf — never HOST2's credentials. -# HOST2 only pulls master_host2.conf — never HOST1's credentials. +# Sparse checkout controls which host*.conf files each server receives. +# HOST1 only pulls host1.conf — never HOST2's credentials. +# HOST2 only pulls host2.conf — never HOST1's credentials. # This loader sources whatever conf files ARE present — sparse checkout handles the rest. # Both servers pull all non-credential conf files (master.conf, common.sh, load_config.sh). # @@ -63,14 +63,14 @@ fi source "$LOAD_CONFIG_DIR/master.conf" -# ━━━ Auto-discover and source all master_host*.conf files ━━━ +# ━━━ Auto-discover and source all host*.conf files ━━━ # Sorted for consistent load order — HOST1 before HOST2 before HOST3 etc. # Each host conf extends the shared PROFILE_* arrays and adds host-specific vars. # Missing files are silently skipped — sparse checkout intentionally withholds some. # At least one host conf must be present or the ecosystem has no identity to work with. _host_confs_loaded=0 - for _conf in $(ls "$LOAD_CONFIG_DIR"/master_host*.conf 2>/dev/null | sort); do + for _conf in $(ls "$LOAD_CONFIG_DIR"/host*.conf 2>/dev/null | sort); do if [[ -f "$_conf" ]]; then source "$_conf" (( _host_confs_loaded++ )) @@ -80,7 +80,7 @@ done if [[ "$_host_confs_loaded" -eq 0 ]]; then - echo "[FATAL] No master_host*.conf files found in $LOAD_CONFIG_DIR" >&2 + echo "[FATAL] No host*.conf files found in $LOAD_CONFIG_DIR" >&2 echo "[FATAL] At least one host conf required — check git pull and sparse checkout" >&2 exit 1 fi diff --git a/master.conf b/master.conf index cef95a6..56b7eea 100644 --- a/master.conf +++ b/master.conf @@ -8,10 +8,10 @@ # ── HOW THE THREE-FILE SYSTEM WORKS ────────────────────────────────────────────────────────── # Scripts source all three files at startup: # source master.conf ← shared config (this file) -# source master_host1.conf ← HOST1 credentials, shares, container lists -# source master_host2.conf ← HOST2 credentials, shares, container lists +# source host1.conf ← HOST1 credentials, shares, container lists +# source host2.conf ← HOST2 credentials, shares, container lists # -# Sparse checkout (git) ensures each server only pulls its own master_host*.conf. +# Sparse checkout (git) ensures each server only pulls its own host*.conf. # HOST2 never sees HOST1 credentials. HOST1 never sees HOST2 credentials. # # What belongs here: thresholds, toggles, intervals, profiles, job lists @@ -99,7 +99,7 @@ # Must match the exact unRAID hostname AND Tailscale device name (case sensitive). # detect_hosts() in common.sh matches the local hostname against these to set MY_ID / REMOTE_ID. # Tailscale IP resolution uses these names — no hardcoded IPs needed. -# To add a new server: add HOST3="unRAID-NewServer" here + create master_host3.conf. +# To add a new server: add HOST3="unRAID-NewServer" here + create host3.conf. HOST1="unRAID-Gmer4Lfe" HOST2="unRAID-Jayred365" @@ -145,7 +145,7 @@ # Manages the relationship lifecycle between two unRAID servers. # HOST1 is always the owner (source of truth) — HOST2 is always the mirror. # PARTNERSHIP_OWNER_HOST flips to "HOST2" after a --transfer operation. -# All identity vars (hostnames, SSH keys) live in master_host*.conf. +# All identity vars (hostnames, SSH keys) live in host*.conf. # Hostnames already match Tailscale device names — IP resolution is automatic. # # State files on /boot/config — survives reboots, available before array starts: @@ -164,13 +164,13 @@ PARTNERSHIP_ENABLED=false PARTNERSHIP_OWNER_HOST="HOST1" # "HOST1" or "HOST2" — flips on --transfer -# Auth containers reconfigured on onboard/offboard — defined per host in master_host*.conf. +# Auth containers reconfigured on onboard/offboard — defined per host in host*.conf. # Format: "ContainerName|WebUIPort" # HOST1_PARTNERSHIP_AUTH_WEBUIS / HOST2_PARTNERSHIP_AUTH_WEBUIS # On onboard → WebUI pointed at owner's Tailscale IP # On offboard → WebUI pointed back at localhost -# Paths to collect during the grace window after offboard — defined per host in master_host*.conf. +# Paths to collect during the grace window after offboard — defined per host in host*.conf. # HOST1_PARTNERSHIP_MIRROR_BACKUPS / HOST2_PARTNERSHIP_MIRROR_BACKUPS # Notified on offboard — no auto-deletion, manual collection. @@ -219,13 +219,6 @@ # ── LOGGING ─────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== -# Silent-by-default output model — ecosystem only speaks when something is wrong. -# true = only warn() and error() produce output (default — reduces notification spam) -# false = all output visible — use for monitor scripts or debugging -# Override per-run: script --log sets ENABLE_LOGGING=true for [LOG] detail -# Monitor scripts (coffee_report, health_digest etc.) set SILENT_MODE=false themselves - SILENT_MODE=true - # Controls verbose [LOG] output across all scripts. # true = show detailed [LOG] lines — useful for debugging or first-time setup # false = show only user-facing output — cleaner for scheduled runs @@ -239,7 +232,7 @@ # normal = job completed successfully / warning = something failed or needs attention NOTIFY_UNRAID=true -# Discord webhook URL — defined per host in master_host*.conf. +# Discord webhook URL — defined per host in host*.conf. # HOST1_DISCORD_WEBHOOK / HOST2_DISCORD_WEBHOOK # Allows different webhooks per server, or only one server notifying. @@ -284,7 +277,6 @@ # Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via # watchdog_orchestrator.sh — NOT launched here. ARRAY_START_SCRIPTS=( - "git_pull_execute.sh" # pull latest scripts before anything starts "Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts "unRAID_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill "unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning @@ -298,11 +290,12 @@ # Schedule: * * * * * (every minute) # NOT in ARRAY_START_SCRIPTS — has its own cron entry. # Order matters — resource first (frees pressure), docker second (heals with freed resources), -# system last (reboots only if prior layers failed). +# storage third (pool/data health — never reboots), system last (last line of defense). WATCHDOG_ORCHESTRATOR_SCRIPTS=( - "unRAID_Essentials/resource_watchdog.sh" # reduce system pressure before healing attempts - "Docker_Essentials/docker_watchdog.sh" # heal containers with freed resources - "unRAID_Essentials/system_watchdog.sh" # reboot if all else fails — last line of defense + "Watchdogs/resource_watchdog.sh" # reduce system pressure before healing attempts + "Watchdogs/docker_watchdog.sh" # heal containers with freed resources + "Watchdogs/storage_watchdog.sh" # pool and appdata health — alert and remediate + "Watchdogs/system_watchdog.sh" # reboot if all else fails — last line of defense ) WATCHDOG_ORCHESTRATOR_HEARTBEAT=true @@ -317,7 +310,7 @@ "Docker_Essentials/downloaders_reset.sh" # clear stuck download states every 15min ) -# Shares synced every 15 minutes — defined per host in master_host*.conf. +# Shares synced every 15 minutes — defined per host in host*.conf. # HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES # Format: "/path/to/share" or "/path/to/share|profile-name" # Order matters — Critical-Data first (auth stack), then Emby dirty sync. @@ -326,7 +319,7 @@ # intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch, # and optional mid-day rsync for any shares that need sub-daily propagation. # Schedule: 0 */4 * * * - # INTERMEDIATE_SYNC_SHARES is host-specific — configure HOST*_INTERMEDIATE_SYNC_SHARES in master_host*.conf. + # INTERMEDIATE_SYNC_SHARES is host-specific — configure HOST*_INTERMEDIATE_SYNC_SHARES in host*.conf. INTERMEDIATE_RSYNC_ENABLED=true # set false to disable mid-day rsync without removing shares INTERMEDIATE_MAINTENANCE_SCRIPTS=( @@ -346,7 +339,7 @@ "Media/media_cleaner.sh media" # remove junk from media shares #"Media/lidarr_cleanup.sh" # remove orphaned music files — enable when ready #"Media/sonarr_cleanup.sh" # remove orphaned TV files — enable when ready - #"Media/radarr_cleanup.sh" # remove orphaned movie files — enable when ready + "Media/radarr_cleanup.sh" # remove orphaned movie files "Media/lidarr_missing_art.sh" # fetch missing album/artist artwork (HOST1 only — self-guards) "Media/radarr_tmdb_removed.sh" # remove movies dropped from TMDb "Media/sonarr_tvdb_removed.sh" # remove series dropped from TVDB @@ -360,7 +353,7 @@ DAILY_CONTAINER_UPDATES=true # Media shares synced daily by daily_sync_maintenance.sh. -# Defined per-host in master_host*.conf — HOST1_DAILY_SYNC_SHARES and HOST2_DAILY_SYNC_SHARES. +# Defined per-host in host*.conf — HOST1_DAILY_SYNC_SHARES and HOST2_DAILY_SYNC_SHARES. # Mesh model: every node pushes every media share. rsync has no --delete so pushes are additive. # arr_sync (union) ensures all arr libraries converge first. arr_cleanup removes true orphans. # Any node can download content to any share — it propagates to all nodes on the next cycle. @@ -368,7 +361,7 @@ # These shares use DEFAULT_RSYNC_OPTS — no profile entry needed. # For shares needing custom options or container stops — create a profile in RSYNC section. -# Personal encrypted shares defined per-host in master_host*.conf. +# Personal encrypted shares defined per-host in host*.conf. # ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content. # See README-Rsync_Setup.md for ZFS encryption setup before uncommenting. @@ -390,7 +383,7 @@ # Set false to skip — docker_weekly_restart.sh still runs regardless. WEEKLY_REMAINING_UPDATES=true -# Shares synced during the weekly maintenance window — defined per host in master_host*.conf. +# Shares synced during the weekly maintenance window — defined per host in host*.conf. # HOST1_WEEKLY_SYNC_SHARES / HOST2_WEEKLY_SYNC_SHARES # Containers stopped both sides before sync — full clean state guaranteed. # Profiles drive container stops, excludes, and options — configure in RSYNC section. @@ -473,8 +466,8 @@ # called by weekly_sync_maintenance.sh — full clean sync weekly # critical-fallback — dirty sync — auth stays running both sides, WAL excluded # called by critical_sync_maintenance.sh every 15min -# host1-appdata — HOST1 server-specific appdata — defined in master_host1.conf -# host2-appdata — HOST2 server-specific appdata — defined in master_host2.conf +# host1-appdata — HOST1 server-specific appdata — defined in host1.conf +# host2-appdata — HOST2 server-specific appdata — defined in host2.conf # important-data — NextCloud + Postgres — NextCloud delayed start after Postgres # emby — weekly clean sync — both Emby stopped, full mirror # called by weekly_sync_maintenance.sh only — do NOT schedule separately @@ -591,7 +584,7 @@ # Failover → start remote DDNS first (Tier 1) # Handback → stop remote DDNS → rsync → start containers → start local DDNS last # -# Per-host container lists and tier delays live in master_host*.conf. +# Per-host container lists and tier delays live in host*.conf. # Shared settings (intervals, state file, thresholds) live here. EXTERNAL_IP="8.8.8.8" @@ -613,7 +606,7 @@ # ━━━ Downloaders Reset ━━━ # Runs every 15 minutes via CRITICAL_MAINTENANCE_SCRIPTS. # Clears stuck states, purges old history, prepares each download client for a clean cycle. -# Per-host URLs and API keys live in master_host*.conf. +# Per-host URLs and API keys live in host*.conf. DOWNLOADER_RETENTION_DAYS=7 # days — purge history older than this # qBittorrent failsafe — removes torrents older than threshold regardless of ratio @@ -623,7 +616,7 @@ # ━━━ Docker Daily Restart ━━━ # Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS. -# Per-host lists live in master_host*.conf: +# Per-host lists live in host*.conf: # HOST1_DAILY_RESTART_CONTAINERS # HOST2_DAILY_RESTART_CONTAINERS # detect_hosts() sets DAILY_RESTART_CONTAINERS to the correct host array at runtime. @@ -631,7 +624,7 @@ # ━━━ Docker Weekly Restart ━━━ # Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am). # Containers already stopped for weekly sync — restart adds zero extra downtime. -# Per-host lists live in master_host*.conf: +# Per-host lists live in host*.conf: # HOST1_WEEKLY_RESTART_CONTAINERS # HOST2_WEEKLY_RESTART_CONTAINERS @@ -653,7 +646,7 @@ # Dead containers — remove and restart # Unexpected exits — non-zero exit code → restart # -# All per-host container lists live in master_host*.conf: +# All per-host container lists live in host*.conf: # HOST*_WATCHDOG_CONTAINERS — memory hard limits per container # HOST*_WATCHDOG_CONTAINER_URLS — HTTP health check URLs # HOST*_WATCHDOG_REQUIRED_CONTAINERS — must always be running @@ -706,10 +699,42 @@ # Notification batching — one summary per cycle instead of one ping per event WATCHDOG_BATCH_NOTIFY=true +# Appdata size monitoring — two-part catch-all for runaway growth and oversized log files. +# +# Part 1 — Growth rate (zero-config): +# Reads per-container dir totals each cycle via du, compares to previous cycle. +# Any container growing more than WATCHDOG_APPDATA_GROWTH_GB triggers a focused *.log scan +# inside that container. No per-container config required — new containers are covered +# automatically. Baseline built on first run after boot; growth detection starts cycle 2. +# +# Part 2 — Absolute log size: +# Finds *.log / *.log.* files over WATCHDOG_APPDATA_LOG_MAX_GB across all appdata paths. +# Catches logs that have already stabilised at a large size and are no longer actively growing. +# +# Strike system (reuses existing watchdog infrastructure): +# Strike 1 — warn + notify: condition first detected +# Strike 2 — warn + escalated notify: still present next cycle +# Strike 3 (WATCHDOG_APPDATA_STRIKE_LIMIT) — action cycle: +# If WATCHDOG_APPDATA_TRUNCATE_LOGS=true: truncate *.log files in-place, clear strikes +# If false: critical notify only, strikes held until condition resolves +# Condition resolves (growth stops / log drops below threshold) → strikes auto-clear +# +# HOST*_WATCHDOG_APPDATA_SIZES (in host*.conf) suppresses growth warnings for a +# container until its dir exceeds the configured ceiling. Only needed when a container +# legitimately has large stable data and you want to guarantee it never triggers a false alarm. + WATCHDOG_CHECK_APPDATA=true + WATCHDOG_APPDATA_PATHS=("/mnt/docker-unraid/appdata") + WATCHDOG_APPDATA_GROWTH_GB=2 # flag containers growing more than this per cycle + WATCHDOG_APPDATA_LOG_MAX_GB=2 # flag *.log files exceeding this size (absolute) + WATCHDOG_APPDATA_TRUNCATE_LOGS=false # set true to auto-truncate oversized *.log files on action cycle + WATCHDOG_APPDATA_STRIKE_LIMIT=3 # cycles before action fires (matches existing watchdog pattern) + WATCHDOG_APPDATA_GROWTH_FILE="/tmp/watchdog_appdata_growth.db" # /tmp — resets on reboot ✅ + STORAGE_WATCHDOG_STATE_FILE="/tmp/storage_watchdog_state.db" # /tmp — resets on reboot ✅ + # ━━━ Docker Network Connect ━━━ # Ensures custom networks exist and connects containers at array start. # Runs once via ARRAY_START_SCRIPTS — idempotent, safe to re-run. -# Container and network lists are host-specific — defined in master_host*.conf: +# Container and network lists are host-specific — defined in host*.conf: # HOST1_NETWORK_CONNECT_CONTAINERS / HOST2_NETWORK_CONNECT_CONTAINERS # HOST1_NETWORK_CONNECT_NETWORKS / HOST2_NETWORK_CONNECT_NETWORKS @@ -816,13 +841,13 @@ PERMISSIONS_FILE_MODE="664" # files — group read/write, no execute PERMISSIONS_OWNER="nobody:users" -# Share list defined per host in master_host*.conf: +# Share list defined per host in host*.conf: # HOST1_MEDIA_PERMISSION_SHARES / HOST2_MEDIA_PERMISSION_SHARES # ━━━ Media Cleaner ━━━ # Removes junk files from media shares — two profiles: anime and media. # Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media -# Folder lists defined per host in master_host*.conf: +# Folder lists defined per host in host*.conf: # HOST1_ANIME_CLEAN_FOLDERS / HOST2_ANIME_CLEAN_FOLDERS # HOST1_MEDIA_CLEAN_FOLDERS / HOST2_MEDIA_CLEAN_FOLDERS @@ -904,7 +929,7 @@ # ━━━ Arr Cleanup ━━━ # Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs. -# Per-host URLs, API keys, and path maps live in master_host*.conf. +# Per-host URLs, API keys, and path maps live in host*.conf. # detect_hosts() selects correct host vars at runtime. # # API versions — update MAJOR version here when script is updated for a new arr version: @@ -940,7 +965,7 @@ LIDARR_ART_MAX_PARALLEL=4 # concurrent background download jobs LIDARR_ART_RETRIES=2 # download retry attempts per image LIDARR_ART_SLEEP_BETWEEN=0.2 # seconds between fanart.tv API calls -# HOST*_FANART_API_KEY / HOST*_LASTFM_API_KEY — set in master_host*.conf +# HOST*_FANART_API_KEY / HOST*_LASTFM_API_KEY — set in host*.conf # Lidarr discovery settings (playback_aware_lidarr_discovery.sh) LIDARR_DISCOVERY_THRESHOLD=70 # score to accept candidate (0-100) @@ -1037,7 +1062,7 @@ # stalled — download stuck with no connections or progress # # Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first. -# Per-host recovery toggles (HOST1_SONARR_RECOVERY etc.) live in master_host*.conf. +# Per-host recovery toggles (HOST1_SONARR_RECOVERY etc.) live in host*.conf. ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this # matches cron interval — items eligible after one missed cycle @@ -1057,7 +1082,7 @@ # Must exist before Emby starts so the symlink resolves correctly. RAMDISK_PATH="/mnt/ramdisk_transcodes" -# Ramdisk size and flip thresholds — defined per host in master_host*.conf. +# Ramdisk size and flip thresholds — defined per host in host*.conf. # All three are coupled — if size changes, thresholds must change with it. # HOST1_RAMDISK_SIZE / HOST2_RAMDISK_SIZE # HOST1_RAMDISK_WARN_GB / HOST2_RAMDISK_WARN_GB ← flip to SSD at this usage @@ -1068,7 +1093,7 @@ # Must match the container path configured in Emby's Extra Parameters. TRANSCODE_LINK="/mnt/ram-transcode" -# SSD fallback location — defined per host in master_host*.conf (cache path differs per server): +# SSD fallback location — defined per host in host*.conf (cache path differs per server): # HOST1_TRANSCODE_SSD / HOST2_TRANSCODE_SSD # Minimum free GB on SSD before allowing flip from ramdisk to SSD. @@ -1098,7 +1123,7 @@ # Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex # Entries with placeholder API keys are skipped automatically. # ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk. -# Defined per host in master_host*.conf — Emby container names and keys differ per server: +# Defined per host in host*.conf — Emby container names and keys differ per server: # HOST1_TRANSCODE_SERVERS / HOST2_TRANSCODE_SERVERS # ============================================================================================== @@ -1108,7 +1133,7 @@ # ━━━ Certificate Monitor ━━━ # Checks SSL certificate expiry via direct openssl connection — no NPM dependency. # Checks the actual certificate served by each domain, not what NPM thinks it has. -# Domains defined per host in master_host*.conf — each server monitors its own domains: +# Domains defined per host in host*.conf — each server monitors its own domains: # HOST1_CERT_MONITOR_DOMAINS / HOST2_CERT_MONITOR_DOMAINS CERT_WARN_DAYS=30 # warn when cert expires within this many days CERT_CRIT_DAYS=7 # critical alert within this many days @@ -1117,7 +1142,7 @@ # ━━━ Backup Verify ━━━ # Verifies rsync mirror health by comparing random file checksums between servers. # Catches silent corruption or incomplete syncs that rsync itself wouldn't detect. -# Defined per host in master_host*.conf — leave empty to use HOST*_DAILY_SYNC_SHARES automatically: +# Defined per host in host*.conf — leave empty to use HOST*_DAILY_SYNC_SHARES automatically: # HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES BACKUP_VERIFY_SAMPLE=10 # random files to check per share BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample @@ -1128,7 +1153,7 @@ # (hot/max/hotssd/maxssd) — these vars are fallback only if dynamix.cfg not found. SMART_TEMP_WARN=45 # fallback — Celsius warn threshold SMART_TEMP_CRIT=55 # fallback — Celsius critical threshold -# Drives to ignore defined per host in master_host*.conf — hardware is server-specific: +# Drives to ignore defined per host in host*.conf — hardware is server-specific: # HOST1_SMART_IGNORE_DRIVES / HOST2_SMART_IGNORE_DRIVES # ━━━ ZFS Memory Snapshot ━━━ @@ -1138,7 +1163,7 @@ ZFS_REPORT_FREE_WARN_GB=10 # warn if less than this GB free RAM ZFS_REPORT_AVAIL_WARN_GB=20 # warn if less than this GB available on ZFS pool ZFS_REPORT_DOCKER_TOP=10 # how many top Docker containers to show by memory -# Pool ignore list defined per host in master_host*.conf — pool names are server-specific: +# Pool ignore list defined per host in host*.conf — pool names are server-specific: # HOST1_ZFS_REPORT_IGNORE_POOLS / HOST2_ZFS_REPORT_IGNORE_POOLS # ━━━ Bandwidth Monitor ━━━ @@ -1168,7 +1193,7 @@ # ━━━ Emby Session Report ━━━ # Weekly Emby usage statistics via API — no persistent writes, queries fresh each run. -# URL and API key pulled from HOST*_EMBY_URL and HOST*_EMBY_API_KEY in master_host*.conf. +# URL and API key pulled from HOST*_EMBY_URL and HOST*_EMBY_API_KEY in host*.conf. EMBY_REPORT_DAYS=7 # days to include in the report period EMBY_REPORT_TOP_N=10 # number of top content items to show @@ -1184,8 +1209,8 @@ # Level 3 (hard) — docker stop optional containers, signal docker_watchdog to defer # # ── PER-HOST CONTAINER LISTS ────────────────────────────────────────────────────────────────── -# HOST*_RW_PAUSE_CONTAINERS — docker pause at medium pressure (in master_host*.conf) -# HOST*_RW_STOP_CONTAINERS — docker stop at hard pressure (in master_host*.conf) +# HOST*_RW_PAUSE_CONTAINERS — docker pause at medium pressure (in host*.conf) +# HOST*_RW_STOP_CONTAINERS — docker stop at hard pressure (in host*.conf) RW_ENABLED=true RW_STATE_FILE="/tmp/resource_watchdog_state.db" @@ -1335,9 +1360,9 @@ SYS_WATCHDOG_MDSTAT_ERROR_LIMIT=5 # new errors in one cycle before acting # ━━━ Check Toggles ━━━ -# Per-host — moved to master_host*.conf +# Per-host — moved to host*.conf # Different servers may have different hardware, NICs, and check requirements -# See HOST*_SYS_WATCHDOG_CHECK_* in master_host*.conf +# See HOST*_SYS_WATCHDOG_CHECK_* in host*.conf # ━━━ Abort Toggles ━━━ # Conditions that prevent reboot even when a threshold is hit. diff --git a/unRAID_Essentials/Manual-Unraid_Essentials.md b/unRAID_Essentials/Manual-Unraid_Essentials.md index c95c54d..ca0543a 100644 --- a/unRAID_Essentials/Manual-Unraid_Essentials.md +++ b/unRAID_Essentials/Manual-Unraid_Essentials.md @@ -190,7 +190,7 @@ RW_RECOVER_CYCLES=3 # consecutive under-threshold runs before de-escalat ### Per-Host Container Lists ```bash -# master_host1.conf +# host1.conf HOST1_RW_PAUSE_CONTAINERS=("Tdarr" "HandBrake") # paused at medium pressure HOST1_RW_STOP_CONTAINERS=("LocalAI" "Satisfactory") # stopped at hard pressure ``` @@ -654,7 +654,7 @@ RW_QBIT_DL_MEDIUM=10240 RW_CRITICAL_CONTAINERS=("Emby" "NginxProxyManager" "Authelia" "Mariadb" "Redis") -# ── Per-Host (master_host*.conf) ─────────────────────────────────────────────── +# ── Per-Host (host*.conf) ─────────────────────────────────────────────── HOST1_RW_PAUSE_CONTAINERS=("Tdarr" "HandBrake") HOST1_RW_STOP_CONTAINERS=("LocalAI" "Satisfactory") HOST1_SABNZBD_URL="http://localhost:8080" diff --git a/user_script_plug-in.sh b/user_script_plug-in.sh index 6873be6..05d25c3 100644 --- a/user_script_plug-in.sh +++ b/user_script_plug-in.sh @@ -44,7 +44,7 @@ # ── PATHS ───────────────────────────────────────────────────────────────────────────────────── # Scripts: /mnt/user/appdata/unraid_scripts/ # Configuration: /mnt/user/appdata/unraid_scripts/master.conf -# Per-host: /mnt/user/appdata/unraid_scripts/master_host1.conf (or master_host2.conf) +# Per-host: /mnt/user/appdata/unraid_scripts/host1.conf (or host2.conf) # Library: /mnt/user/appdata/unraid_scripts/common.sh # # ── PLUGIN SETTINGS (apply to every entry) ──────────────────────────────────────────────────── @@ -168,9 +168,9 @@ # Overlap protection: acquire_lock() exits immediately if a prior cycle is still running — # prevents pile-up when a restart attempt or daemon check takes longer than 60 seconds. # -# bash /mnt/user/appdata/unraid_scripts/Orchestrators/watchdog_orchestrator.sh -# bash /mnt/user/appdata/unraid_scripts/Orchestrators/watchdog_orchestrator.sh --dry-run -# bash /mnt/user/appdata/unraid_scripts/Orchestrators/watchdog_orchestrator.sh --status +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/watchdog_orchestrator.sh +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/watchdog_orchestrator.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/watchdog_orchestrator.sh --status # ── TRANSCODE MANAGEMENT ────────────────────────────────────────────────────────────────────── @@ -570,7 +570,7 @@ # docker_watchdog.sh — two-tier container self-healing monitor (single-pass) # Called every minute by watchdog_orchestrator.sh — NOT started by array_started.sh. Single-pass. # -# Tier 1 — explicit per-container (configured in master_host*.conf): +# Tier 1 — explicit per-container (configured in host*.conf): # Memory hard limits: immediate restart when exceeded — no strikes, no waiting # CPU strike system: 2 consecutive cycles above HARD_CPU_THRESHOLD → restart # HTTP health checks: curl to configured URL — 2 consecutive failures → restart @@ -589,8 +589,8 @@ # RAM emergency: reads mem_shutdown_active from system_watchdog state file, defers all restarts # Silent on clean cycles — only outputs events and hourly heartbeat # -# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_watchdog.sh --status -# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_watchdog.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/docker_watchdog.sh --status +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/docker_watchdog.sh --dry-run # watchdog_skip_list_manager.sh — view and manage the container skip list # When docker_watchdog restarts the same container 3 times in 1hr → skip-listed. @@ -605,11 +605,11 @@ # docker start ContainerName → confirm fix works before handing back to watchdog # watchdog resumes normal monitoring on next cycle automatically # -# bash /mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --status -# bash /mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --clear ContainerName -# bash /mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --clear ContainerName --force -# bash /mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --clear-all -# bash /mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/watchdog_skip_list_manager.sh --status +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/watchdog_skip_list_manager.sh --clear ContainerName +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/watchdog_skip_list_manager.sh --clear ContainerName --force +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/watchdog_skip_list_manager.sh --clear-all +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/watchdog_skip_list_manager.sh --dry-run # docker_container_stop.sh — stop all running containers sequentially with verification # Called by array_stopping.sh as the final step in the planned shutdown sequence. @@ -631,7 +631,7 @@ # Running → docker restart (graceful). Stopped → left stopped (state respected). Missing → skip. # Dependency ordering via WATCHDOG_DEPENDENCIES — databases before applications. # Restart verification: checks container still up after settle period, notifies if not. -# Configured via HOST*_DAILY_RESTART_CONTAINERS (master_host*.conf): +# Configured via HOST*_DAILY_RESTART_CONTAINERS (host*.conf): # NginxProxyManager, Authelia, Dispatcharr, Dispatcharr-Basic, ErsatzTV-Emby # # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_daily_restart.sh --dry-run @@ -642,7 +642,7 @@ # Called by weekly_sync_maintenance.sh after sync completes and containers are back up. # Targets services that benefit from weekly clean start but don't stop for the sync itself. # Same rules as daily: running→restart, stopped→leave, missing→skip. -# Configured via HOST*_WEEKLY_RESTART_CONTAINERS (master_host*.conf): +# Configured via HOST*_WEEKLY_RESTART_CONTAINERS (host*.conf): # NextCloud, AdGuard-Home, Immich # # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh --dry-run @@ -711,7 +711,7 @@ # system_watchdog.sh — three-tier server stability last-resort watchdog (single-pass) # Called every minute by watchdog_orchestrator.sh — NOT started by array_started.sh. Single-pass. -# All 18 checks independently toggleable per host in master_host*.conf. +# All 18 checks independently toggleable per host in host*.conf. # # Tier 1 CRITICAL — bypass ALL strikes, reboot immediately: # Docker daemon hung → attempt rc.docker restart → still hung → reboot @@ -733,8 +733,8 @@ # Reboot loop protection: N reboots in X hours → shutdown instead. # State file heartbeat: writes watchdog_cycle=N every cycle (docker_watchdog stale guard). # -# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/system_watchdog.sh --status -# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/system_watchdog.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/system_watchdog.sh --status +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/system_watchdog.sh --dry-run # resource_watchdog.sh — three-level pressure reduction layer (single-pass, called by watchdog_orchestrator) # Reduces system load intelligently BEFORE docker_watchdog attempts container restarts. @@ -748,11 +748,11 @@ # Recovery: pressure must stay below current threshold for RW_RECOVER_CYCLES consecutive runs # before de-escalating. One level at a time — prevents flip-flopping. # Coordination: at Level 3 writes mem_shutdown_active=true → docker_watchdog defers all restarts. -# HOST*_RW_PAUSE_CONTAINERS and HOST*_RW_STOP_CONTAINERS configured per host in master_host*.conf. +# HOST*_RW_PAUSE_CONTAINERS and HOST*_RW_STOP_CONTAINERS configured per host in host*.conf. # -# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/resource_watchdog.sh --status -# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/resource_watchdog.sh --dry-run -# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/resource_watchdog.sh +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/resource_watchdog.sh --status +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/resource_watchdog.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/resource_watchdog.sh # inotify_tuning.sh — raise Linux inotify kernel limits at array start # Called by array_started.sh FIRST — must run before containers start (they inherit limits). @@ -858,7 +858,7 @@ # Run BEFORE arr cleanup — removes non-media files that would otherwise appear as orphans. # Patterns: *.sfv *.md5 *.sha1 *.nfo *.url *.lnk *.rar *.zip *.info *.torrent # *.sample* *.proof* *sync-conflict* *.scr *.exe *.srr *.log *.json -# Two profiles with separate folder lists (configured in master_host*.conf): +# Two profiles with separate folder lists (configured in host*.conf): # anime HOST*_ANIME_CLEAN_FOLDERS — anime share folders # media HOST*_MEDIA_CLEAN_FOLDERS — Movies, Tv_Shows, Music, Sports etc. # ALWAYS --dry-run when adding new patterns or folders — verify before committing. @@ -1080,7 +1080,7 @@ # Catches: renewed-but-not-reloaded (nginx not reloaded after certbot renewal), # wrong cert served, chain issues visible externally but not internally. # If a user would see a certificate error in their browser, this catches it first. -# Configured via HOST*_CERT_MONITOR_DOMAINS in master_host*.conf. +# Configured via HOST*_CERT_MONITOR_DOMAINS in host*.conf. # Thresholds: > 30 days = silent, <= 30 = warning, <= CERT_CRIT_DAYS (7) = urgent. # # bash /mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh --status @@ -1295,7 +1295,7 @@ # Orchestrators/array_stopping.sh (single entry — graceful shutdown) # # * * * * * every minute: -# Orchestrators/watchdog_orchestrator.sh +# Watchdogs/watchdog_orchestrator.sh # # */3 * * * * every 3 minutes: # Orchestrators/transcode_management.sh