diff --git a/user_script_plug-in.sh b/user_script_plug-in.sh index e6760ff..e465739 100644 --- a/user_script_plug-in.sh +++ b/user_script_plug-in.sh @@ -17,6 +17,7 @@ # array_start.sh at array start — launches ALL startup scripts in order # transcode_management.sh every 3 min — cleanup then manager (order critical) # critical_sync_maintenance.sh every 15 min — auth + Emby dirty sync + partnership +# intermediate_sync_maintenance.sh every 4 hours — arr library sync + artwork fetch # arrs_failed_stalled_recovery.sh every 6 hours — blocklist + re-search failed imports # daily_sync_maintenance.sh 1am daily — git + rsync + media + restart # rsync.sh --profile=emby-fallback every 30 min — Emby watch state dirty sync @@ -70,6 +71,15 @@ # individually below for standalone use. Inline descriptions on every entry. # Correct schedules verified against script headers. sunday_morning_coffee_report # and rsync_emby_failover added as separate scheduled entries. +# v2.3 — intermediate_sync_maintenance.sh new 4-hour orchestrator (arr_sync + artwork). +# docker_update.sh + docker_update_remaining.sh: daily/weekly image pulls. +# downloaders_reset.sh: slskd + SABnzbd + qBittorrent maintenance. +# arr_sync.sh: bidirectional Lidarr/Sonarr/Radarr library mesh. +# lidarr_missing_art.sh: fetch missing album/artist artwork. +# radarr_tmdb_removed.sh + sonarr_tvdb_removed.sh: dropped-entry cleanup. +# Initial_run/ section: ssh_setup.sh + partnership_onboard.sh. +# Moved: arrs_failed_stalled_recovery → Media/, fallback_state_reset + +# watchdog_skip_list_manager → Tools/. Renamed: partnership_manager.sh. # ============================================================================================== @@ -158,6 +168,31 @@ # bash /mnt/user/appdata/unraid_scripts/Orchestrators/critical_sync_maintenance.sh +# ── INTERMEDIATE SYNC MAINTENANCE ──────────────────────────────────────────────────────────── +# Schedule: 0 */4 * * * (every 4 hours) +# Background: YES +# +# Arr library reconciliation, artwork fetching, and optional mid-day rsync. +# Run on BOTH servers independently — each handles its own arr types. +# +# Execution order: +# 1. arr_sync.sh bidirectional Lidarr/Sonarr/Radarr mesh — all nodes agree +# Union model: if any node tracks it, all nodes get it. +# Remote API keys read via SSH on the fly — never stored. +# 2. Rsync window (optional) INTERMEDIATE_SYNC_SHARES — empty by default, skipped +# Add shares here if a subset needs mid-day propagation. +# 3. INTERMEDIATE_MAINTENANCE_SCRIPTS — lidarr_missing_art.sh and any future 4-hour jobs. +# lidarr_missing_art.sh is idempotent — skips existing files. +# +# Why separate from daily: arr libraries need to converge more than once a day. If a remote +# node adds something at 2am, waiting 23 hours means both arrs search for content already owned. +# 4-hour cycles close that gap. Artwork catches up within 4 hours of a new album landing. +# +# Silent on success — runs 6x/day, only failures produce notification. +# +# bash /mnt/user/appdata/unraid_scripts/Orchestrators/intermediate_sync_maintenance.sh + + # ── ARRS FAILED/STALLED RECOVERY ───────────────────────────────────────────────────────────── # Schedule: 0 */6 * * * (every 6 hours) # Background: YES @@ -174,7 +209,7 @@ # Never touches: items with status "downloading" or "imported" # Age gate: items newer than ARR_IMPORT_RECOVERY_AGE (6hr default) are skipped — arr may still retry. # -# bash /mnt/user/appdata/unraid_scripts/Orchestrators/arrs_failed_stalled_recovery.sh +# bash /mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh # ── DAILY SYNC MAINTENANCE ──────────────────────────────────────────────────────────────────── @@ -196,7 +231,8 @@ # 5. lidarr_cleanup.sh delete orphaned media — 7 safety checks before any deletion # sonarr_cleanup.sh notify_emby_scan() after each — ghost entries cleared immediately # radarr_cleanup.sh -# 6. docker_daily_restart.sh restart: NPM, Authelia, Dispatcharr, Dispatcharr-Basic, ErsatzTV-Emby +# 6. docker_update.sh pull latest images for DAILY_RESTART_CONTAINERS (no downtime) +# 7. docker_daily_restart.sh restart: NPM, Authelia, Dispatcharr, Dispatcharr-Basic, ErsatzTV-Emby # # Notify: silent on success. One failure notification with job list on any failure. # @@ -238,6 +274,7 @@ # 7. Start remote containers dependency order, new image, verify each container up # 8. Start local containers dependency order, new image, verify each container up # 9. docker_weekly_restart.sh restart less-critical services: NextCloud, AdGuard, Immich +# 10. docker_update_remaining.sh pull latest images for all containers not in daily/weekly lists # # Why weekly not nightly for Emby: HOST2 builds warm thumbnail cache throughout the week. # Nightly sync resets that cache — cold loads every morning. Weekly: cache warm 6 days, @@ -415,9 +452,9 @@ # Verify before resetting: right containers on right server, DDNS correct, # both servers Tailscale visible, fallback.sh not currently running. # -# bash /mnt/user/appdata/unraid_scripts/Fallback/fallback_state_reset.sh --status -# bash /mnt/user/appdata/unraid_scripts/Fallback/fallback_state_reset.sh -# bash /mnt/user/appdata/unraid_scripts/Fallback/fallback_state_reset.sh --force +# bash /mnt/user/appdata/unraid_scripts/Tools/fallback_state_reset.sh --status +# bash /mnt/user/appdata/unraid_scripts/Tools/fallback_state_reset.sh +# bash /mnt/user/appdata/unraid_scripts/Tools/fallback_state_reset.sh --force # ────────────────────────────────────────────────────────────────────────────────────────────── @@ -461,7 +498,7 @@ # Smart mode (default): kills rsync subprocess only, orchestrator exits cleanly, # any stopped containers are recovered by docker_watchdog.sh on next cycle. # --full-stop: kills orchestrator first then rsync — emergency full stop. -# --rsync-only: stop rsync, skip container recovery check (used by partnership_manage --offboard). +# --rsync-only: stop rsync, skip container recovery check (used by partnership_manager --offboard). # # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/rsync_stop.sh --status # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/rsync_stop.sh @@ -510,11 +547,11 @@ # docker start ContainerName → confirm fix works before handing back to watchdog # watchdog resumes normal monitoring on next cycle automatically # -# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/watchdog_skip_list_manager.sh --status -# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/watchdog_skip_list_manager.sh --clear ContainerName -# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/watchdog_skip_list_manager.sh --clear ContainerName --force -# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/watchdog_skip_list_manager.sh --clear-all -# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/watchdog_skip_list_manager.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --status +# bash /mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --clear ContainerName +# bash /mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --clear ContainerName --force +# bash /mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --clear-all +# bash /mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --dry-run # docker_daily_restart.sh — restart degradation-prone containers nightly # Called by daily_sync_maintenance.sh. Also safe to run ad hoc. @@ -550,6 +587,50 @@ # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh --dry-run # bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh +# docker_update.sh — pull latest images for daily restart containers, no downtime +# Called by daily_sync_maintenance.sh BEFORE docker_daily_restart.sh. +# Targets HOST*_DAILY_RESTART_CONTAINERS — same list as daily restart, no separate config. +# Containers stay running during the pull. docker_daily_restart.sh picks up the new image. +# Reports "updated" vs "already current" per container — useful to see what actually changed. +# Toggle: DAILY_CONTAINER_UPDATES=false in master.conf skips all pulls, restart still runs. +# +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_update.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_update.sh --status +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_update.sh + +# docker_update_remaining.sh — pull latest images for all containers not in daily/weekly lists +# Called by weekly_sync_maintenance.sh at the end of the Sunday window. +# Derives target set from `docker ps` minus DAILY_RESTART_CONTAINERS + WEEKLY_RESTART_CONTAINERS. +# Only running containers — stopped containers are excluded (likely paused intentionally). +# Together with docker_update.sh and the weekly image pull: every deployed container gets +# at least one image pull per week with no second list to maintain. +# Toggle: WEEKLY_REMAINING_UPDATES=false in master.conf skips all pulls. +# +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_update_remaining.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_update_remaining.sh --status +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_update_remaining.sh + +# downloaders_reset.sh — maintenance reset for all download clients on this server +# Called every 15 minutes by critical_sync_maintenance.sh via CRITICAL_MAINTENANCE_SCRIPTS. +# Host-aware: sections skip gracefully if a downloader URL is empty for this server. +# +# slskd: +# Stuck searches — clears Completed/Errored searches left by Soularr crashes +# Dead transfers — removes completed/errored/aborted transfer records per user +# NEVER removes InProgress or Queued transfers +# Failed imports — purges albums Soularr downloaded but Lidarr rejected +# SABnzbd: +# Completed history — removes completed records older than DOWNLOADER_RETENTION_DAYS +# Failed history — removes failed records older than DOWNLOADER_RETENTION_DAYS +# Stalled queue — removes Paused or Stuck queue items (never active downloads) +# qBittorrent: +# Age failsafe — removes torrents older than QBIT_FAILSAFE_MIN_DAYS +# deleteFiles=false — leaves files for arrs to manage +# +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/downloaders_reset.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/downloaders_reset.sh --status +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/downloaders_reset.sh + # ────────────────────────────────────────────────────────────────────────────────────────────── # UNRAID ESSENTIALS @@ -744,6 +825,70 @@ # bash /mnt/user/appdata/unraid_scripts/Media/radarr_cleanup.sh --status # bash /mnt/user/appdata/unraid_scripts/Media/radarr_cleanup.sh +# arr_sync.sh — bidirectional arr library sync across all nodes in the ecosystem +# Called by intermediate_sync_maintenance.sh every 4 hours. Can be run standalone. +# Union mesh: every node syncs with every other — no primary, no hierarchy. +# If any node tracks an item, all nodes get it (unless blocklisted). +# +# What syncs: library items keyed on stable IDs (MusicBrainz / TVDB / TMDB). +# Adding to a remote: that remote's own quality profile, metadata profile, and root +# folder are used — settings are never copied from the source node. +# Remote API keys read via SSH from each arr's config.xml — never stored anywhere. +# Key regeneration is picked up automatically on next run. +# Blocklist: ARR_SYNC_BLOCKLIST tombstones IDs that must never be re-added anywhere. +# --blocklist-add does three things: tombstones, deletes local, deletes all remotes via SSH. +# Files become orphans on all nodes — arr_cleanup.sh removes them on next run. +# +# bash /mnt/user/appdata/unraid_scripts/Media/arr_sync.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Media/arr_sync.sh --status +# bash /mnt/user/appdata/unraid_scripts/Media/arr_sync.sh +# bash /mnt/user/appdata/unraid_scripts/Media/arr_sync.sh --blocklist-add radarr "reason" +# bash /mnt/user/appdata/unraid_scripts/Media/arr_sync.sh --blocklist-add sonarr "reason" +# bash /mnt/user/appdata/unraid_scripts/Media/arr_sync.sh --blocklist-add lidarr "reason" +# bash /mnt/user/appdata/unraid_scripts/Media/arr_sync.sh --blocklist-remove radarr +# bash /mnt/user/appdata/unraid_scripts/Media/arr_sync.sh --blocklist-list + +# arrs_failed_stalled_recovery.sh — blocklist + re-search failed/stalled arr imports (see ORCHESTRATORS) +# bash /mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh --status + +# lidarr_missing_art.sh — fetch missing album and artist artwork for the Lidarr music library +# Called by intermediate_sync_maintenance.sh via INTERMEDIATE_MAINTENANCE_SCRIPTS. +# HOST1 only — exits cleanly on HOST2 (HOST2 does not run Lidarr). +# READS from Lidarr only — no writes back, never modifies audio tags or renames files. +# NEVER overwrites existing artwork — idempotent re-runs are safe. +# +# Artwork targets: +# Album: cover.jpg cdart.png back.jpg +# Artist: folder.jpg fanart.jpg logo.png banner.jpg +# Sources: fanart.tv → iTunes (album) / Deezer / Last.fm (artist) fallbacks +# +# bash /mnt/user/appdata/unraid_scripts/Media/lidarr_missing_art.sh --dry-run --log +# bash /mnt/user/appdata/unraid_scripts/Media/lidarr_missing_art.sh --status +# bash /mnt/user/appdata/unraid_scripts/Media/lidarr_missing_art.sh + +# radarr_tmdb_removed.sh — remove movies dropped from TMDb (status="deleted") +# These generate Radarr system health errors and can never be monitored or downloaded. +# 99% are future/announced movies that were delisted before release. +# Files NOT deleted by default — use --delete-files to also remove from disk. +# Import exclusion added by default — prevents Radarr re-adding the same dropped movie. +# HOST1 only — exits cleanly on HOST2. +# +# bash /mnt/user/appdata/unraid_scripts/Media/radarr_tmdb_removed.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Media/radarr_tmdb_removed.sh --status +# bash /mnt/user/appdata/unraid_scripts/Media/radarr_tmdb_removed.sh +# bash /mnt/user/appdata/unraid_scripts/Media/radarr_tmdb_removed.sh --delete-files + +# sonarr_tvdb_removed.sh — remove series dropped from TVDB (status="deleted") +# Same pattern as radarr_tmdb_removed: Sonarr health errors, can't be monitored. +# Files NOT deleted by default — use --delete-files to also remove from disk. +# Import exclusion added by default. HOST1 only. +# +# bash /mnt/user/appdata/unraid_scripts/Media/sonarr_tvdb_removed.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Media/sonarr_tvdb_removed.sh --status +# bash /mnt/user/appdata/unraid_scripts/Media/sonarr_tvdb_removed.sh +# bash /mnt/user/appdata/unraid_scripts/Media/sonarr_tvdb_removed.sh --delete-files + # ────────────────────────────────────────────────────────────────────────────────────────────── # TRANSCODES @@ -908,8 +1053,10 @@ # PARTNERSHIP # ────────────────────────────────────────────────────────────────────────────────────────────── -# partnership_manage.sh — two-server auth-sharing relationship lifecycle manager +# partnership_manager.sh — two-server auth-sharing relationship lifecycle manager # Automated --check runs inside critical_sync_maintenance.sh. Below are manual operation modes. +# Blocklist: after offboard, former partner hostname is written to partnership_blocklist.db. +# --onboard is hard-blocked if remote is on blocklist. --unblock clears it. # # --status: show both server state files (ACTIVE/INACTIVE), offline counter, owner server, # current WebUI redirect state on HOST2, whether critical sync is enabled. @@ -929,14 +1076,56 @@ # verifies both servers healthy before proceeding. Requires confirmation string: # "i-understand-this-transfers-ownership" (long by design — cannot type accidentally). # -# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --status -# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --onboard --dry-run -# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --onboard -# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --offboard --dry-run -# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --offboard -# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh --transfer --dry-run -# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manage.sh \ +# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manager.sh --status +# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manager.sh --onboard --dry-run +# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manager.sh --onboard +# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manager.sh --offboard --dry-run +# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manager.sh --offboard +# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manager.sh --transfer --dry-run +# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manager.sh \ # --transfer --confirm=i-understand-this-transfers-ownership +# bash /mnt/user/appdata/unraid_scripts/Partnership/partnership_manager.sh --unblock hostname + + +# ────────────────────────────────────────────────────────────────────────────────────────────── +# INITIAL RUN +# ────────────────────────────────────────────────────────────────────────────────────────────── +# +# Run ONCE when setting up a new server or joining a new partnership. Not scheduled. + +# ssh_setup.sh — generate the rsync SSH keypair and install it on the remote server +# Key named from hostname: unRAID-Gmer4Lfe → gmer4lfe_rsync_automation +# Idempotent — skips generation if key already exists. --force to regenerate and re-copy. +# +# --validate: health check mode. Called during partnership --check to detect broken SSH auth. +# Remote Tailscale IP unreachable = network issue → NOT a strike. +# Remote reachable but SSH auth fails = key issue → strike incremented. +# At SSH_MAX_STRIKES (5): notify + return exit 2 for caller to escalate. +# Strikes reset after SSH_STRIKE_RESET_HRS (24) of clean connectivity. +# +# bash /mnt/user/appdata/unraid_scripts/Initial_run/ssh_setup.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Initial_run/ssh_setup.sh --status +# bash /mnt/user/appdata/unraid_scripts/Initial_run/ssh_setup.sh +# bash /mnt/user/appdata/unraid_scripts/Initial_run/ssh_setup.sh --force +# bash /mnt/user/appdata/unraid_scripts/Initial_run/ssh_setup.sh --validate + +# partnership_onboard.sh — full partnership setup orchestrator (run once to join a partner server) +# Orchestrates the complete setup sequence in order: +# Step 1: ssh_setup.sh generate keypair, install on remote, update conf +# Step 2: partnership_manager.sh --onboard +# verify both servers, reconfigure auth WebUIs → owner IP, +# write ACTIVE state, create FolderView3 folder if enabled +# Step 3: arr_sync.sh bidirectional library bootstrap — both sides start with the +# merged library from day one (non-fatal: skipped if arrs not live) +# +# Pre-requisites: both servers on same Tailscale tailnet, remote has password auth for root, +# PARTNERSHIP_OWNER_HOST correct in master.conf. +# ALWAYS --dry-run first. +# +# bash /mnt/user/appdata/unraid_scripts/Initial_run/partnership_onboard.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Initial_run/partnership_onboard.sh +# bash /mnt/user/appdata/unraid_scripts/Initial_run/partnership_onboard.sh --skip-ssh +# bash /mnt/user/appdata/unraid_scripts/Initial_run/partnership_onboard.sh --skip-arr-sync # ────────────────────────────────────────────────────────────────────────────────────────────── @@ -1023,8 +1212,11 @@ # */30 * * * * every 30 minutes: # Rsync/rsync.sh /mnt/user/Media_Server/Emby --profile=emby-fallback # +# 0 */4 * * * every 4 hours: +# Orchestrators/intermediate_sync_maintenance.sh +# # 0 */6 * * * every 6 hours: -# Orchestrators/arrs_failed_stalled_recovery.sh +# Media/arrs_failed_stalled_recovery.sh # Monitors/system_tuning_monitor.sh # # 0 1 * * * 1am daily: