diff --git a/Docker_Essentials/README-Docker_Essentials.md b/Docker_Essentials/README-Docker_Essentials.md index ffd93d1..6e2f208 100644 --- a/Docker_Essentials/README-Docker_Essentials.md +++ b/Docker_Essentials/README-Docker_Essentials.md @@ -229,8 +229,7 @@ full coordination model between all four watchdogs. |--------|------|-------------| | `docker_daily_restart.sh` | Nightly proactive restart of degradation-prone containers | 1am via `daily_sync_maintenance.sh` | | `docker_weekly_restart.sh` | Weekly restart of less-critical services | 2:30am Sunday via `weekly_sync_maintenance.sh` | -| `docker_update.sh` | Container image updates — daily list + weekly remainder mode | Daily before restart; weekly end of window | -| `docker_update_remaining.sh` | Image update + prune for all containers not in managed lists | End of weekly maintenance window | +| `docker_update.sh` | Container image updates — daily list + weekly remainder mode | Daily before restart; weekly remainder pass | | `docker_network_connect.sh` | Network existence + container connection enforcement | Every array start | | `docker_container_stop.sh` | Ordered container shutdown — graceful then forced | Called by `array_stopping.sh` | | `downloaders_reset.sh` | Download client hygiene — slskd / SABnzbd / qBittorrent | Every 30min via `critical_sync_maintenance.sh` | @@ -259,8 +258,7 @@ Daily maintenance window (1am): Weekly maintenance window (2:30am Sunday): weekly_sync_maintenance.sh ├── docker_weekly_restart.sh ──── restart less-critical services - ├── docker_update.sh --remainder ─ update containers not in managed lists - └── docker_update_remaining.sh ── prune dangling images + └── docker_update.sh --remainder ─ update containers not in managed lists Critical maintenance window (every 30min): critical_sync_maintenance.sh diff --git a/Media/Manual-Media.md b/Media/Manual-Media.md index 4602040..6b4f607 100644 --- a/Media/Manual-Media.md +++ b/Media/Manual-Media.md @@ -297,14 +297,8 @@ HOST1_JELLYFIN_API_KEY="" ```bash # master.conf -WEBHOOK_PORT=9456 # 0 = disable listener +WEBHOOK_PORT=7821 # 0 = disable listener WEBHOOK_SECRET="" # auto-generated on first start if empty -WEBHOOK_UPGRADE_RSYNC_ENABLED=true # push upgraded file to remote nodes -``` - -```bash -# host*.conf -HOST1_WEBHOOK_LAN_IP="192.168.50.2" # IP arrs call for webhook delivery ``` When `WEBHOOK_PORT=0`, `start_webhook_listener.sh` exits cleanly and no listener starts. diff --git a/Orchestrators/README-Orchestrators.md b/Orchestrators/README-Orchestrators.md index 489b171..8e6b993 100644 --- a/Orchestrators/README-Orchestrators.md +++ b/Orchestrators/README-Orchestrators.md @@ -204,22 +204,28 @@ background process — and reports which succeeded and which failed. # ARRAY_START_SCRIPTS=( # ── One-shot scripts — run and exit naturally ───────────────────────────── - "System_Essentials/unraid_api_key_renew.sh" # re-register API key FIRST — unraid-api - # registry is ephemeral, lost on service restart - "System_Essentials/inotify_tuning.sh" # raise inotify BEFORE containers start - # containers inherit limits at startup — - # if Code-Server starts with low limits - # it keeps them until restart - "System_Essentials/docker_syslog_filter.sh" # suppress veth noise BEFORE containers create - # veth interfaces — otherwise the first boot - # always has unfiltered veth spam - "System_Essentials/php_fpm_max_children.sh" # WebGUI tuning — before any WebGUI requests + "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh" # re-register API key FIRST — + # registry is ephemeral, + # lost on service restart + "System_Essentials/conf_sync.sh" # pull conf from remote BEFORE anything + # needs current config + "System_Essentials/conf_cache_restore.sh" # restore cached conf if pull failed — + # ensures config is always available "Transcodes/ramdisk_setup.sh" # create tmpfs + symlink BEFORE Emby starts — # Emby needs the transcode path to exist + "System_Essentials/docker_syslog_filter.sh" # suppress veth noise BEFORE containers create + # veth interfaces — otherwise first boot has + # unfiltered veth spam + "Plugin/unraid/System_Essentials/php_fpm_max_children.sh" # WebGUI tuning — + # before any WebGUI requests + "System_Essentials/inotify_tuning.sh" # raise inotify BEFORE containers start — + # containers inherit limits at startup "Docker_Essentials/docker_network_connect.sh" # ensure networks + connections BEFORE # watchdogs check container states + "Tools/claude_startup.sh" # symlink Claude persistent storage on /boot # ── Continuous scripts — run until array stops ───────────────────────────── + "Media/start_webhook_listener.sh" # start webhook listener before arrs POST events "Fallback/fallback.sh" # fallback LAST — needs everything else stable ) diff --git a/README.md b/README.md index 3977cfa..d464343 100644 --- a/README.md +++ b/README.md @@ -585,11 +585,16 @@ The Varaverk scheduler runs only orchestrators — never individual scripts dire ``` Array start (Varaverk event hook): array_started.sh launches all startup scripts in order - → inotify_tuning.sh raise kernel limits before containers start + → unraid_api_key_renew.sh re-register API key before anything needs it + → conf_sync.sh pull conf from remote before scripts need current config + → conf_cache_restore.sh restore cached conf if remote pull failed + → ramdisk_setup.sh create ramdisk before Emby starts → docker_syslog_filter.sh suppress veth noise before containers create interfaces → php_fpm_max_children.sh WebGUI tuning before first request - → ramdisk_setup.sh create ramdisk before Emby starts + → inotify_tuning.sh raise kernel limits before containers start → docker_network_connect.sh connect containers to extra networks + → claude_startup.sh symlink Claude persistent storage (one-shot) + → start_webhook_listener.sh continuous — Node.js webhook server for arr upgrades → fallback.sh continuous — mutual fallback state machine Every 15 minutes: @@ -608,7 +613,7 @@ Every 30 minutes: Every 30 minutes: rsync.sh --profile=emby-fallback Emby watch state sync (separate cron entry) -Every 6 hours: +Every 4 hours: arrs_failed_stalled_recovery.sh blocklist + re-search failed imports 1am daily: diff --git a/Tools/Manual-Tools.md b/Tools/Manual-Tools.md index 9e70a83..a5d53a9 100644 --- a/Tools/Manual-Tools.md +++ b/Tools/Manual-Tools.md @@ -18,8 +18,9 @@ making any changes. - [container_data_export.sh](#container_data_exportsh) - [emby_database_repair.sh](#emby_database_repairsh) - [zfs_pool_scrub.sh](#zfs_pool_scrubsh) -- [recreate_shares.sh](#recreate_sharessh) -- [continuous_scripts_status.sh](#continuous_scripts_statussh) +- [smart_long_test.sh](#smart_long_testsh) +- [arr_profile_enforcer.sh](#arr_profile_enforcersh) +- [webhook_setup.sh](#webhook_setupsh) - [claude_startup.sh](#claude_startupsh) - [ramdisk_stop.sh](#ramdisk_stopsh) - [Adding a New Tool](#adding-a-new-tool) @@ -181,7 +182,7 @@ fallback_state_reset.sh --force # reset without confirmation prompt (for scr ```bash # Right containers on right server? -continuous_scripts_status.sh # shows fallback current state +cat "$STATE_DIR/fallback_state" # shows fallback current state # DDNS pointing correctly? nslookup Gmer4Lfe.com # confirm it resolves to the right IP @@ -508,86 +509,159 @@ zfs_pool_scrub.sh --log --- -## recreate_shares.sh +## smart_long_test.sh -Creates share directories on the correct disks after a fresh unRAID install or disk -rebuild. Run once on HOST2 before the first rsync from HOST1. +Runs a SMART extended (long) self-test on all drives sequentially and reports results. +Extended tests read every sector — they catch bad sectors and pre-failure reallocations +that the short test skips. Called monthly by `monthly_maintenance.sh`. ### When to Use ``` -After a fresh unRAID install where /boot/config/shares/*.cfg were restored: - The share definitions exist → UI shows shares → directories are missing on disk - rsync.sh tries to write to /mnt/user/Movies → path doesn't exist → aborts +Monthly via monthly_maintenance.sh — automatic, no manual trigger needed +After a disk replacement or rebuild — verify the new drive before it enters production +After a power cut or hard shutdown — check for newly reallocated sectors +When a drive shows elevated reallocated sectors in ZFS scrub or system logs +``` -After a disk replacement or rebuild where share folders were lost: - Replacement disk is blank → no share directories on the new disk - unRAID won't create them automatically +### Configuration (host*.conf) + +```bash +HOST1_SMART_IGNORE_DRIVES=("/dev/sdb") # boot USB — no useful SMART data +HOST2_SMART_IGNORE_DRIVES=() +``` + +Aliased by `detect_hosts()` → `SMART_IGNORE_DRIVES`. + +### Usage + +```bash +# Test all drives (skips SMART_IGNORE_DRIVES): +smart_long_test.sh + +# Test a specific drive — bypasses the ignore list: +smart_long_test.sh /dev/sda + +# Show last self-test result for all drives without starting a new test: +smart_long_test.sh --status + +# Show which drives would be tested, no tests started: +smart_long_test.sh --dry-run + +# Verbose — show poll progress every 60s per drive: +smart_long_test.sh --log +``` + +### What Gets Reported + +Each drive reports one of: `completed without error`, `completed with errors`, +`interrupted`, `in progress`. Any result other than `completed without error` +triggers a notification. Drives are tested one at a time — total runtime depends +on drive count and size (a full 8TB HDD takes roughly 90–180 minutes). + +--- + +## arr_profile_enforcer.sh + +Ensures every series in Sonarr and every movie in Radarr is on the correct quality +profile based on where it lives on disk. Only touches items with the wrong profile — +idempotent, safe to re-run. + +### Profile Rules + +``` +Root folder path contains "kids" or "anime" → ARR_KIDS_PROFILE_NAME +All other root folders → ARR_SONARR_DEFAULT_PROFILE (Sonarr) + ARR_RADARR_DEFAULT_PROFILE (Radarr) +``` + +Profile IDs are resolved from the API at runtime by name — no IDs need to be +hardcoded and the same script works across hosts. + +### When to Use + +``` +After initial arr setup — library was imported with a generic profile +After adding a new root folder — new imports may use the wrong default +After a profile rename — profile names in master.conf must match arr exactly +Any time arr is reporting downloads to the wrong quality level +``` + +### Configuration (master.conf) + +```bash +ARR_KIDS_PROFILE_NAME="Kids shows" # must exactly match the profile name in Sonarr/Radarr +ARR_SONARR_DEFAULT_PROFILE="Any" +ARR_RADARR_DEFAULT_PROFILE="Any (mine)" +``` + +### Usage + +```bash +# Dry run first — shows what would change without touching anything: +arr_profile_enforcer.sh --dry-run + +# Fix all Sonarr and Radarr libraries: +arr_profile_enforcer.sh + +# Sonarr only: +arr_profile_enforcer.sh --sonarr-only + +# Radarr only: +arr_profile_enforcer.sh --radarr-only +``` + +--- + +## webhook_setup.sh + +Registers the Varaverk upgrade webhook notification in Sonarr, Radarr, and Lidarr. +Idempotent — skips any arr that already has the webhook registered. Runs on both +hosts unless `--local-only` is passed. + +### When to Use + +``` +After initial Varaverk install +After adding a new arr to the stack +After adding a new host — both hosts need the webhook registered locally +After WEBHOOK_SECRET was regenerated — re-register to push the new secret ``` ### What It Does ``` -For each .cfg file in /boot/config/shares/: - 1. Read the share name (e.g., Movies) - 2. Read the shareInclude list (e.g., disk1,disk2,disk5) - 3. Create /mnt/disk1/Movies, /mnt/disk2/Movies, /mnt/disk5/Movies - 4. Place a .recovery marker in /mnt/user/Movies/ +1. Generates WEBHOOK_SECRET in master.conf if empty + (openssl rand -hex 32 → written to master.conf in place) -The .recovery marker tells rsync.sh this is a fresh share: - .recovery present → rsync WITHOUT --delete (safe — new files only, nothing removed) - .recovery absent → rsync WITH --delete (normal mirror mode) +2. Registers webhook in local Sonarr, Radarr, and Lidarr + POST to /api/v3/notification with the correct event triggers -Self-cleaning: after the first successful rsync, the source side has no .recovery file, -so the second nightly run deletes it from the mirror and normal --delete resumes. -No manual cleanup needed. +3. SSHes to remote host and runs itself --local-only + Both hosts register using the same secret +``` + +The webhook URL registered is `http://:/webhook?key=` — +uses the server's own LAN IP so arrs talk directly to the local listener. + +### Configuration (master.conf) + +```bash +WEBHOOK_PORT=7821 # port the listener runs on +WEBHOOK_SECRET="" # auto-generated on first run if empty ``` ### Usage ```bash -recreate_shares.sh # create all missing share directories + .recovery markers -recreate_shares.sh --dry-run # show what would be created without creating -recreate_shares.sh --log # verbose — show each directory created per disk -recreate_shares.sh --status # show share configs and current directory state -``` +# Register on both hosts (normal first-time setup): +webhook_setup.sh ---- +# Local host only (used internally when SSH'd from the other host): +webhook_setup.sh --local-only -## continuous_scripts_status.sh - -Live status dashboard for all continuously running scripts. Read-only — makes no -changes to any running process, container, or state file. - -### What It Shows - -``` -stability_watchdog - Running state, PID, uptime, approximate cycle count - Active strikes, recent restart history - Live snapshot: rootfs, RAM, ZFS ARC, load, zombie count, CPU temp - -docker_watchdog - Running state, PID, uptime - Running / stopped / unhealthy container counts - Required containers status - Memory-monitored containers - Recent restart history + skip list - -fallback (fallback.sh) - Current state (NORMAL / FALLBACK / HANDBACK) - Tier flags and timestamps - Remote Tailscale visibility -``` - -State files are read as-is — if a script is mid-cycle, the display reflects the last -completed cycle, not the current in-progress state. - -### Usage - -```bash -continuous_scripts_status.sh # show full dashboard -continuous_scripts_status.sh --log # verbose output with additional detail per section +# Show what would be registered without making changes: +webhook_setup.sh --dry-run ``` --- diff --git a/Tools/README-Tools.md b/Tools/README-Tools.md index d68df3e..c437e3c 100644 --- a/Tools/README-Tools.md +++ b/Tools/README-Tools.md @@ -49,13 +49,6 @@ inconsistent backup, and tar without stopping the container is equally unreliabl Fix: `container_data_export.sh` — stops the container cleanly, archives appdata to a timestamped `.tar.gz`, verifies archive integrity, restarts the container. -**Fresh HOST2 Has Shares Configured But Directories Missing** -Fresh install on HOST2. Restored `/boot/config/shares/` from backup. Array starts. -Shares show in the UI. But the actual `/mnt/diskN/sharename` directories don't exist — -unRAID created the share definitions but not the directories. rsync.sh aborts. -Fix: `recreate_shares.sh` — reads every `.cfg` file, creates directories on each -included disk, places `.recovery` markers so the first rsync won't delete anything. - --- ## ━━━ WHAT THIS FOLDER DOES ━━━ @@ -70,13 +63,13 @@ something new, write the tool. Store it here. Find it at 2am next time. `fallback_state_reset.sh`, `watchdog_skip_list_manager.sh` **Diagnostic Tools** — Inspect and verify before acting -`emby_database_repair.sh`, `continuous_scripts_status.sh` +`emby_database_repair.sh`, `zfs_pool_scrub.sh`, `smart_long_test.sh` **Repair Tools** — Fix a specific known problem -`bulk_permissions_repair.sh`, `zfs_pool_scrub.sh` +`bulk_permissions_repair.sh`, `arr_profile_enforcer.sh` **Lifecycle Tools** — Backup, setup, and migration support -`container_data_export.sh`, `recreate_shares.sh`, `claude_startup.sh`, `ramdisk_stop.sh` +`container_data_export.sh`, `claude_startup.sh`, `ramdisk_stop.sh`, `webhook_setup.sh` **Library Sync Bootstrap** — Close the gap between Emby and arr libraries `emby_to_lidarr_sync.sh`, `emby_to_sonarr_sync.sh`, `emby_to_radarr_sync.sh` @@ -102,9 +95,6 @@ Fallback/ Docker_Essentials/ docker_watchdog.sh ── writes skip list + history ──► watchdog_skip_list_manager.sh manages them - -Docker_Essentials/ + System_Essentials/ + Fallback/ - All continuous scripts ──────────────────────────► continuous_scripts_status.sh reads their state ``` Tools never call scripts in other folders. Other folders never call Tools scripts. @@ -122,11 +112,12 @@ The relationship is one-way: Tools act on state that other scripts have written. | `container_data_export.sh` | Need a clean backup before a risky container update or migration | Before major updates, appdata migrations, or container removals | | `emby_database_repair.sh` | Emby crashing with database errors after power loss or crash | When Emby logs show corruption or repeated crashes | | `zfs_pool_scrub.sh` | Verify ZFS pool integrity — catch silent corruption before it spreads | Monthly, or after any disk or power event | -| `recreate_shares.sh` | Share directories missing after fresh install or disk rebuild | After fresh unRAID install or disk replacement on HOST2 | -| `continuous_scripts_status.sh` | Need a live view of watchdog and fallback state | Any time — manual dashboard, no schedule | +| `smart_long_test.sh` | Run SMART extended self-test on all drives — full sector scan | Monthly via monthly_maintenance.sh, or after any disk event | | `claude_startup.sh` | Claude Code session setup after reboot — symlinks persistent storage | After each unRAID reboot, or called by array_started.sh | | `docker_prune_images.sh` | Remove dangling or unused Docker images accumulated after updates | After update cycles, or when disk space is low | | `ramdisk_stop.sh` | Safely stop the transcode ramdisk — redirect symlink to SSD, unmount, update state | Before re-running ramdisk_setup.sh with new size or thresholds | +| `arr_profile_enforcer.sh` | Enforce correct quality profiles across all Sonarr/Radarr libraries | After arr setup, profile changes, or when library was imported with wrong profile | +| `webhook_setup.sh` | Register the Varaverk upgrade webhook in Sonarr, Radarr, and Lidarr | After initial install or when adding a new arr or host | | `emby_to_lidarr_sync.sh` | Add all Emby album artists not yet tracked in Lidarr | After Lidarr setup, database wipe, or when you suspect gaps | | `emby_to_sonarr_sync.sh` | Add all Emby TV series not yet tracked in Sonarr | After Sonarr setup, database wipe, or when you suspect gaps | | `emby_to_radarr_sync.sh` | Add all Emby movies not yet tracked in Radarr | After Radarr setup, database wipe, or when you suspect gaps | @@ -150,8 +141,9 @@ Situation arises │ container_data_export ◄── before a risky update or migration │ │ emby_database_repair ◄── Emby logs show corruption │ │ zfs_pool_scrub ◄── monthly integrity check / post-event │ -│ recreate_shares ◄── fresh HOST2 setup or disk rebuild │ -│ continuous_scripts_status ◄── manual status check at any time │ +│ smart_long_test ◄── monthly SMART scan / post-disk event │ +│ arr_profile_enforcer ◄── wrong profiles after import or setup │ +│ webhook_setup ◄── after install or adding a new arr │ │ claude_startup ◄── after each unRAID reboot │ │ ramdisk_stop ◄── before ramdisk resize / remount │ │ │