feat: orchestrator housekeeping — watchdog array, script mapping, orch reorder

- master.conf: add WATCHDOG_ORCHESTRATOR_SCRIPTS array; watchdog config moved
  from standalone section into orch block alongside all other tiers
- watchdog_orchestrator.sh: iterate WATCHDOG_ORCHESTRATOR_SCRIPTS instead of
  hardcoded paths; display name derived from filename; status block updated
- master.conf orch section reordered: Array Stop → Array Start → Watchdog (1min)
  → Critical (15min) → Intermediate (4hr) → Daily (1am) → Weekly (Sunday)
- arrs_failed_stalled_recovery.sh added to INTERMEDIATE_MAINTENANCE_SCRIPTS
- lidarr_missing_art.sh moved from intermediate to daily
- clear_logs.sh added to WEEKLY_MAINTENANCE_SCRIPTS (was documented but orphaned)
- playback_aware_lidarr_discovery.sh placed in weekly (commented out, WIP)
- master_host1.conf: Slskd added to DAILY_RESTART_CONTAINERS

Orchestrators/watchdog_orchestrator.sh

@@ -2,13 +2,12 @@
 # ==============================================================================================
 # ============================ Watchdog Orchestrator ===========================================
 # ==============================================================================================
-# Runs the three-layer watchdog system in the correct sequence each cycle.
+# Runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
 # Schedule: * * * * * (every minute via User Scripts plugin)
 #
 # ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
-#   1. resource_watchdog.sh   — reduce system pressure intelligently
+#   Driven by WATCHDOG_ORCHESTRATOR_SCRIPTS in master.conf — add, remove, or reorder there.
-#   2. docker_watchdog.sh    — heal containers with freed resources
+#   Default: resource_watchdog → docker_watchdog → system_watchdog
-#   3. system_watchdog.sh    — reboot if all else fails (last line of defense)
 #
 # ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
 #   Resource Watchdog first — frees RAM and CPU before healing attempts container restarts.
@@ -32,13 +31,14 @@
 #   Remove system_watchdog.sh and docker_watchdog.sh from ARRAY_START_SCRIPTS.
 #
 # ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
+#   WATCHDOG_ORCHESTRATOR_SCRIPTS         — watchdogs to run, in order
 #   WATCHDOG_STARTUP_GRACE                — seconds after boot before checks activate
 #   WATCHDOG_ORCHESTRATOR_HEARTBEAT       — periodic heartbeat log toggle
 #   WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours
 #
 # ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
 #   watchdog_orchestrator.sh           — normal run (called by cron every minute)
-#   watchdog_orchestrator.sh --dry-run — pass --dry-run to all three sub-scripts
+#   watchdog_orchestrator.sh --dry-run — pass --dry-run to all sub-scripts
 #   watchdog_orchestrator.sh --status  — show script paths and current grace state
 #   watchdog_orchestrator.sh --log     — verbose output from all sub-scripts
 # ==============================================================================================
@@ -63,12 +63,17 @@ acquire_lock
 
 detect_hosts
 
-RESOURCE_WATCHDOG="$ECOSYSTEM_ROOT/unRAID_Essentials/resource_watchdog.sh"
-DOCKER_WATCHDOG="$ECOSYSTEM_ROOT/Docker_Essentials/docker_watchdog.sh"
-SYSTEM_WATCHDOG="$ECOSYSTEM_ROOT/unRAID_Essentials/system_watchdog.sh"
-
 [[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"
 
+# Derive a display name from a script path: "resource_watchdog.sh" → "Resource Watchdog"
+_watchdog_display_name() {
+    local path="$1"
+    local base="${path##*/}"
+    base="${base%.sh}"
+    base="${base//_/ }"
+    echo "$base" | awk '{for(i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) substr($i,2); print}'
+}
+
 # ==============================================================================================
 # ━━━ Status ━━━
 # ==============================================================================================
@@ -87,17 +92,14 @@ if [[ "$SHOW_STATUS" == true ]]; then
 
     echo ""
     echo "── Sub-scripts ──"
-    for pair in \
+    for entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
-        "Resource Watchdog:$RESOURCE_WATCHDOG" \
+        local_path="$ECOSYSTEM_ROOT/$entry"
-        "Docker Watchdog:$DOCKER_WATCHDOG" \
+        label="$(_watchdog_display_name "$entry")"
-        "System Watchdog:$SYSTEM_WATCHDOG"; do
+        if [[ -f "$local_path" ]]; then
-        label="${pair%%:*}"
+            [[ -x "$local_path" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
-        script="${pair#*:}"
+            echo "  $icon $label — ${local_path##*/}"
-        if [[ -f "$script" ]]; then
-            [[ -x "$script" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
-            echo "  $icon $label — ${script##*/}"
         else
-            echo "  $ICON_ERROR $label — NOT FOUND: $script"
+            echo "  $ICON_ERROR $label — NOT FOUND: $local_path"
         fi
     done
 
@@ -150,9 +152,9 @@ run_watchdog() {
     fi
 }
 
-run_watchdog "Resource Watchdog" "$RESOURCE_WATCHDOG"
+for _entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
-run_watchdog "Docker Watchdog"  "$DOCKER_WATCHDOG"
+    run_watchdog "$(_watchdog_display_name "$_entry")" "$ECOSYSTEM_ROOT/$_entry"
-run_watchdog "System Watchdog"  "$SYSTEM_WATCHDOG"
+done
 
 CYCLE_END=$(date +%s)
 DURATION=$(( CYCLE_END - CYCLE_START ))

master.conf

@@ -262,6 +262,7 @@
 # ==============================================================================================
 # All orchestrator job lists live here — edit arrays to add/remove scripts.
 # No changes to orchestrator scripts needed when adding or removing jobs.
+# Sections ordered by run frequency: array events first, then shortest interval to longest.
 
 # ━━━ Array Stop ━━━
 # Scripts run by array_stopping.sh for a planned shutdown — stops everything cleanly in order.
@@ -292,6 +293,35 @@
         "Fallback/fallback.sh"                        # mutual failover — continuous
     )
 
+# ━━━ Watchdog Orchestrator ━━━
+# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
+# Schedule: * * * * * (every minute)
+# NOT in ARRAY_START_SCRIPTS — has its own cron entry.
+# Order matters — resource first (frees pressure), docker second (heals with freed resources),
+# system last (reboots only if prior layers failed).
+    WATCHDOG_ORCHESTRATOR_SCRIPTS=(
+        "unRAID_Essentials/resource_watchdog.sh"  # reduce system pressure before healing attempts
+        "Docker_Essentials/docker_watchdog.sh"    # heal containers with freed resources
+        "unRAID_Essentials/system_watchdog.sh"    # reboot if all else fails — last line of defense
+    )
+
+    WATCHDOG_ORCHESTRATOR_HEARTBEAT=true
+    WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS=1
+
+# ━━━ Critical Sync Maintenance ━━━
+# critical_sync_maintenance.sh runs every 15 minutes.
+# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check
+# partnership --check always runs last regardless of rsync gate.
+# Comment out entries to disable without removing.
+    CRITICAL_MAINTENANCE_SCRIPTS=(
+        "Docker_Essentials/downloaders_reset.sh"        # clear stuck download states every 15min
+    )
+
+# Shares synced every 15 minutes — defined per host in master_host*.conf.
+# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES
+# Format: "/path/to/share" or "/path/to/share|profile-name"
+# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
+
 # ━━━ Intermediate Sync Maintenance ━━━
 # intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch,
 # and optional mid-day rsync for any shares that need sub-daily propagation.
@@ -300,7 +330,7 @@
     INTERMEDIATE_RSYNC_ENABLED=true       # set false to disable mid-day rsync without removing shares
 
     INTERMEDIATE_MAINTENANCE_SCRIPTS=(
-        "Media/lidarr_missing_art.sh"     # fetch missing album/artist artwork (HOST1 only — self-guards)
+        "Media/arrs_failed_stalled_recovery.sh"  # blocklist + re-search failed/stalled arr queue items
     )
     # arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here.
     # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).
@@ -317,6 +347,7 @@
         #"Media/lidarr_cleanup.sh"                    # remove orphaned music files — enable when ready
         #"Media/sonarr_cleanup.sh"                    # remove orphaned TV files — enable when ready
         #"Media/radarr_cleanup.sh"                    # remove orphaned movie files — enable when ready
+        "Media/lidarr_missing_art.sh"                 # fetch missing album/artist artwork (HOST1 only — self-guards)
         "Media/radarr_tmdb_removed.sh"                # remove movies dropped from TMDb
         "Media/sonarr_tvdb_removed.sh"                # remove series dropped from TVDB
         "Docker_Essentials/docker_update.sh"          # pull container image updates before restart
@@ -348,6 +379,8 @@
     WEEKLY_MAINTENANCE_SCRIPTS=(
         "Docker_Essentials/docker_weekly_restart.sh"    # weekly container restarts after sync
         "Docker_Essentials/docker_update_remaining.sh"  # pull updates for all other containers
+        "unRAID_Essentials/clear_logs.sh"               # purge aged logs — Sunday only, low priority
+        #"Media/playback_aware_lidarr_discovery.sh"     # behavior-driven music discovery using weekly Emby playback history — WIP
     )
 
 # Pull updates for all running containers NOT in daily/weekly restart lists.
@@ -367,22 +400,6 @@
     WEEKLY_SYNC_UPDATES=true          # pull container updates locally during weekly window
     WEEKLY_SYNC_UPDATES_REMOTE=true   # pull container updates on remote via SSH
 
-# ━━━ Critical Sync Maintenance ━━━
-# critical_sync_maintenance.sh runs every 15 minutes.
-# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check
-# partnership --check always runs last regardless of rsync gate.
-
-# Jobs run every 15 minutes before the rsync shares.
-# Comment out to disable without removing.
-    CRITICAL_MAINTENANCE_SCRIPTS=(
-        "Docker_Essentials/downloaders_reset.sh"        # clear stuck download states every 15min
-    )
-
-# Shares synced every 15 minutes — defined per host in master_host*.conf.
-# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES
-# Format: "/path/to/share" or "/path/to/share|profile-name"
-# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
-
 # ==============================================================================================
 # ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
 # ==============================================================================================
@@ -1115,16 +1132,6 @@
     EMBY_REPORT_DAYS=7              # days to include in the report period
     EMBY_REPORT_TOP_N=10            # number of top content items to show
 
-# ==============================================================================================
-# ── WATCHDOG ORCHESTRATOR ─────────────────────────────────────────────────────────────────────
-# ==============================================================================================
-# Runs resource_watchdog → docker_watchdog → system_watchdog in sequence each cron cycle.
-# Schedule: * * * * * (every minute via User Scripts plugin)
-# NOT in ARRAY_START_SCRIPTS — has its own cron entry.
-
-    WATCHDOG_ORCHESTRATOR_HEARTBEAT=true
-    WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS=1
-
 # ==============================================================================================
 # ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
 # ==============================================================================================

master_host1.conf

@@ -252,6 +252,7 @@
         "Dispatcharr"               # Live TV scheduler — degrades without daily restart
         "Dispatcharr-Basic"
         "ErsatzTV-Emby"
+        "Slskd"                     # Soulseek connection drops after extended uptime; restart refreshes share index
     )
 
 # ━━━ Docker Weekly Restart ━━━