Make all arr library-list fetches cache-first with live fallback

Every script that fetches the full Lidarr/Sonarr/Radarr tracked-library
list now goes through arr_get_tracked_data() instead of hitting the API
directly -- cache-first when fresh, live fetch as fallback when stale,
waits out an active rescan before either. Per-item file data (trackFile/
episodefile/moviefile) stays live-only everywhere, since that's the
actual disk-truth these scripts' decisions depend on and was never part
of what's cached.

Also adds arr_cache_prefill.sh to CRITICAL_MAINTENANCE_SCRIPTS (30min
tier) with a short 1min wait ceiling, so the cache stays consistently
fresh instead of only refreshing whenever some other script happens to
write through. A full cache refresh for all three arrs measured at ~12s
total live -- nothing like the multi-hour cost of an actual rescan.

Arrs_Stack/arr_sync.sh

@@ -659,9 +659,20 @@ REMOTE
 
 _local_library() {
     local url="$1" api_key="$2" api_ver="$3" endpoint="$4"
-    curl -sf --max-time "$ARR_SYNC_API_TIMEOUT" \
-        -H "X-Api-Key: $api_key" \
-        "${url}/api/${api_ver}/${endpoint}" 2>/dev/null
+    # endpoint doubles as arr_type here — artist/series/movie match lidarr/sonarr/radarr's
+    # ARR_LIBRARY_ENDPOINT values exactly, so the same lookup works either direction.
+    local arr_type
+    case "$endpoint" in
+        artist) arr_type="lidarr" ;;
+        series) arr_type="sonarr" ;;
+        movie)  arr_type="radarr" ;;
+    esac
+    # Cache-first — arr_get_tracked_data() serves the shared cache when it's fresh (kept
+    # current every 30min by arr_cache_prefill.sh in CRITICAL_MAINTENANCE_SCRIPTS), falls back
+    # to a live fetch when it's stale, and waits out an active rescan before either. This is
+    # the LOCAL host's library only — the remote side of this sync isn't cached, since the
+    # shared cache is per-host by design.
+    arr_get_tracked_data "$arr_type" "$url" "$api_key" "$api_ver"
 }
 
 _local_defaults() {

Arrs_Stack/lidarr_cleanup.sh

@@ -299,15 +299,17 @@ check_arr_version "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$LIDARR_VERSION_MAJOR" "
 
 info "Querying Lidarr API..."
 
-# Fetch all artists
-ARTIST_RESPONSE=$(arr_api "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "artist" "Lidarr") || {
+# Cache-first — arr_get_tracked_data() serves the shared cache when it's fresh (now kept
+# current every 30min by arr_cache_prefill.sh in CRITICAL_MAINTENANCE_SCRIPTS), falls back to
+# a live fetch when it's stale, and waits out an active rescan before either. Only the artist
+# list itself is cached — the per-artist trackFile data below is never cached and always live,
+# since that's the actual disk-truth this script's cleanup decisions depend on.
+ARTIST_RESPONSE=$(arr_get_tracked_data "lidarr" "$LIDARR_URL" "$LIDARR_API_KEY" "v1") || {
     error "Failed to fetch artists from Lidarr"
     notify "Lidarr cleanup failed on $(hostname) — could not fetch artists" \
         "Lidarr Cleanup" "warning"
     exit 1
 }
-# Write-through — free cache refresh from a fetch this script already needed.
-arr_cache_write "lidarr" "$ARTIST_RESPONSE"
 
 ARTIST_IDS=$(echo "$ARTIST_RESPONSE" | jq -r '.[].id' 2>/dev/null)
 ARTIST_COUNT=$(echo "$ARTIST_IDS" | grep -c "[0-9]" 2>/dev/null || echo 0)

Arrs_Stack/lidarr_missing_art.sh

@@ -254,10 +254,10 @@ echo ""
 echo "━━━ $ICON_SYNC Building Album Directory Map ━━━"
 
 declare -A ALBUM_DIR_MAP
-_artist_list=$(curl_json "$LIDARR_URL/api/v1/artist?apikey=$LIDARR_API_KEY")
-# Write-through — keeps the shared tracked-data cache fresh as a side effect of a fetch
-# this script already needed for its own purposes. See arr_get_tracked_data() in common.sh.
-arr_cache_write "lidarr" "$_artist_list"
+# Cache-first — arr_get_tracked_data() serves the shared cache when it's fresh (now kept
+# current every 30min by arr_cache_prefill.sh in CRITICAL_MAINTENANCE_SCRIPTS), falls back to
+# a live fetch when it's stale, and waits out an active rescan before either.
+_artist_list=$(arr_get_tracked_data "lidarr" "$LIDARR_URL" "$LIDARR_API_KEY" "v1")
 _map_artist_count=$(echo "$_artist_list" | jq '. | length')
 info "Fetching track files for $_map_artist_count artists..."
 
@@ -364,7 +364,9 @@ info "Checked: $ALBUMS_CHECKED | Complete: $ALBUMS_COMPLETE | Needed art: $ALBUM
 echo ""
 echo "━━━ $ICON_EMBY Artists ━━━"
 
-artists=$(curl_json "$LIDARR_URL/api/v1/artist?apikey=$LIDARR_API_KEY")
+# Cache-first — see the earlier album-map fetch above for why (arr_get_tracked_data serves
+# the shared cache when fresh, live fetch as fallback, waits out an active rescan first).
+artists=$(arr_get_tracked_data "lidarr" "$LIDARR_URL" "$LIDARR_API_KEY" "v1")
 
 if [[ -z "$artists" || "$artists" == "null" ]]; then
     error "Lidarr artist API returned empty — aborting"

Arrs_Stack/lidarr_release_fixer.sh

@@ -302,13 +302,13 @@ echo "━━━ $ICON_SYNC Fetching artist paths ━━━"
 declare -A ARTIST_PATH_CACHE
 declare -A ARTIST_NAME_CACHE
 
-ALL_ARTISTS=$(lidarr_api "artist") || {
+# Cache-first — arr_get_tracked_data() serves the shared cache when it's fresh (now kept
+# current every 30min by arr_cache_prefill.sh in CRITICAL_MAINTENANCE_SCRIPTS), falls back to
+# a live fetch when it's stale, and waits out an active rescan before either.
+ALL_ARTISTS=$(arr_get_tracked_data "lidarr" "$LIDARR_URL" "$LIDARR_API_KEY" "v1") || {
     error "Failed to fetch artists"
     exit 1
 }
-# Write-through — keeps the shared tracked-data cache fresh as a side effect of a fetch
-# this script already needed for its own purposes. See arr_get_tracked_data() in common.sh.
-arr_cache_write "lidarr" "$ALL_ARTISTS"
 
 while IFS= read -r artist; do
     aid=$(echo "$artist" | jq -r '.id')

Arrs_Stack/playback_aware_lidarr_discovery.sh

@@ -516,7 +516,10 @@ fi
 echo ""
 echo "━━━ $ICON_SYNC Existing Libraries ━━━"
 
-LIDARR_ARTISTS_JSON=$(_lidarr_get "artist") || { error "Could not fetch Lidarr artists"; exit 1; }
+# Cache-first — arr_get_tracked_data() serves the shared cache when it's fresh (kept current
+# every 30min by arr_cache_prefill.sh in CRITICAL_MAINTENANCE_SCRIPTS), falls back to a live
+# fetch when it's stale, and waits out an active rescan before either.
+LIDARR_ARTISTS_JSON=$(arr_get_tracked_data "lidarr" "$LIDARR_URL" "$LIDARR_API_KEY" "v1") || { error "Could not fetch Lidarr artists"; exit 1; }
 LIDARR_NAMES=$(echo "$LIDARR_ARTISTS_JSON" | jq -r '.[].artistName' 2>/dev/null)
 LIDARR_COUNT=$(echo "$LIDARR_NAMES" | grep -c . 2>/dev/null || echo 0)
 log "$LIDARR_COUNT artists in Lidarr"

Arrs_Stack/playback_aware_radarr_discovery.sh

@@ -485,7 +485,10 @@ fi
 echo ""
 echo "━━━ $ICON_SYNC Existing Libraries ━━━"
 
-RADARR_MOVIES_JSON=$(_radarr_get "movie") || { error "Could not fetch Radarr library"; exit 1; }
+# Cache-first — arr_get_tracked_data() serves the shared cache when it's fresh (kept current
+# every 30min by arr_cache_prefill.sh in CRITICAL_MAINTENANCE_SCRIPTS), falls back to a live
+# fetch when it's stale, and waits out an active rescan before either.
+RADARR_MOVIES_JSON=$(arr_get_tracked_data "radarr" "$RADARR_URL" "$RADARR_API_KEY" "v3") || { error "Could not fetch Radarr library"; exit 1; }
 
 declare -A RADARR_TMDB   # tmdb_id → 1
 while read -r tmdb_id; do

Arrs_Stack/playback_aware_sonarr_discovery.sh

@@ -591,7 +591,10 @@ fi
 echo ""
 echo "━━━ $ICON_SYNC Existing Libraries ━━━"
 
-SONARR_SERIES_JSON=$(_sonarr_get "series") || { error "Could not fetch Sonarr library"; exit 1; }
+# Cache-first — arr_get_tracked_data() serves the shared cache when it's fresh (kept current
+# every 30min by arr_cache_prefill.sh in CRITICAL_MAINTENANCE_SCRIPTS), falls back to a live
+# fetch when it's stale, and waits out an active rescan before either.
+SONARR_SERIES_JSON=$(arr_get_tracked_data "sonarr" "$SONARR_URL" "$SONARR_API_KEY" "v3") || { error "Could not fetch Sonarr library"; exit 1; }
 
 declare -A SONARR_TVDB   # tvdb_id → 1
 declare -A SONARR_TMDB   # tmdb_id → 1 (Sonarr v4 exposes tmdbId)

Arrs_Stack/radarr_cleanup.sh

@@ -278,15 +278,17 @@ check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" "
 
 info "Querying Radarr API..."
 
-# Fetch all movies
-MOVIES_RESPONSE=$(arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "movie" "Radarr") || {
+# Cache-first — arr_get_tracked_data() serves the shared cache when it's fresh (now kept
+# current every 30min by arr_cache_prefill.sh in CRITICAL_MAINTENANCE_SCRIPTS), falls back to
+# a live fetch when it's stale, and waits out an active rescan before either. Only the movie
+# list itself is cached — the per-movie moviefile data below is never cached and always live,
+# since that's the actual disk-truth this script's cleanup decisions depend on.
+MOVIES_RESPONSE=$(arr_get_tracked_data "radarr" "$RADARR_URL" "$RADARR_API_KEY" "v3") || {
     error "Failed to fetch movies from Radarr"
     notify "Radarr cleanup failed on $(hostname) — could not fetch movies" \
         "Radarr Cleanup" "warning"
     exit 1
 }
-# Write-through — free cache refresh from a fetch this script already needed.
-arr_cache_write "radarr" "$MOVIES_RESPONSE"
 
 MOVIE_IDS=$(echo "$MOVIES_RESPONSE" | jq -r '.[].id' 2>/dev/null)
 MOVIE_COUNT=$(echo "$MOVIE_IDS" | grep -c "." 2>/dev/null || echo 0)

Arrs_Stack/radarr_tmdb_removed.sh

@@ -153,8 +153,10 @@ if ! curl -sf --connect-timeout 5 --max-time 10 \
     exit 1
 fi
 
-MOVIES=$(curl -sf --connect-timeout 5 --max-time 30 \
-    "$RADARR_URL/api/v3/movie?apikey=$RADARR_API_KEY" 2>/dev/null)
+# Cache-first — arr_get_tracked_data() serves the shared cache when it's fresh (kept current
+# every 30min by arr_cache_prefill.sh in CRITICAL_MAINTENANCE_SCRIPTS), falls back to a live
+# fetch when it's stale, and waits out an active rescan before either.
+MOVIES=$(arr_get_tracked_data "radarr" "$RADARR_URL" "$RADARR_API_KEY" "v3")
 
 if [[ -z "$MOVIES" || "$MOVIES" == "null" ]]; then
     error "Radarr movie API returned empty"

Arrs_Stack/sonarr_cleanup.sh

@@ -279,15 +279,17 @@ check_arr_version "$SONARR_URL" "$SONARR_API_KEY" "v3" "$SONARR_VERSION_MAJOR" "
 
 info "Querying Sonarr API..."
 
-# Fetch all series
-SERIES_RESPONSE=$(arr_api "$SONARR_URL" "$SONARR_API_KEY" "v3" "series" "Sonarr") || {
+# Cache-first — arr_get_tracked_data() serves the shared cache when it's fresh (now kept
+# current every 30min by arr_cache_prefill.sh in CRITICAL_MAINTENANCE_SCRIPTS), falls back to
+# a live fetch when it's stale, and waits out an active rescan before either. Only the series
+# list itself is cached — the per-series episodeFile data below is never cached and always
+# live, since that's the actual disk-truth this script's cleanup decisions depend on.
+SERIES_RESPONSE=$(arr_get_tracked_data "sonarr" "$SONARR_URL" "$SONARR_API_KEY" "v3") || {
     error "Failed to fetch series from Sonarr"
     notify "Sonarr cleanup failed on $(hostname) — could not fetch series" \
         "Sonarr Cleanup" "warning"
     exit 1
 }
-# Write-through — free cache refresh from a fetch this script already needed.
-arr_cache_write "sonarr" "$SERIES_RESPONSE"
 
 SERIES_IDS=$(echo "$SERIES_RESPONSE" | jq -r '.[].id' 2>/dev/null)
 SERIES_COUNT=$(echo "$SERIES_IDS" | grep -c "." 2>/dev/null || echo 0)

Arrs_Stack/sonarr_tvdb_removed.sh

@@ -152,8 +152,10 @@ if ! curl -sf --connect-timeout 5 --max-time 10 \
     exit 1
 fi
 
-SERIES=$(curl -sf --connect-timeout 5 --max-time 30 \
-    "$SONARR_URL/api/v3/series?apikey=$SONARR_API_KEY" 2>/dev/null)
+# Cache-first — arr_get_tracked_data() serves the shared cache when it's fresh (kept current
+# every 30min by arr_cache_prefill.sh in CRITICAL_MAINTENANCE_SCRIPTS), falls back to a live
+# fetch when it's stale, and waits out an active rescan before either.
+SERIES=$(arr_get_tracked_data "sonarr" "$SONARR_URL" "$SONARR_API_KEY" "v3")
 
 if [[ -z "$SERIES" || "$SERIES" == "null" ]]; then
     error "Sonarr series API returned empty"

Deployment/master.conf.template

@@ -359,6 +359,7 @@
     CRITICAL_MAINTENANCE_SCRIPTS=(
         "Docker_Essentials/downloaders_reset.sh"        # clear stuck download states every 30min
         "Media/play_state_sync.sh"                      # sync watched/resume state across Emby + Jellyfin
+        "Arrs_Stack/arr_cache_prefill.sh ARR_PREFILL_WAIT_MINUTES=1"  # keep the shared arr tracked-data cache fresh — a live fetch+write takes seconds, not the boot-time 10min wait ceiling
     )
 
 # Shares synced every 30 minutes — defined per host in host*.conf.

Tools/emby_to_lidarr_sync.sh

@@ -180,7 +180,10 @@ fi
 echo ""
 echo "━━━ $ICON_SYNC Lidarr Library ━━━"
 
-LIDARR_ARTISTS_JSON=$(_lidarr_get "artist") || { error "Could not fetch Lidarr artists"; exit 1; }
+# Cache-first — arr_get_tracked_data() serves the shared cache when it's fresh (kept current
+# every 30min by arr_cache_prefill.sh in CRITICAL_MAINTENANCE_SCRIPTS), falls back to a live
+# fetch when it's stale, and waits out an active rescan before either.
+LIDARR_ARTISTS_JSON=$(arr_get_tracked_data "lidarr" "$LIDARR_URL" "$LIDARR_API_KEY" "v1") || { error "Could not fetch Lidarr artists"; exit 1; }
 LIDARR_NAMES=$(echo "$LIDARR_ARTISTS_JSON" | jq -r '.[].artistName' 2>/dev/null)
 LIDARR_COUNT=$(echo "$LIDARR_NAMES" | grep -c . 2>/dev/null || echo 0)
 log "$LIDARR_COUNT artists already in Lidarr"

Tools/emby_to_radarr_sync.sh

@@ -195,7 +195,10 @@ fi
 echo ""
 echo "━━━ $ICON_SYNC Radarr Library ━━━"
 
-RADARR_MOVIES_JSON=$(_radarr_get "movie") || { error "Could not fetch Radarr movies"; exit 1; }
+# Cache-first — arr_get_tracked_data() serves the shared cache when it's fresh (kept current
+# every 30min by arr_cache_prefill.sh in CRITICAL_MAINTENANCE_SCRIPTS), falls back to a live
+# fetch when it's stale, and waits out an active rescan before either.
+RADARR_MOVIES_JSON=$(arr_get_tracked_data "radarr" "$RADARR_URL" "$RADARR_API_KEY" "v3") || { error "Could not fetch Radarr movies"; exit 1; }
 
 declare -A RADARR_TMDB    # tmdb_id → 1
 declare -A RADARR_TITLES  # lower(title) → 1

Tools/emby_to_sonarr_sync.sh

@@ -189,7 +189,10 @@ fi
 echo ""
 echo "━━━ $ICON_SYNC Sonarr Library ━━━"
 
-SONARR_SERIES_JSON=$(_sonarr_get "series") || { error "Could not fetch Sonarr series"; exit 1; }
+# Cache-first — arr_get_tracked_data() serves the shared cache when it's fresh (kept current
+# every 30min by arr_cache_prefill.sh in CRITICAL_MAINTENANCE_SCRIPTS), falls back to a live
+# fetch when it's stale, and waits out an active rescan before either.
+SONARR_SERIES_JSON=$(arr_get_tracked_data "sonarr" "$SONARR_URL" "$SONARR_API_KEY" "v3") || { error "Could not fetch Sonarr series"; exit 1; }
 
 declare -A SONARR_TVDB    # tvdb_id → 1
 declare -A SONARR_TITLES  # lower(title) → 1