diff --git a/Arrs_Stack/radarr_cleanup.sh b/Arrs_Stack/radarr_cleanup.sh index 7066feb..48729f3 100755 --- a/Arrs_Stack/radarr_cleanup.sh +++ b/Arrs_Stack/radarr_cleanup.sh @@ -18,19 +18,26 @@ # false-alarm abort. Mirrors the same fix built for lidarr_cleanup.sh 2026-07-16 after # a whole-library rescan there made trackFileCount read 22% of normal mid-scan. # -# Cache-first movie list, no per-movie API calls at all (2026-07-17). The movie list comes -# from the shared tracked-data cache via arr_get_tracked_data() — fresh (kept warm every -# 30min by arr_cache_prefill.sh), live fetch as fallback. Radarr is structurally different -# from Lidarr/Sonarr here: its movie list already embeds movieFile.path directly on every -# hasFile=true entry (confirmed live, zero exceptions across the full library), so there's -# no separate per-movie moviefile?movieId=X walk needed at all — what used to be up to 2896 -# individual API calls is now a jq filter over data already in hand. The filesystem is -# walked once per run, not twice — classification records which paths are eligible for -# deletion as it goes, and the delete pass (once the size-threshold check below passes) -# just acts on that list instead of re-walking and re-classifying the whole tree. That single -# walk also gets size+mtime straight from find -printf instead of a separate stat fork per -# file — find already has to stat() every entry to know it's -type f, so this is free by -# comparison. Measured ~130x faster per file (0.033ms vs 4.3ms). +# Cache-first movie list (2026-07-17), batched moviefile fetch (2026-07-19). The movie list +# comes from the shared tracked-data cache via arr_get_tracked_data() — fresh (kept warm +# every 30min by arr_cache_prefill.sh), live fetch as fallback. Radarr's movie list embeds +# movieFile.path directly on every hasFile=true entry, but that's only the *primary* file — +# Radarr 6+ supports a second tracked file per movie (alternate editions/extras) that never +# shows up there, so relying on it alone misclassified a movie's second edition as an orphan +# (confirmed live 2026-07-19: The Crash, They Will Kill You, The Drama, Lee Cronin's The +# Mummy, and Ready or Not: Here I Come all had a legitimately-tracked second file deleted-flagged +# this way). /moviefile?movieId=X returns every file for a movie, including secondaries, and +# accepts movieId as a repeated query param for a bulk fetch — but the whole library in one +# request 414s (Request-URI Too Long, confirmed live), so _fetch_tracked_files() batches +# movieId params BATCH_SIZE at a time instead: ~14 requests for a ~2800-movie library rather +# than the up-to-2896 individual per-movie calls the 2026-07-17 optimization eliminated, and +# rather than the one-shot list read that missed secondary files. The filesystem is walked +# once per run, not twice — classification records which paths are eligible for deletion as +# it goes, and the delete pass (once the size-threshold check below passes) just acts on that +# list instead of re-walking and re-classifying the whole tree. That single walk also gets +# size+mtime straight from find -printf instead of a separate stat fork per file — find +# already has to stat() every entry to know it's -type f, so this is free by comparison. +# Measured ~130x faster per file (0.033ms vs 4.3ms). # # ============================================================================================== # OPERATIONAL MODEL @@ -320,26 +327,43 @@ info "Found $MOVIE_COUNT movies — fetching movie files..." TRACKED_FILE="$TMP_DIR/tracked_paths.txt" > "$TRACKED_FILE" -# Fetches every movie's file path fresh into TRACKED_FILE/TRACKED_MAP/TRACKED_COUNT. Pulled -# into a function so the rescan-aware retry below can re-fetch after waiting without +# Fetches every movie's file path(s) fresh into TRACKED_FILE/TRACKED_MAP/TRACKED_COUNT. +# Pulled into a function so the rescan-aware retry below can re-fetch after waiting without # duplicating this whole loop inline. # -# Radarr's movie list already embeds movieFile.path directly on every entry with -# hasFile=true (confirmed live 2026-07-17, zero exceptions across the full library) — -# unlike Lidarr/Sonarr, where trackFile/episodefile genuinely require a separate per-item -# call. A single live movie-list fetch replaces what used to be one API call per movie -# (2896 of them) for the same data already sitting in that response. Still a fresh live -# fetch on every call (not cache-first) — this function's whole purpose during the +# Batched, not per-movie and not a single one-shot list read (2026-07-19) — see the header +# comment above for why movie.movieFile.path alone misses secondary edition files. Still a +# fresh live fetch on every call (not cache-first) — this function's whole purpose during the # rescan-aware retry below is to see Radarr's progress as the rescan updates hasFile/ # movieFile, so it needs genuinely current data each time, not a stale snapshot. _fetch_tracked_files() { > "$TRACKED_FILE" local movies_now movies_now=$(arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "movie" "Radarr" 2>/dev/null) - while IFS= read -r api_path; do + + local _ids=() _id _qs="" _batch_count=0 + local BATCH_SIZE=200 # 250 confirmed working live 2026-07-19; kept under that for margin + mapfile -t _ids < <(echo "$movies_now" | jq -r '.[] | select(.hasFile==true) | .id' 2>/dev/null) + + { + for _id in "${_ids[@]}"; do + _qs+="movieId=${_id}&" + (( _batch_count++ )) + if [[ "$_batch_count" -ge "$BATCH_SIZE" ]]; then + arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "moviefile?${_qs%&}" "Radarr" 2>/dev/null | \ + jq -r '.[].path' 2>/dev/null + _qs="" + _batch_count=0 + fi + done + if [[ -n "$_qs" ]]; then + arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "moviefile?${_qs%&}" "Radarr" 2>/dev/null | \ + jq -r '.[].path' 2>/dev/null + fi + } | while IFS= read -r api_path; do [[ -z "$api_path" ]] && continue translate_path "$api_path" >> "$TRACKED_FILE" - done < <(echo "$movies_now" | jq -r '.[] | select(.hasFile==true) | .movieFile.path' 2>/dev/null) + done sort -u "$TRACKED_FILE" -o "$TRACKED_FILE"