From 2c3f0b9cb1d9d79b1567d2a50e3284e98e9b7ab9 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Thu, 16 Jul 2026 22:57:54 -0400 Subject: [PATCH] Guard arr cache writes against in-flight rescans, add rescan monitor tool A direct arr_cache_write() call mid-rescan wrote a partial snapshot that looked like real data loss to every consumer of the cache. The guard now lives in arr_cache_write() itself so every caller is protected, not just arr_get_tracked_data(). arr_rescan_monitor.sh closes the resulting gap for rescans triggered outside arr_full_rescan.sh's own trigger-and-wait path. --- Arrs_Stack/arr_full_rescan.sh | 2 +- Tools/Manual-Tools.md | 55 ++++++++++++++++++++ Tools/README-Tools.md | 13 ++++- Tools/arr_rescan_monitor.sh | 97 +++++++++++++++++++++++++++++++++++ common.sh | 50 ++++++++++++++++++ 5 files changed, 215 insertions(+), 2 deletions(-) create mode 100755 Tools/arr_rescan_monitor.sh diff --git a/Arrs_Stack/arr_full_rescan.sh b/Arrs_Stack/arr_full_rescan.sh index 47d8447..d42d4eb 100755 --- a/Arrs_Stack/arr_full_rescan.sh +++ b/Arrs_Stack/arr_full_rescan.sh @@ -86,7 +86,7 @@ for arr in lidarr sonarr radarr; do active=$(arr_active_rescan_command "$arr" "$url" "$key" "$ver") if [[ -n "$active" ]]; then - warn "${arr^} already mid-rescan ($active) — skipping, will catch it next scheduled run" + warn "${arr^} already mid-rescan ($active) — skipping, will catch it next scheduled run (run Tools/arr_rescan_monitor.sh ${arr} to refresh its cache as soon as this one finishes instead of waiting)" (( SKIPPED++ )) continue fi diff --git a/Tools/Manual-Tools.md b/Tools/Manual-Tools.md index 862289e..0cd8a3b 100644 --- a/Tools/Manual-Tools.md +++ b/Tools/Manual-Tools.md @@ -22,6 +22,7 @@ making any changes. - [arr_profile_enforcer.sh](#arr_profile_enforcersh) - [webhook_setup.sh](#webhook_setupsh) - [ramdisk_stop.sh](#ramdisk_stopsh) +- [arr_rescan_monitor.sh](#arr_rescan_monitorsh) - [Adding a New Tool](#adding-a-new-tool) --- @@ -733,6 +734,60 @@ ramdisk_stop.sh --log # verbose — show each step --- +## arr_rescan_monitor.sh + +Waits for an already-active rescan-type command (RescanFolders/RescanSeries/RescanMovie, +DownloadedXScan, RefreshX) on the given arr to finish, then refreshes the shared tracked-data +cache (`arr_cache_write()` in common.sh) with the real post-scan numbers. Triggers nothing +itself — for watching a rescan that was already started some other way (manual API call, +the arr's own UI, an ad-hoc debugging session). + +### When to Use + +``` +You manually trigger a RescanFolders/RescanSeries/RescanMovie outside arr_full_rescan.sh + → arr_cache_write() correctly refuses to write while it's active (2026-07-17 — a mid-scan + write once read as real data loss to every consumer of the cache) + → nothing writes the real number through once the scan finishes unless something is + watching for completion — that's this tool + +arr_full_rescan.sh's weekly run finds one arr already mid-rescan and skips it + → that arr's cache stays stale until its own rescan finishes; run this to close the gap + without waiting for next week's scheduled run +``` + +### What It Does NOT Do + +Does not trigger a rescan — if none is active, it exits immediately with nothing to wait +for. Use the arr's own UI/API, or `Arrs_Stack/arr_full_rescan.sh`, to start one. + +### Usage + +```bash +# Wait for lidarr's active rescan (if any) to finish, then refresh its cache: +arr_rescan_monitor.sh lidarr + +# Override the default 2-hour wait ceiling (seconds): +arr_rescan_monitor.sh lidarr 3600 + +# Check whether a rescan is active and the cache's current age, no waiting: +arr_rescan_monitor.sh lidarr --status + +# Show what would be waited on without waiting or writing the cache: +arr_rescan_monitor.sh lidarr --dry-run +``` + +### Notes + +- Safe to run against sonarr/radarr the same way — arr_type is the only required arg +- If the wait times out (default 7200s), the cache is left untouched — re-run once the scan + actually finishes, or let it pick up naturally next time something calls + `arr_get_tracked_data()` and finds the cache stale with no active rescan +- Does not conflict with `arr_full_rescan.sh` — that script already refreshes its own cache + on completion for scans it triggers itself; this covers everything it didn't start + +--- + ## docker_prune_images.sh Removes orphaned Docker images that accumulate after container updates. Two modes: diff --git a/Tools/README-Tools.md b/Tools/README-Tools.md index c5cbb97..c924d70 100644 --- a/Tools/README-Tools.md +++ b/Tools/README-Tools.md @@ -49,6 +49,15 @@ inconsistent backup, and tar without stopping the container is equally unreliabl Fix: `container_data_export.sh` — stops the container cleanly, archives appdata to a timestamped `.tar.gz`, verifies archive integrity, restarts the container. +**Manually Kicked Off a Rescan and the Cache Never Caught Up** +Triggered a Lidarr `RescanFolders` by hand to fix a stats-drift problem. It ran for two +hours. `arr_cache_write()` correctly refuses to write while a rescan's active (2026-07-17 — +a mid-scan write once looked exactly like real data loss to every script trusting the +cache), but that means nothing writes the real post-scan number through once it finishes, +unless something was watching for completion. +Fix: `arr_rescan_monitor.sh` — waits for the arr's active rescan to finish, then refreshes +its cache with the real number. + --- ## ━━━ WHAT THIS FOLDER DOES ━━━ @@ -66,7 +75,7 @@ something new, write the tool. Store it here. Find it at 2am next time. `emby_database_repair.sh`, `zfs_pool_scrub.sh`, `smart_long_test.sh` **Repair Tools** — Fix a specific known problem -`bulk_permissions_repair.sh`, `arr_profile_enforcer.sh` +`bulk_permissions_repair.sh`, `arr_profile_enforcer.sh`, `arr_rescan_monitor.sh` **Lifecycle Tools** — Backup, setup, and migration support `container_data_export.sh`, `ramdisk_stop.sh`, `webhook_setup.sh` @@ -117,6 +126,7 @@ The relationship is one-way: Tools act on state that other scripts have written. | `ramdisk_stop.sh` | Safely stop the transcode ramdisk — redirect symlink to SSD, unmount, update state | Before re-running ramdisk_setup.sh with new size or thresholds | | `arr_profile_enforcer.sh` | Enforce correct quality profiles across all Sonarr/Radarr libraries | After arr setup, profile changes, or when library was imported with wrong profile | | `webhook_setup.sh` | Register the Varaverk upgrade webhook in Sonarr, Radarr, and Lidarr | After initial install or when adding a new arr or host | +| `arr_rescan_monitor.sh` | Shared tracked-data cache stuck stale after a manually-triggered rescan | After manually kicking off a RescanFolders/RescanSeries/RescanMovie outside arr_full_rescan.sh | | `emby_to_lidarr_sync.sh` | Add all Emby album artists not yet tracked in Lidarr | After Lidarr setup, database wipe, or when you suspect gaps | | `emby_to_sonarr_sync.sh` | Add all Emby TV series not yet tracked in Sonarr | After Sonarr setup, database wipe, or when you suspect gaps | | `emby_to_radarr_sync.sh` | Add all Emby movies not yet tracked in Radarr | After Radarr setup, database wipe, or when you suspect gaps | @@ -144,6 +154,7 @@ Situation arises │ arr_profile_enforcer ◄── wrong profiles after import or setup │ │ webhook_setup ◄── after install or adding a new arr │ │ ramdisk_stop ◄── before ramdisk resize / remount │ +│ arr_rescan_monitor ◄── after manually triggering a rescan │ │ │ │ emby_to_lidarr_sync ◄── Lidarr setup / database wipe / gap │ │ emby_to_sonarr_sync ◄── Sonarr setup / database wipe / gap │ diff --git a/Tools/arr_rescan_monitor.sh b/Tools/arr_rescan_monitor.sh new file mode 100755 index 0000000..abace99 --- /dev/null +++ b/Tools/arr_rescan_monitor.sh @@ -0,0 +1,97 @@ +#!/bin/bash +# ============================================================================================== +# ========================= Arr Rescan Monitor ================================================== +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Waits for an already-active rescan-type command (RescanFolders/RescanSeries/RescanMovie, +# DownloadedXScan, RefreshX — see ARR_RESCAN_COMMANDS in common.sh) on the given arr to finish, +# then refreshes the shared tracked-data cache (arr_cache_write) with the real post-scan +# numbers. Triggers nothing itself — for watching a rescan that was already started some +# other way (manual API call, arr's own UI, another script). +# +# Exists because arr_cache_write() now refuses to write while a rescan is active for that arr +# (2026-07-17 — a direct cache write mid-scan read as real data loss to every consumer of the +# cache, e.g. check_tracked_count_floor, instead of what it was: an in-flight scan). That +# guard protects the cache, but it also means nothing writes the real number through once the +# scan finishes unless something is watching for completion — arr_full_rescan.sh already does +# this for scans it triggers itself; this tool covers everything else. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# arr_rescan_monitor.sh lidarr +# Wait for lidarr's active rescan (if any) to finish, then refresh its cache. +# Exits immediately if no rescan is currently active — nothing to wait for. +# +# arr_rescan_monitor.sh lidarr 3600 +# Override the default 2-hour wait ceiling (second positional arg, seconds). +# +# arr_rescan_monitor.sh lidarr --status +# Show whether a rescan is currently active and the cache's current age, no waiting. +# +# arr_rescan_monitor.sh lidarr --dry-run +# Show what would be waited on without actually waiting or writing the cache. +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../load_config.sh" +parse_args "$@" +detect_hosts + +ARR_TYPE="${PARSED_ARGS[0]:-}" +TIMEOUT="${PARSED_ARGS[1]:-7200}" + +case "$ARR_TYPE" in + lidarr|sonarr|radarr) ;; + *) error "Usage: arr_rescan_monitor.sh lidarr|sonarr|radarr [timeout_seconds] [--status] [--dry-run]"; exit 1 ;; +esac + +url_var="${ARR_TYPE^^}_URL"; key_var="${ARR_TYPE^^}_API_KEY" +URL="${!url_var:-}"; KEY="${!key_var:-}" +VER="${ARR_API_VERSION[$ARR_TYPE]}" + +require_var "$url_var" +require_var "$key_var" + +ACTIVE=$(arr_active_rescan_command "$ARR_TYPE" "$URL" "$KEY" "$VER") +CACHE_AGE=$(arr_cache_age_seconds "$ARR_TYPE") + +if [[ "$SHOW_STATUS" == true ]]; then + info "${ARR_TYPE^} active rescan: ${ACTIVE:-none}" + info "${ARR_TYPE^} cache age: $(( CACHE_AGE / 60 )) minutes" + exit 0 +fi + +if [[ -z "$ACTIVE" ]]; then + info "No active rescan on ${ARR_TYPE^} — nothing to wait for." + exit 0 +fi + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would wait for ${ARR_TYPE^}'s active $ACTIVE (timeout ${TIMEOUT}s), then refresh its cache" + exit 0 +fi + +info "Waiting for ${ARR_TYPE^}'s active $ACTIVE to finish (timeout ${TIMEOUT}s)..." +if ! arr_wait_for_active_rescan "$ARR_TYPE" "$URL" "$KEY" "$VER" "$TIMEOUT"; then + error "Timed out after ${TIMEOUT}s waiting for ${ARR_TYPE^}'s rescan — cache not refreshed. Re-run once it finishes, or it'll pick up naturally next time something calls arr_get_tracked_data()." + exit 1 +fi + +ENDPOINT="${ARR_LIBRARY_ENDPOINT[$ARR_TYPE]}" +EXPR="${ARR_TRACKED_COUNT_EXPR[$ARR_TYPE]}" +ITEMS=$(curl -sf --max-time 60 -H "X-Api-Key: $KEY" "${URL}/api/${VER}/${ENDPOINT}" 2>/dev/null) +TOTAL=$(echo "$ITEMS" | jq "$EXPR" 2>/dev/null) + +if [[ -z "$ITEMS" || -z "$TOTAL" || "$TOTAL" == "null" ]]; then + error "${ARR_TYPE^} rescan finished but couldn't fetch fresh library data — cache not refreshed" + exit 1 +fi + +arr_cache_write "$ARR_TYPE" "$ITEMS" +success "${ARR_TYPE^} rescan finished — cache refreshed, tracked: ${TOTAL}" +notify "${ARR_TYPE^} rescan finished on $(hostname) — cache refreshed, tracked: ${TOTAL}" "Arr Rescan Monitor" "normal" diff --git a/common.sh b/common.sh index edf15b8..0f5177b 100755 --- a/common.sh +++ b/common.sh @@ -2287,6 +2287,13 @@ declare -gA ARR_LIBRARY_ENDPOINT=( [radarr]="movie" ) +# API version per arr — Lidarr is still v1, Sonarr/Radarr are v3. +declare -gA ARR_API_VERSION=( + [lidarr]="v1" + [sonarr]="v3" + [radarr]="v3" +) + arr_cache_file() { echo "${DATA_DIR}/${1}_tracked_cache.json"; } arr_rescan_duration_db() { echo "${DATA_DIR}/${1}_rescan_duration.db"; } @@ -2296,11 +2303,31 @@ arr_rescan_duration_db() { echo "${DATA_DIR}/${1}_rescan_duration.db"; } # same class of bug the queue-pagination fix (2026-07-16, arrs_failed_stalled_recovery.sh) # hit before. Fixed the same way: write the payload to a temp file and use --slurpfile, # which reads from disk instead of argv. +# +# Refuses to write while a rescan-type command is active for arr_type (2026-07-17). Every +# direct caller here fetches the library list for its own purposes and writes it through as +# a side effect — none of them go through arr_get_tracked_data()'s fresh/stale/active-rescan +# branching, so none of them knew to check scan state first. Confirmed live: a caller wrote +# Lidarr's cache mid-RescanFolders at 22% of the real total, and that partial number then +# looked exactly like real data loss to every consumer (check_tracked_count_floor et al.) +# until the scan finished. The guard lives here instead of in each caller so it applies +# uniformly — last-known-good stays in place until something writes through the real +# post-scan number (arr_full_rescan.sh does this itself on completion; for a rescan someone +# else triggered, see arr_rescan_monitor.sh in Tools/). # Args: arr_type, items_json (the full library-list response body) arr_cache_write() { local arr_type="$1" items_json="$2" local expr="${ARR_TRACKED_COUNT_EXPR[$arr_type]:-}" [[ -z "$expr" ]] && return 1 + + local url_var="${arr_type^^}_URL" key_var="${arr_type^^}_API_KEY" + local url="${!url_var:-}" api_key="${!key_var:-}" api_version="${ARR_API_VERSION[$arr_type]:-}" + if [[ -n "$url" && -n "$api_key" && -n "$api_version" ]]; then + local active_cmd + active_cmd=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version") + [[ -n "$active_cmd" ]] && return 1 + fi + local total total=$(echo "$items_json" | jq "$expr" 2>/dev/null) [[ -z "$total" || "$total" == "null" ]] && return 1 @@ -2388,6 +2415,29 @@ arr_active_rescan_command() { )] | .[0].name // empty' 2>/dev/null } +# Waits for an already-active rescan-type command for arr_type to finish — unlike +# trigger_and_await_command(), this never triggers anything, it only watches. For a rescan +# someone/something else started (manual intervention, another script) where the caller just +# needs to know when it's safe to fetch+cache the real post-scan numbers. Returns 0 once no +# active rescan-type command remains (including immediately if none was active to begin +# with), 1 on timeout. +# Args: arr_type, url, api_key, api_version, poll_timeout (default 7200s), poll_interval (default 15s) +arr_wait_for_active_rescan() { + local arr_type="$1" url="$2" api_key="$3" api_version="$4" + local poll_timeout="${5:-7200}" poll_interval="${6:-15}" + local polled=0 active + active=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version") + [[ -z "$active" ]] && return 0 + while [[ "$polled" -lt "$poll_timeout" ]]; do + sleep "$poll_interval" + (( polled += poll_interval )) + active=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version") + [[ -z "$active" ]] && return 0 + [[ $(( polled % 300 )) -lt "$poll_interval" ]] && log " Still waiting on ${arr_type}'s ${active} (${polled}s elapsed)" + done + return 1 +} + # Main entry point — the only function consuming scripts should call for tracked library # data. Handles fresh/stale-no-rescan/stale-rescan-active branching and always writes through # on any live fetch it performs. Echoes the library-list JSON array on success, returns 1 if