Generalize tracked-data cache from Lidarr-only to all three arrs
Shared cache/rescan-duration logic in common.sh now takes an arr_type param instead of being Lidarr-specific, so Sonarr and Radarr cleanup scripts get the same cache-first fetch + rescan-aware retry Lidarr had. Avoids redundant full-library API calls across scripts run back to back, and stops false failures when a fetch lands mid-rescan.
This commit is contained in:
Gmer4Lfe
@@ -2176,16 +2176,19 @@ arr_api() {
|
||||
# an unreachable/timed-out scan as fatal, matching original per-script behavior.
|
||||
#
|
||||
# On a genuine "completed" observation, records the actual elapsed duration keyed by the
|
||||
# command's own name via lidarr_record_rescan_duration() — this is the one place in the
|
||||
# codebase that reliably watches a command from trigger to completion, so it's the natural
|
||||
# spot to build up real historical duration data for the wait-calibration logic in
|
||||
# lidarr_get_tracked_data(). A timeout doesn't record anything — we only know a lower bound,
|
||||
# not the true duration, and recording that would corrupt future wait calculations downward.
|
||||
# command's own name via arr_record_rescan_duration() (needs arr_type to know which arr's
|
||||
# duration DB to write to) — this is the one place in the codebase that reliably watches a
|
||||
# command from trigger to completion, so it's the natural spot to build up real historical
|
||||
# duration data for the wait-calibration logic in arr_get_tracked_data(). A timeout doesn't
|
||||
# record anything — we only know a lower bound, not the true duration, and recording that
|
||||
# would corrupt future wait calculations downward. arr_type is optional — omit it (or pass
|
||||
# empty) to skip duration recording entirely, e.g. for one-off commands that aren't one of
|
||||
# the three tracked arrs.
|
||||
#
|
||||
# Usage: trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" \
|
||||
# '{"name": "DownloadedAlbumsScan"}' "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}"
|
||||
# '{"name": "DownloadedAlbumsScan"}' "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}" "lidarr"
|
||||
trigger_and_await_command() {
|
||||
local base_url="$1" api_key="$2" api_version="$3" payload="$4" poll_timeout="${5:-600}"
|
||||
local base_url="$1" api_key="$2" api_version="$3" payload="$4" poll_timeout="${5:-600}" arr_type="${6:-}"
|
||||
|
||||
local cmd_name
|
||||
cmd_name=$(echo "$payload" | jq -r '.name // empty' 2>/dev/null)
|
||||
@@ -2215,7 +2218,8 @@ trigger_and_await_command() {
|
||||
case "$scan_status" in
|
||||
completed)
|
||||
info "Import scan complete ✅"
|
||||
[[ -n "$cmd_name" ]] && lidarr_record_rescan_duration "$cmd_name" "$(( $(date +%s) - start_epoch ))"
|
||||
[[ -n "$cmd_name" && -n "$arr_type" ]] && \
|
||||
arr_record_rescan_duration "$arr_type" "$cmd_name" "$(( $(date +%s) - start_epoch ))"
|
||||
return 0
|
||||
;;
|
||||
failed)
|
||||
@@ -2232,146 +2236,208 @@ trigger_and_await_command() {
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── LIDARR TRACKED-DATA CACHE ─────────────────────────────────────────────────────────────────
|
||||
# ── ARR TRACKED-DATA CACHE (Lidarr, Sonarr, Radarr) ──────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Shared cache for Lidarr's tracked-artist/file-count data. Multiple scripts (lidarr_cleanup.sh,
|
||||
# lidarr_duplicate_artist_cleanup.sh, lidarr_missing_art.sh, lidarr_release_fixer.sh) all need a
|
||||
# reasonably-current snapshot of "what does Lidarr think it has tracked." Hitting the live API
|
||||
# fresh every time is wasteful, and during an active rescan the live number is actively
|
||||
# misleading — tracked counts dip and recover as files are detached/re-verified one by one
|
||||
# (confirmed 2026-07-16: a whole-library RescanFolders made trackFileCount read 22% of normal
|
||||
# mid-scan, which is exactly the false-alarm lidarr_cleanup.sh's count-drop guard is meant to
|
||||
# catch, but a genuine rescan isn't the "something's actually wrong" case that guard exists for).
|
||||
# Shared cache for each arr's tracked-library data (Lidarr artists, Sonarr series, Radarr
|
||||
# movies). Multiple scripts across all three arrs need a reasonably-current snapshot of "what
|
||||
# does this arr think it has tracked." Hitting the live API fresh every time is wasteful, and
|
||||
# during an active rescan the live number is actively misleading — tracked counts dip and
|
||||
# recover as files are detached/re-verified one by one (confirmed 2026-07-16 on Lidarr: a
|
||||
# whole-library RescanFolders made trackFileCount read 22% of normal mid-scan, which is
|
||||
# exactly the false-alarm lidarr_cleanup.sh's count-drop guard is meant to catch, but a
|
||||
# genuine rescan isn't the "something's actually wrong" case that guard exists for — the same
|
||||
# risk applies to Sonarr's RescanSeries and Radarr's RescanMovie).
|
||||
#
|
||||
# Write-through: any script that already does a live artist-list fetch for its own purposes
|
||||
# writes the result here as a side effect via lidarr_cache_write() — no dedicated polling timer
|
||||
# needed. Arrs_Stack/lidarr_cache_prefill.sh closes the cold-boot gap by populating the cache
|
||||
# once at array start, before anything else needs it.
|
||||
# Built Lidarr-only first (2026-07-16), generalized the same day to cover all three arrs —
|
||||
# identical mechanism, keyed by arr_type ("lidarr"/"sonarr"/"radarr") so each arr's cache and
|
||||
# duration history stay separate.
|
||||
#
|
||||
# Consumers should call lidarr_get_tracked_data() — never read the cache file directly. It
|
||||
# handles the fresh/stale-no-rescan/stale-rescan-active branching so no script reimplements it.
|
||||
# Write-through: any script that already does a live library-list fetch for its own purposes
|
||||
# writes the result here as a side effect via arr_cache_write() — no dedicated polling timer
|
||||
# needed. Arrs_Stack/arr_cache_prefill.sh closes the cold-boot gap by populating all three
|
||||
# caches once at array start, before anything else needs them.
|
||||
#
|
||||
# Consumers should call arr_get_tracked_data() — never read a cache file directly. It handles
|
||||
# the fresh/stale-no-rescan/stale-rescan-active branching so no script reimplements it.
|
||||
# ==============================================================================================
|
||||
|
||||
LIDARR_CACHE_FILE="${LIDARR_CACHE_FILE:-$DATA_DIR/lidarr_tracked_cache.json}"
|
||||
LIDARR_RESCAN_DURATION_DB="${LIDARR_RESCAN_DURATION_DB:-$DATA_DIR/lidarr_rescan_duration.db}"
|
||||
# Rescan-type command names per arr — operations long/heavy enough that overlapping with one
|
||||
# should trigger a wait rather than a live fetch. Verified live against each arr's API
|
||||
# 2026-07-16 (not assumed from memory — RescanSeries/RescanMovie confirmed to scan the whole
|
||||
# library when given no id, matching Lidarr's RescanFolders).
|
||||
declare -gA ARR_RESCAN_COMMANDS=(
|
||||
[lidarr]="RescanFolders DownloadedAlbumsScan RefreshArtist"
|
||||
[sonarr]="RescanSeries DownloadedEpisodesScan RefreshSeries"
|
||||
[radarr]="RescanMovie DownloadedMoviesScan RefreshMovie"
|
||||
)
|
||||
|
||||
# Writes the current artist list + total tracked count to the shared cache.
|
||||
# Args: artists_json (the full /api/v1/artist response body)
|
||||
lidarr_cache_write() {
|
||||
local artists_json="$1"
|
||||
local total_files
|
||||
total_files=$(echo "$artists_json" | jq '[.[].statistics.trackFileCount] | add' 2>/dev/null)
|
||||
[[ -z "$total_files" || "$total_files" == "null" ]] && return 1
|
||||
# jq expression computing each arr's "total tracked" number from its library-list response —
|
||||
# schemas differ: Lidarr/Sonarr have a per-item file count to sum, Radarr is a binary hasFile
|
||||
# per movie (one file at most), so the aggregate has to be a count of true values instead.
|
||||
declare -gA ARR_TRACKED_COUNT_EXPR=(
|
||||
[lidarr]='[.[].statistics.trackFileCount] | add'
|
||||
[sonarr]='[.[].statistics.episodeFileCount] | add'
|
||||
[radarr]='[.[] | select(.hasFile==true)] | length'
|
||||
)
|
||||
|
||||
mkdir -p "$(dirname "$LIDARR_CACHE_FILE")" 2>/dev/null
|
||||
jq -c -n --argjson artists "$artists_json" --argjson total "$total_files" --argjson ts "$(date +%s)" \
|
||||
'{ts:$ts, totalTrackedFiles:$total, artists:$artists}' > "${LIDARR_CACHE_FILE}.tmp" 2>/dev/null \
|
||||
&& mv "${LIDARR_CACHE_FILE}.tmp" "$LIDARR_CACHE_FILE"
|
||||
# Library-list endpoint name per arr, for arr_api().
|
||||
declare -gA ARR_LIBRARY_ENDPOINT=(
|
||||
[lidarr]="artist"
|
||||
[sonarr]="series"
|
||||
[radarr]="movie"
|
||||
)
|
||||
|
||||
arr_cache_file() { echo "${DATA_DIR}/${1}_tracked_cache.json"; }
|
||||
arr_rescan_duration_db() { echo "${DATA_DIR}/${1}_rescan_duration.db"; }
|
||||
|
||||
# Writes the current library-list JSON + computed total tracked count to arr_type's cache.
|
||||
# Radarr's movie list alone runs ~16MB — passing that through jq's --argjson as a literal
|
||||
# command-line argument blows past the OS's ARG_MAX ("Argument list too long"), the exact
|
||||
# same class of bug the queue-pagination fix (2026-07-16, arrs_failed_stalled_recovery.sh)
|
||||
# hit before. Fixed the same way: write the payload to a temp file and use --slurpfile,
|
||||
# which reads from disk instead of argv.
|
||||
# Args: arr_type, items_json (the full library-list response body)
|
||||
arr_cache_write() {
|
||||
local arr_type="$1" items_json="$2"
|
||||
local expr="${ARR_TRACKED_COUNT_EXPR[$arr_type]:-}"
|
||||
[[ -z "$expr" ]] && return 1
|
||||
local total
|
||||
total=$(echo "$items_json" | jq "$expr" 2>/dev/null)
|
||||
[[ -z "$total" || "$total" == "null" ]] && return 1
|
||||
|
||||
local cache_file items_tmp
|
||||
cache_file=$(arr_cache_file "$arr_type")
|
||||
items_tmp=$(mktemp)
|
||||
echo "$items_json" > "$items_tmp"
|
||||
mkdir -p "$(dirname "$cache_file")" 2>/dev/null
|
||||
jq -c -n --slurpfile items "$items_tmp" --argjson total "$total" --argjson ts "$(date +%s)" \
|
||||
'{ts:$ts, totalTracked:$total, items:$items[0]}' > "${cache_file}.tmp" 2>/dev/null \
|
||||
&& mv "${cache_file}.tmp" "$cache_file"
|
||||
rm -f "$items_tmp"
|
||||
}
|
||||
|
||||
# Echoes the cached artists JSON array if a cache file exists, regardless of age — staleness
|
||||
# is lidarr_get_tracked_data()'s decision, not this raw reader's.
|
||||
lidarr_cache_read_raw() {
|
||||
[[ -f "$LIDARR_CACHE_FILE" ]] || return 1
|
||||
jq -c '.artists // empty' "$LIDARR_CACHE_FILE" 2>/dev/null
|
||||
# Echoes the cached library-list JSON array for arr_type if a cache file exists, regardless
|
||||
# of age — staleness is arr_get_tracked_data()'s decision, not this raw reader's.
|
||||
arr_cache_read_raw() {
|
||||
local arr_type="$1"
|
||||
local cache_file
|
||||
cache_file=$(arr_cache_file "$arr_type")
|
||||
[[ -f "$cache_file" ]] || return 1
|
||||
jq -c '.items // empty' "$cache_file" 2>/dev/null
|
||||
}
|
||||
|
||||
# Echoes the cache's age in seconds, or a very large number if no cache exists (so age
|
||||
# Echoes arr_type's cache age in seconds, or a very large number if no cache exists (so age
|
||||
# comparisons naturally treat "no cache" the same as "very stale cache").
|
||||
lidarr_cache_age_seconds() {
|
||||
[[ -f "$LIDARR_CACHE_FILE" ]] || { echo 999999999; return; }
|
||||
arr_cache_age_seconds() {
|
||||
local arr_type="$1"
|
||||
local cache_file
|
||||
cache_file=$(arr_cache_file "$arr_type")
|
||||
[[ -f "$cache_file" ]] || { echo 999999999; return; }
|
||||
local ts
|
||||
ts=$(jq -r '.ts // 0' "$LIDARR_CACHE_FILE" 2>/dev/null)
|
||||
ts=$(jq -r '.ts // 0' "$cache_file" 2>/dev/null)
|
||||
echo $(( $(date +%s) - ${ts:-0} ))
|
||||
}
|
||||
|
||||
# Records how long a rescan-type command actually took, keyed by command name, so future
|
||||
# waits can be calibrated per command type instead of guessed or blended across very
|
||||
# different operations — a whole-library RescanFolders takes vastly longer than a targeted
|
||||
# DownloadedAlbumsScan, and averaging them would miscalibrate the wait for both.
|
||||
# Args: command_name, duration_seconds
|
||||
lidarr_record_rescan_duration() {
|
||||
local cmd_name="$1" duration="$2"
|
||||
# Records how long a rescan-type command actually took for arr_type, keyed by command name,
|
||||
# so future waits can be calibrated per command type instead of guessed or blended across
|
||||
# very different operations — a whole-library RescanFolders/RescanSeries/RescanMovie takes
|
||||
# vastly longer than a targeted DownloadedXScan, and averaging them would miscalibrate the
|
||||
# wait for both.
|
||||
# Args: arr_type, command_name, duration_seconds
|
||||
arr_record_rescan_duration() {
|
||||
local arr_type="$1" cmd_name="$2" duration="$3"
|
||||
[[ -z "$cmd_name" || -z "$duration" ]] && return 1
|
||||
mkdir -p "$(dirname "$LIDARR_RESCAN_DURATION_DB")" 2>/dev/null
|
||||
local db
|
||||
db=$(arr_rescan_duration_db "$arr_type")
|
||||
mkdir -p "$(dirname "$db")" 2>/dev/null
|
||||
local tmp
|
||||
tmp=$(mktemp)
|
||||
[[ -f "$LIDARR_RESCAN_DURATION_DB" ]] && grep -v "^${cmd_name}|" "$LIDARR_RESCAN_DURATION_DB" > "$tmp" 2>/dev/null
|
||||
[[ -f "$db" ]] && grep -v "^${cmd_name}|" "$db" > "$tmp" 2>/dev/null
|
||||
echo "${cmd_name}|${duration}" >> "$tmp"
|
||||
mv "$tmp" "$LIDARR_RESCAN_DURATION_DB"
|
||||
mv "$tmp" "$db"
|
||||
}
|
||||
|
||||
# Echoes the last recorded duration (seconds) for a given command name, or a fallback
|
||||
# Echoes the last recorded duration (seconds) for arr_type + command name, or a fallback
|
||||
# default if none has ever been recorded.
|
||||
# Args: command_name, fallback_default_seconds
|
||||
lidarr_get_rescan_duration() {
|
||||
local cmd_name="$1" fallback="${2:-300}"
|
||||
[[ -f "$LIDARR_RESCAN_DURATION_DB" ]] || { echo "$fallback"; return; }
|
||||
# Args: arr_type, command_name, fallback_default_seconds
|
||||
arr_get_rescan_duration() {
|
||||
local arr_type="$1" cmd_name="$2" fallback="${3:-300}"
|
||||
local db
|
||||
db=$(arr_rescan_duration_db "$arr_type")
|
||||
[[ -f "$db" ]] || { echo "$fallback"; return; }
|
||||
local val
|
||||
val=$(grep "^${cmd_name}|" "$LIDARR_RESCAN_DURATION_DB" 2>/dev/null | tail -1 | cut -d'|' -f2)
|
||||
val=$(grep "^${cmd_name}|" "$db" 2>/dev/null | tail -1 | cut -d'|' -f2)
|
||||
echo "${val:-$fallback}"
|
||||
}
|
||||
|
||||
# Checks Lidarr's command queue for any currently-active (started or queued) rescan-type
|
||||
# command. Echoes the command name of the first one found (so the caller can look up its
|
||||
# specific historical duration), empty if none active.
|
||||
# Args: url, api_key, api_version
|
||||
lidarr_active_rescan_command() {
|
||||
local url="$1" api_key="$2" api_version="$3"
|
||||
# Checks arr_type's command queue for any currently-active (started or queued) rescan-type
|
||||
# command (per ARR_RESCAN_COMMANDS). Echoes the command name of the first one found (so the
|
||||
# caller can look up its specific historical duration), empty if none active.
|
||||
# Args: arr_type, url, api_key, api_version
|
||||
arr_active_rescan_command() {
|
||||
local arr_type="$1" url="$2" api_key="$3" api_version="$4"
|
||||
local names="${ARR_RESCAN_COMMANDS[$arr_type]:-}"
|
||||
[[ -z "$names" ]] && return 1
|
||||
local jq_names
|
||||
jq_names=$(printf '%s\n' $names | jq -R . | jq -sc .)
|
||||
curl -sf --max-time 15 -H "X-Api-Key: $api_key" \
|
||||
"${url}/api/${api_version}/command" 2>/dev/null | \
|
||||
jq -r '[.[] | select(
|
||||
jq -r --argjson names "$jq_names" '[.[] | select(
|
||||
(.status=="started" or .status=="queued") and
|
||||
(.name=="RescanFolders" or .name=="DownloadedAlbumsScan" or .name=="RefreshArtist")
|
||||
(.name as $n | $names | index($n) != null)
|
||||
)] | .[0].name // empty' 2>/dev/null
|
||||
}
|
||||
|
||||
# Main entry point — the only function consuming scripts should call for tracked artist data.
|
||||
# Handles fresh/stale-no-rescan/stale-rescan-active branching and always writes through on any
|
||||
# live fetch it performs. Echoes the artists JSON array on success, returns 1 if no usable data
|
||||
# (no cache and live fetch impossible) could be obtained.
|
||||
# Args: url, api_key, api_version, max_age_days (default 1), max_wait_strikes (default 3)
|
||||
lidarr_get_tracked_data() {
|
||||
local url="$1" api_key="$2" api_version="$3"
|
||||
local max_age_days="${4:-${LIDARR_CACHE_MAX_AGE_DAYS:-1}}" max_strikes="${5:-3}"
|
||||
# Main entry point — the only function consuming scripts should call for tracked library
|
||||
# data. Handles fresh/stale-no-rescan/stale-rescan-active branching and always writes through
|
||||
# on any live fetch it performs. Echoes the library-list JSON array on success, returns 1 if
|
||||
# no usable data (no cache and live fetch impossible) could be obtained.
|
||||
# Args: arr_type, url, api_key, api_version, max_age_days (default 1), max_wait_strikes (default 3)
|
||||
arr_get_tracked_data() {
|
||||
local arr_type="$1" url="$2" api_key="$3" api_version="$4"
|
||||
local max_age_days="${5:-${LIDARR_CACHE_MAX_AGE_DAYS:-1}}" max_strikes="${6:-3}"
|
||||
local max_age_seconds=$(( max_age_days * 86400 ))
|
||||
local age
|
||||
age=$(lidarr_cache_age_seconds)
|
||||
age=$(arr_cache_age_seconds "$arr_type")
|
||||
|
||||
if [[ "$age" -lt "$max_age_seconds" ]]; then
|
||||
lidarr_cache_read_raw && return 0
|
||||
arr_cache_read_raw "$arr_type" && return 0
|
||||
fi
|
||||
|
||||
# Cache missing or stale — check for an active rescan before deciding how to proceed.
|
||||
local active_cmd
|
||||
active_cmd=$(lidarr_active_rescan_command "$url" "$api_key" "$api_version")
|
||||
active_cmd=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version")
|
||||
|
||||
if [[ -n "$active_cmd" ]]; then
|
||||
local wait_duration strike
|
||||
wait_duration=$(( $(lidarr_get_rescan_duration "$active_cmd" 300) / 2 ))
|
||||
local wait_duration strike arr_label
|
||||
wait_duration=$(( $(arr_get_rescan_duration "$arr_type" "$active_cmd" 300) / 2 ))
|
||||
[[ "$wait_duration" -lt 30 ]] && wait_duration=30
|
||||
arr_label="$(tr '[:lower:]' '[:upper:]' <<< "${arr_type:0:1}")${arr_type:1}"
|
||||
for (( strike=1; strike<=max_strikes; strike++ )); do
|
||||
# Redirected to stderr — this function's stdout is a data channel (callers
|
||||
# capture it via command substitution), never mix log/warn output into it.
|
||||
warn " Lidarr busy ($active_cmd) — waiting ${wait_duration}s (strike ${strike}/${max_strikes})" >&2
|
||||
warn " $arr_label busy ($active_cmd) — waiting ${wait_duration}s (strike ${strike}/${max_strikes})" >&2
|
||||
sleep "$wait_duration"
|
||||
active_cmd=$(lidarr_active_rescan_command "$url" "$api_key" "$api_version")
|
||||
active_cmd=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version")
|
||||
[[ -z "$active_cmd" ]] && break
|
||||
done
|
||||
if [[ -n "$active_cmd" ]]; then
|
||||
warn " Lidarr still busy ($active_cmd) after ${max_strikes} strikes — using cache if present, else skipping" >&2
|
||||
lidarr_cache_read_raw && return 0
|
||||
warn " $arr_label still busy ($active_cmd) after ${max_strikes} strikes — using cache if present, else skipping" >&2
|
||||
arr_cache_read_raw "$arr_type" && return 0
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# No active rescan (or it just finished mid-wait) — safe to fetch live and refresh cache.
|
||||
local fresh
|
||||
fresh=$(arr_api "$url" "$api_key" "$api_version" "artist" "Lidarr") || {
|
||||
lidarr_cache_read_raw && return 0
|
||||
local endpoint fresh arr_label
|
||||
endpoint="${ARR_LIBRARY_ENDPOINT[$arr_type]:-}"
|
||||
[[ -z "$endpoint" ]] && return 1
|
||||
arr_label="$(tr '[:lower:]' '[:upper:]' <<< "${arr_type:0:1}")${arr_type:1}"
|
||||
fresh=$(arr_api "$url" "$api_key" "$api_version" "$endpoint" "$arr_label") || {
|
||||
arr_cache_read_raw "$arr_type" && return 0
|
||||
return 1
|
||||
}
|
||||
lidarr_cache_write "$fresh"
|
||||
arr_cache_write "$arr_type" "$fresh"
|
||||
echo "$fresh"
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user