Add Lidarr tracked-data cache + duplicate artist cleanup
Shared cache (lidarr_get_tracked_data() in common.sh) so scripts stop hitting Lidarr's live API for tracked counts every run, and stop treating a mid-rescan dip as a genuine problem — a whole-library RescanFolders legitimately makes trackFileCount read far below normal while it re-verifies every file (confirmed 2026-07-16: 22% of normal mid-scan). Cache reads fresh-if-recent, waits out an active rescan (calibrated to that command's own historical duration, tracked per command name since RescanFolders and DownloadedAlbumsScan take wildly different amounts of time), then falls back to a stale cache rather than hard-failing after a few strikes. lidarr_cleanup.sh: no longer stacks a fresh DownloadedAlbumsScan on top of one already running, and the tracked-count floor check now waits out a genuine rescan instead of aborting on every overlap. lidarr_duplicate_artist_cleanup.sh (new): finds case-insensitive duplicate artist entries — same display name, different MusicBrainz ID, added when a search/list-sync matches the wrong same-named artist. Deletes the empty phantom side and blocks it from Import List Exclusions, leaves genuinely-different-real-artists alone (checked by album title overlap, deduped per-artist first so a legitimate reissue under an artist's own catalog doesn't false-flag as cross-artist overlap), and only notifies for the rare case where both sides have real, overlapping content. lidarr_cache_prefill.sh (new): warms the cache at array start so nothing reads it cold after boot. lidarr_missing_art.sh, lidarr_release_fixer.sh: write-through the cache as a side effect of fetches they already needed for their own purposes.
This commit is contained in:
Gmer4Lfe
@@ -2174,11 +2174,22 @@ arr_api() {
|
||||
# Triggers an arr "command" endpoint with the given JSON payload, then polls every 10s until
|
||||
# completed/failed/timeout, logging progress every 60s. Best-effort pre-flight — never treats
|
||||
# an unreachable/timed-out scan as fatal, matching original per-script behavior.
|
||||
#
|
||||
# On a genuine "completed" observation, records the actual elapsed duration keyed by the
|
||||
# command's own name via lidarr_record_rescan_duration() — this is the one place in the
|
||||
# codebase that reliably watches a command from trigger to completion, so it's the natural
|
||||
# spot to build up real historical duration data for the wait-calibration logic in
|
||||
# lidarr_get_tracked_data(). A timeout doesn't record anything — we only know a lower bound,
|
||||
# not the true duration, and recording that would corrupt future wait calculations downward.
|
||||
#
|
||||
# Usage: trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" \
|
||||
# '{"name": "DownloadedAlbumsScan"}' "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}"
|
||||
trigger_and_await_command() {
|
||||
local base_url="$1" api_key="$2" api_version="$3" payload="$4" poll_timeout="${5:-600}"
|
||||
|
||||
local cmd_name
|
||||
cmd_name=$(echo "$payload" | jq -r '.name // empty' 2>/dev/null)
|
||||
|
||||
local scan_response scan_cmd_id
|
||||
scan_response=$(curl -sf --max-time 30 -X POST \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
@@ -2194,6 +2205,7 @@ trigger_and_await_command() {
|
||||
fi
|
||||
|
||||
info "Import scan queued (command ID: $scan_cmd_id) — waiting for completion..."
|
||||
local start_epoch=$(date +%s)
|
||||
local polled=0 scan_status
|
||||
while [[ "$polled" -lt "$poll_timeout" ]]; do
|
||||
scan_status=$(curl -sf --max-time 10 \
|
||||
@@ -2201,8 +2213,15 @@ trigger_and_await_command() {
|
||||
"${base_url}/api/${api_version}/command/${scan_cmd_id}" 2>/dev/null | \
|
||||
jq -r '.status // empty' 2>/dev/null)
|
||||
case "$scan_status" in
|
||||
completed) info "Import scan complete ✅"; return 0 ;;
|
||||
failed) warn "Import scan reported failed — proceeding anyway"; return 0 ;;
|
||||
completed)
|
||||
info "Import scan complete ✅"
|
||||
[[ -n "$cmd_name" ]] && lidarr_record_rescan_duration "$cmd_name" "$(( $(date +%s) - start_epoch ))"
|
||||
return 0
|
||||
;;
|
||||
failed)
|
||||
warn "Import scan reported failed — proceeding anyway"
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
sleep 10
|
||||
(( polled += 10 ))
|
||||
@@ -2212,6 +2231,150 @@ trigger_and_await_command() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── LIDARR TRACKED-DATA CACHE ─────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Shared cache for Lidarr's tracked-artist/file-count data. Multiple scripts (lidarr_cleanup.sh,
|
||||
# lidarr_duplicate_artist_cleanup.sh, lidarr_missing_art.sh, lidarr_release_fixer.sh) all need a
|
||||
# reasonably-current snapshot of "what does Lidarr think it has tracked." Hitting the live API
|
||||
# fresh every time is wasteful, and during an active rescan the live number is actively
|
||||
# misleading — tracked counts dip and recover as files are detached/re-verified one by one
|
||||
# (confirmed 2026-07-16: a whole-library RescanFolders made trackFileCount read 22% of normal
|
||||
# mid-scan, which is exactly the false-alarm lidarr_cleanup.sh's count-drop guard is meant to
|
||||
# catch, but a genuine rescan isn't the "something's actually wrong" case that guard exists for).
|
||||
#
|
||||
# Write-through: any script that already does a live artist-list fetch for its own purposes
|
||||
# writes the result here as a side effect via lidarr_cache_write() — no dedicated polling timer
|
||||
# needed. Arrs_Stack/lidarr_cache_prefill.sh closes the cold-boot gap by populating the cache
|
||||
# once at array start, before anything else needs it.
|
||||
#
|
||||
# Consumers should call lidarr_get_tracked_data() — never read the cache file directly. It
|
||||
# handles the fresh/stale-no-rescan/stale-rescan-active branching so no script reimplements it.
|
||||
# ==============================================================================================
|
||||
|
||||
LIDARR_CACHE_FILE="${LIDARR_CACHE_FILE:-$DATA_DIR/lidarr_tracked_cache.json}"
|
||||
LIDARR_RESCAN_DURATION_DB="${LIDARR_RESCAN_DURATION_DB:-$DATA_DIR/lidarr_rescan_duration.db}"
|
||||
|
||||
# Writes the current artist list + total tracked count to the shared cache.
|
||||
# Args: artists_json (the full /api/v1/artist response body)
|
||||
lidarr_cache_write() {
|
||||
local artists_json="$1"
|
||||
local total_files
|
||||
total_files=$(echo "$artists_json" | jq '[.[].statistics.trackFileCount] | add' 2>/dev/null)
|
||||
[[ -z "$total_files" || "$total_files" == "null" ]] && return 1
|
||||
|
||||
mkdir -p "$(dirname "$LIDARR_CACHE_FILE")" 2>/dev/null
|
||||
jq -c -n --argjson artists "$artists_json" --argjson total "$total_files" --argjson ts "$(date +%s)" \
|
||||
'{ts:$ts, totalTrackedFiles:$total, artists:$artists}' > "${LIDARR_CACHE_FILE}.tmp" 2>/dev/null \
|
||||
&& mv "${LIDARR_CACHE_FILE}.tmp" "$LIDARR_CACHE_FILE"
|
||||
}
|
||||
|
||||
# Echoes the cached artists JSON array if a cache file exists, regardless of age — staleness
|
||||
# is lidarr_get_tracked_data()'s decision, not this raw reader's.
|
||||
lidarr_cache_read_raw() {
|
||||
[[ -f "$LIDARR_CACHE_FILE" ]] || return 1
|
||||
jq -c '.artists // empty' "$LIDARR_CACHE_FILE" 2>/dev/null
|
||||
}
|
||||
|
||||
# Echoes the cache's age in seconds, or a very large number if no cache exists (so age
|
||||
# comparisons naturally treat "no cache" the same as "very stale cache").
|
||||
lidarr_cache_age_seconds() {
|
||||
[[ -f "$LIDARR_CACHE_FILE" ]] || { echo 999999999; return; }
|
||||
local ts
|
||||
ts=$(jq -r '.ts // 0' "$LIDARR_CACHE_FILE" 2>/dev/null)
|
||||
echo $(( $(date +%s) - ${ts:-0} ))
|
||||
}
|
||||
|
||||
# Records how long a rescan-type command actually took, keyed by command name, so future
|
||||
# waits can be calibrated per command type instead of guessed or blended across very
|
||||
# different operations — a whole-library RescanFolders takes vastly longer than a targeted
|
||||
# DownloadedAlbumsScan, and averaging them would miscalibrate the wait for both.
|
||||
# Args: command_name, duration_seconds
|
||||
lidarr_record_rescan_duration() {
|
||||
local cmd_name="$1" duration="$2"
|
||||
[[ -z "$cmd_name" || -z "$duration" ]] && return 1
|
||||
mkdir -p "$(dirname "$LIDARR_RESCAN_DURATION_DB")" 2>/dev/null
|
||||
local tmp
|
||||
tmp=$(mktemp)
|
||||
[[ -f "$LIDARR_RESCAN_DURATION_DB" ]] && grep -v "^${cmd_name}|" "$LIDARR_RESCAN_DURATION_DB" > "$tmp" 2>/dev/null
|
||||
echo "${cmd_name}|${duration}" >> "$tmp"
|
||||
mv "$tmp" "$LIDARR_RESCAN_DURATION_DB"
|
||||
}
|
||||
|
||||
# Echoes the last recorded duration (seconds) for a given command name, or a fallback
|
||||
# default if none has ever been recorded.
|
||||
# Args: command_name, fallback_default_seconds
|
||||
lidarr_get_rescan_duration() {
|
||||
local cmd_name="$1" fallback="${2:-300}"
|
||||
[[ -f "$LIDARR_RESCAN_DURATION_DB" ]] || { echo "$fallback"; return; }
|
||||
local val
|
||||
val=$(grep "^${cmd_name}|" "$LIDARR_RESCAN_DURATION_DB" 2>/dev/null | tail -1 | cut -d'|' -f2)
|
||||
echo "${val:-$fallback}"
|
||||
}
|
||||
|
||||
# Checks Lidarr's command queue for any currently-active (started or queued) rescan-type
|
||||
# command. Echoes the command name of the first one found (so the caller can look up its
|
||||
# specific historical duration), empty if none active.
|
||||
# Args: url, api_key, api_version
|
||||
lidarr_active_rescan_command() {
|
||||
local url="$1" api_key="$2" api_version="$3"
|
||||
curl -sf --max-time 15 -H "X-Api-Key: $api_key" \
|
||||
"${url}/api/${api_version}/command" 2>/dev/null | \
|
||||
jq -r '[.[] | select(
|
||||
(.status=="started" or .status=="queued") and
|
||||
(.name=="RescanFolders" or .name=="DownloadedAlbumsScan" or .name=="RefreshArtist")
|
||||
)] | .[0].name // empty' 2>/dev/null
|
||||
}
|
||||
|
||||
# Main entry point — the only function consuming scripts should call for tracked artist data.
|
||||
# Handles fresh/stale-no-rescan/stale-rescan-active branching and always writes through on any
|
||||
# live fetch it performs. Echoes the artists JSON array on success, returns 1 if no usable data
|
||||
# (no cache and live fetch impossible) could be obtained.
|
||||
# Args: url, api_key, api_version, max_age_days (default 1), max_wait_strikes (default 3)
|
||||
lidarr_get_tracked_data() {
|
||||
local url="$1" api_key="$2" api_version="$3"
|
||||
local max_age_days="${4:-${LIDARR_CACHE_MAX_AGE_DAYS:-1}}" max_strikes="${5:-3}"
|
||||
local max_age_seconds=$(( max_age_days * 86400 ))
|
||||
local age
|
||||
age=$(lidarr_cache_age_seconds)
|
||||
|
||||
if [[ "$age" -lt "$max_age_seconds" ]]; then
|
||||
lidarr_cache_read_raw && return 0
|
||||
fi
|
||||
|
||||
# Cache missing or stale — check for an active rescan before deciding how to proceed.
|
||||
local active_cmd
|
||||
active_cmd=$(lidarr_active_rescan_command "$url" "$api_key" "$api_version")
|
||||
|
||||
if [[ -n "$active_cmd" ]]; then
|
||||
local wait_duration strike
|
||||
wait_duration=$(( $(lidarr_get_rescan_duration "$active_cmd" 300) / 2 ))
|
||||
[[ "$wait_duration" -lt 30 ]] && wait_duration=30
|
||||
for (( strike=1; strike<=max_strikes; strike++ )); do
|
||||
# Redirected to stderr — this function's stdout is a data channel (callers
|
||||
# capture it via command substitution), never mix log/warn output into it.
|
||||
warn " Lidarr busy ($active_cmd) — waiting ${wait_duration}s (strike ${strike}/${max_strikes})" >&2
|
||||
sleep "$wait_duration"
|
||||
active_cmd=$(lidarr_active_rescan_command "$url" "$api_key" "$api_version")
|
||||
[[ -z "$active_cmd" ]] && break
|
||||
done
|
||||
if [[ -n "$active_cmd" ]]; then
|
||||
warn " Lidarr still busy ($active_cmd) after ${max_strikes} strikes — using cache if present, else skipping" >&2
|
||||
lidarr_cache_read_raw && return 0
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# No active rescan (or it just finished mid-wait) — safe to fetch live and refresh cache.
|
||||
local fresh
|
||||
fresh=$(arr_api "$url" "$api_key" "$api_version" "artist" "Lidarr") || {
|
||||
lidarr_cache_read_raw && return 0
|
||||
return 1
|
||||
}
|
||||
lidarr_cache_write "$fresh"
|
||||
echo "$fresh"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ARR VERSION CHECK ─────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
Reference in New Issue
Block a user