Add Lidarr tracked-data cache + duplicate artist cleanup
Shared cache (lidarr_get_tracked_data() in common.sh) so scripts stop hitting Lidarr's live API for tracked counts every run, and stop treating a mid-rescan dip as a genuine problem — a whole-library RescanFolders legitimately makes trackFileCount read far below normal while it re-verifies every file (confirmed 2026-07-16: 22% of normal mid-scan). Cache reads fresh-if-recent, waits out an active rescan (calibrated to that command's own historical duration, tracked per command name since RescanFolders and DownloadedAlbumsScan take wildly different amounts of time), then falls back to a stale cache rather than hard-failing after a few strikes. lidarr_cleanup.sh: no longer stacks a fresh DownloadedAlbumsScan on top of one already running, and the tracked-count floor check now waits out a genuine rescan instead of aborting on every overlap. lidarr_duplicate_artist_cleanup.sh (new): finds case-insensitive duplicate artist entries — same display name, different MusicBrainz ID, added when a search/list-sync matches the wrong same-named artist. Deletes the empty phantom side and blocks it from Import List Exclusions, leaves genuinely-different-real-artists alone (checked by album title overlap, deduped per-artist first so a legitimate reissue under an artist's own catalog doesn't false-flag as cross-artist overlap), and only notifies for the rare case where both sides have real, overlapping content. lidarr_cache_prefill.sh (new): warms the cache at array start so nothing reads it cold after boot. lidarr_missing_art.sh, lidarr_release_fixer.sh: write-through the cache as a side effect of fetches they already needed for their own purposes.
This commit is contained in:
Gmer4Lfe
@@ -10,6 +10,16 @@
|
||||
# that is old enough to be past the import window. Triggers an Emby library
|
||||
# clean after runs where files were deleted so ghost entries disappear immediately.
|
||||
#
|
||||
# Pre-flight and the tracked-count floor check are both rescan-aware: a library-wide
|
||||
# rescan legitimately makes tracked counts read low mid-scan (confirmed 2026-07-16 —
|
||||
# 22% of normal during an active RescanFolders), which used to trigger this script's own
|
||||
# hard abort every time it overlapped with a real rescan. Now it checks for an active
|
||||
# rescan-type command first — if one's running, it waits (calibrated to that command's
|
||||
# own historical duration via lidarr_get_rescan_duration(), up to 3 strikes) and re-fetches
|
||||
# rather than either stacking a duplicate scan or crying wolf on a normal, if slow, state.
|
||||
# Only escalates to the scary abort-and-notify when the count is genuinely low AND nothing
|
||||
# is actively rescanning.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
@@ -246,16 +256,32 @@ for _cp in "${!ARR_PATH_MAP[@]}"; do
|
||||
done
|
||||
unset _cp
|
||||
|
||||
if [[ -n "$LIDARR_CONTAINER_ROOT" ]]; then
|
||||
# Don't stack a fresh scan on top of one already running — confirmed 2026-07-16 that
|
||||
# repeated runs each firing their own DownloadedAlbumsScan piled up in Lidarr's command
|
||||
# queue behind each other rather than replacing/coalescing, contributing to a multi-hour
|
||||
# backlog. If something's already scanning, just wait for that one instead.
|
||||
_already_active=$(lidarr_active_rescan_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1")
|
||||
|
||||
if [[ -n "$_already_active" ]]; then
|
||||
info "$_already_active already in progress — waiting for it instead of starting a new scan"
|
||||
_wait=$(( $(lidarr_get_rescan_duration "$_already_active" "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}") ))
|
||||
_polled=0
|
||||
while [[ "$_polled" -lt "$_wait" ]]; do
|
||||
[[ -z "$(lidarr_active_rescan_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1")" ]] && break
|
||||
sleep 15
|
||||
(( _polled += 15 ))
|
||||
[[ $(( _polled % 60 )) -eq 0 ]] && log " Still waiting on $_already_active... (${_polled}s elapsed)"
|
||||
done
|
||||
elif [[ -n "$LIDARR_CONTAINER_ROOT" ]]; then
|
||||
info "Triggering DownloadedAlbumsScan on: $LIDARR_CONTAINER_ROOT"
|
||||
SCAN_PAYLOAD="{\"name\": \"DownloadedAlbumsScan\", \"path\": \"$LIDARR_CONTAINER_ROOT\"}"
|
||||
trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$SCAN_PAYLOAD" "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}"
|
||||
else
|
||||
info "No path map match — triggering DownloadedAlbumsScan (all root folders)"
|
||||
SCAN_PAYLOAD='{"name": "DownloadedAlbumsScan"}'
|
||||
trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$SCAN_PAYLOAD" "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}"
|
||||
fi
|
||||
|
||||
trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$SCAN_PAYLOAD" "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Fetch Lidarr Tracked Files ━━━
|
||||
# ==============================================================================================
|
||||
@@ -295,30 +321,38 @@ fi
|
||||
info "Found $ARTIST_COUNT artists — fetching track files..."
|
||||
|
||||
TRACKED_FILE="$TMP_DIR/tracked_paths.txt"
|
||||
> "$TRACKED_FILE"
|
||||
|
||||
while IFS= read -r artist_id; do
|
||||
[[ -z "$artist_id" ]] && continue
|
||||
ARTIST_TRACKS=$(arr_api "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "trackFile?artistId=${artist_id}" "Lidarr" 2>/dev/null)
|
||||
if [[ -n "$ARTIST_TRACKS" ]]; then
|
||||
while IFS= read -r api_path; do
|
||||
[[ -z "$api_path" ]] && continue
|
||||
translate_path "$api_path" >> "$TRACKED_FILE"
|
||||
done < <(echo "$ARTIST_TRACKS" | jq -r '.[].path' 2>/dev/null)
|
||||
fi
|
||||
done <<< "$ARTIST_IDS"
|
||||
# Fetches every artist's track-file paths fresh into TRACKED_FILE/TRACKED_MAP/TRACKED_COUNT.
|
||||
# Pulled into a function so the rescan-aware retry below can re-fetch after waiting without
|
||||
# duplicating this whole loop inline.
|
||||
_fetch_tracked_files() {
|
||||
> "$TRACKED_FILE"
|
||||
while IFS= read -r artist_id; do
|
||||
[[ -z "$artist_id" ]] && continue
|
||||
ARTIST_TRACKS=$(arr_api "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "trackFile?artistId=${artist_id}" "Lidarr" 2>/dev/null)
|
||||
if [[ -n "$ARTIST_TRACKS" ]]; then
|
||||
while IFS= read -r api_path; do
|
||||
[[ -z "$api_path" ]] && continue
|
||||
translate_path "$api_path" >> "$TRACKED_FILE"
|
||||
done < <(echo "$ARTIST_TRACKS" | jq -r '.[].path' 2>/dev/null)
|
||||
fi
|
||||
done <<< "$ARTIST_IDS"
|
||||
|
||||
sort -u "$TRACKED_FILE" -o "$TRACKED_FILE"
|
||||
sort -u "$TRACKED_FILE" -o "$TRACKED_FILE"
|
||||
|
||||
# Build in-memory lookup map — O(1) per lookup vs O(n) grep per file
|
||||
# Eliminates the main performance bottleneck for large libraries
|
||||
declare -A TRACKED_MAP
|
||||
while IFS= read -r _tracked_path; do
|
||||
[[ -n "$_tracked_path" ]] && TRACKED_MAP["$_tracked_path"]=1
|
||||
done < "$TRACKED_FILE"
|
||||
unset _tracked_path
|
||||
# Build in-memory lookup map — O(1) per lookup vs O(n) grep per file
|
||||
# Eliminates the main performance bottleneck for large libraries
|
||||
unset TRACKED_MAP
|
||||
declare -gA TRACKED_MAP
|
||||
while IFS= read -r _tracked_path; do
|
||||
[[ -n "$_tracked_path" ]] && TRACKED_MAP["$_tracked_path"]=1
|
||||
done < "$TRACKED_FILE"
|
||||
unset _tracked_path
|
||||
TRACKED_COUNT=$(wc -l < "$TRACKED_FILE")
|
||||
}
|
||||
|
||||
_fetch_tracked_files
|
||||
info "Built in-memory lookup map: ${#TRACKED_MAP[@]} tracked paths"
|
||||
TRACKED_COUNT=$(wc -l < "$TRACKED_FILE")
|
||||
|
||||
# Safety Layer 5 — tracked count > 0
|
||||
if [[ "$TRACKED_COUNT" -eq 0 ]]; then
|
||||
@@ -330,7 +364,41 @@ fi
|
||||
|
||||
info "$ARTIST_COUNT artists | $TRACKED_COUNT tracked files"
|
||||
|
||||
# Safety Layer 6 — percentage drop vs last known count
|
||||
# Safety Layer 6 — percentage drop vs last known count, with rescan-aware retry.
|
||||
# A library-wide rescan legitimately makes tracked counts read low mid-scan — sometimes
|
||||
# dramatically (confirmed 2026-07-16: 22% of normal during an active RescanFolders).
|
||||
# That's not "something's wrong," it's Lidarr actively re-verifying every file. Wait it out
|
||||
# (calibrated to that command's own historical duration) before treating a drop as a genuine
|
||||
# problem worth the scary abort-and-notify. Only escalates to the hard abort in
|
||||
# check_tracked_count_floor if the count is still low AND nothing is actively rescanning —
|
||||
# that combination is the actually-suspicious case the floor check exists to catch.
|
||||
_last_known=$(cat "$LIDARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0)
|
||||
if [[ "$_last_known" -gt 0 ]]; then
|
||||
_strike=1
|
||||
while [[ "$_strike" -le 3 ]]; do
|
||||
_pct=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $_last_known) * 100}")
|
||||
[[ "$_pct" -ge "${LIDARR_MIN_TRACKED_PCT:-50}" ]] && break
|
||||
|
||||
_active_cmd=$(lidarr_active_rescan_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1")
|
||||
[[ -z "$_active_cmd" ]] && break # low count, nothing rescanning — genuine, don't retry
|
||||
|
||||
_wait=$(( $(lidarr_get_rescan_duration "$_active_cmd" 300) / 2 ))
|
||||
[[ "$_wait" -lt 30 ]] && _wait=30
|
||||
warn "Tracked count ${_pct}% of last run, but $_active_cmd active — waiting ${_wait}s (strike ${_strike}/3)"
|
||||
sleep "$_wait"
|
||||
_fetch_tracked_files
|
||||
(( _strike++ ))
|
||||
done
|
||||
|
||||
if [[ "$_strike" -gt 3 ]]; then
|
||||
_active_cmd=$(lidarr_active_rescan_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1")
|
||||
if [[ -n "$_active_cmd" ]]; then
|
||||
warn "Lidarr still busy ($_active_cmd) after 3 strikes — deferring to next scheduled run"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
check_tracked_count_floor "$TRACKED_COUNT" "$LIDARR_TRACKED_COUNT_FILE" "$LIDARR_MIN_TRACKED_PCT" "Lidarr Cleanup"
|
||||
|
||||
# ==============================================================================================
|
||||
|
||||
Reference in New Issue
Block a user