Cache fanart.tv misses so the nightly art run stops re-querying artwork upstream has never had

This commit is contained in:
Gmer4Lfe
2026-07-28 18:51:06 -04:00
parent a1db8f7b2d
commit 19a9f6410d
2 changed files with 206 additions and 39 deletions
+204 -39
View File
@@ -32,6 +32,14 @@
# walk for its own cleanup decisions), falling back to its own live per-artist walk only if
# that cache is missing or from outside the current window.
#
# Negative art cache (2026-07-28). fanart.tv has no cdart/back art for most of the long tail —
# roughly 80% of this library — so the nightly run was re-asking about the same ~11.8K albums
# every night and getting the same nothing back. Measured 2h41m for 3 images fetched. A flat TSV
# in DATA_DIR now remembers "upstream has no <art type> for <mbid>" and skips the API call
# entirely until LIDARR_ART_RECHECK_DAYS has passed, so new fanart.tv contributions are still
# picked up, just monthly instead of nightly. Only genuine no-art-upstream results are cached —
# a failed download of a URL that did exist stays retryable on the next run.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
@@ -64,6 +72,8 @@
# Parallel job cap — LIDARR_ART_MAX_PARALLEL — avoids hammering external APIs
# Download retries — LIDARR_ART_RETRIES attempts per image before giving up
# Rate limiting — LIDARR_ART_SLEEP_BETWEEN between fanart.tv API calls
# Negative cache — skips entities whose every missing target is a known upstream miss
# Cache expiry on merge — entries past the recheck window are dropped, file stays bounded
#
# ==============================================================================================
# CONFIGURATION
@@ -82,6 +92,8 @@
# LIDARR_ART_MAX_PARALLEL — concurrent background download jobs
# LIDARR_ART_RETRIES — download retry attempts per image
# LIDARR_ART_SLEEP_BETWEEN — seconds between fanart.tv API calls
# LIDARR_ART_RECHECK_DAYS — days before re-querying art upstream didn't have
# LIDARR_ART_MISS_CACHE — path to the negative cache TSV
#
# ==============================================================================================
# RUNTIME MODES
@@ -90,6 +102,7 @@
# lidarr_missing_art.sh — fetch all missing artwork
# lidarr_missing_art.sh --dry-run — preview without downloading
# lidarr_missing_art.sh --log — verbose per-item output
# lidarr_missing_art.sh --refresh — ignore the negative cache, re-query everything
# lidarr_missing_art.sh --status — show config and exit
#
# ==============================================================================================
@@ -100,6 +113,15 @@ source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# --refresh ignores the negative cache for this run — use after fanart.tv has had time to gain
# new contributions, or to re-prove a miss set by hand. Does not clear the cache; misses found
# this run simply overwrite their old stamps.
ART_REFRESH=false
for _arg in "${PARSED_ARGS[@]}"; do
[[ "$_arg" == "--refresh" ]] && ART_REFRESH=true
done
unset _arg
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
@@ -138,6 +160,13 @@ unset _key
info "$MY_ID ($LOCAL_SERVER_NAME) — tools OK"
log "$ICON_GEAR Config: url=${LIDARR_URL}"
# Defaulted here rather than relying solely on master.conf: Configurations/master.conf is
# gitignored, so these reach a node through conf_upgrade seeding them from
# Deployment/master.conf.template — which lands in the same window as this script, not before
# it. Same guard style as arr_corruption_scan.sh's state file.
LIDARR_ART_RECHECK_DAYS="${LIDARR_ART_RECHECK_DAYS:-30}"
LIDARR_ART_MISS_CACHE="${LIDARR_ART_MISS_CACHE:-$DATA_DIR/lidarr_art_miss_cache.tsv}"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
@@ -152,6 +181,8 @@ if [[ "$SHOW_STATUS" == true ]]; then
echo "$ICON_GEAR Parallel: $LIDARR_ART_MAX_PARALLEL jobs"
echo "$ICON_RETRY Retries: $LIDARR_ART_RETRIES"
echo "$ICON_TIME API sleep: ${LIDARR_ART_SLEEP_BETWEEN}s"
echo "$ICON_GEAR Miss cache: $LIDARR_ART_MISS_CACHE ($([[ -f "$LIDARR_ART_MISS_CACHE" ]] && wc -l < "$LIDARR_ART_MISS_CACHE" || echo 0) entries)"
echo "$ICON_TIME Recheck: every ${LIDARR_ART_RECHECK_DAYS}d"
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
@@ -163,8 +194,21 @@ fi
# ── Temp dir for subshell fetch/fail counters ─────────────────────────────────────────────────
LIDARR_TMP=$(mktemp -d)
trap '_release_all_locks; rm -rf "$LIDARR_TMP"' EXIT
touch "$LIDARR_TMP/album_fetches" "$LIDARR_TMP/album_fails" \
"$LIDARR_TMP/artist_fetches" "$LIDARR_TMP/artist_fails"
touch "$LIDARR_TMP/album_fetches" "$LIDARR_TMP/album_nourl" "$LIDARR_TMP/album_dlfail" \
"$LIDARR_TMP/artist_fetches" "$LIDARR_TMP/artist_nourl" "$LIDARR_TMP/artist_dlfail" \
"$LIDARR_TMP/album_misses" "$LIDARR_TMP/artist_misses"
# ── Negative art cache state ──────────────────────────────────────────────────────────────────
NOW=$(date +%s)
ART_RECHECK_SECS=$(( LIDARR_ART_RECHECK_DAYS * 86400 ))
mkdir -p "$(dirname "$LIDARR_ART_MISS_CACHE")"
touch "$LIDARR_ART_MISS_CACHE"
declare -A ART_MISS
while IFS=$'\t' read -r _m_key _m_stamp; do
[[ -n "$_m_key" ]] && ART_MISS["$_m_key"]="$_m_stamp"
done < "$LIDARR_ART_MISS_CACHE"
unset _m_key _m_stamp
# ==============================================================================================
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
@@ -185,7 +229,12 @@ wait_for_slot() {
}
# Downloads URL to dest only if dest doesn't exist and downloaded size >= MIN_SIZE.
# Returns 0 on success or skip (file already exists), 1 on failure.
# Returns 0 on success or skip (file already exists), 1 when the source had no URL to offer,
# 2 when a URL existed but every download attempt failed.
#
# The 1-vs-2 split is what makes the negative cache safe: 1 means upstream genuinely has no
# such artwork and is worth remembering, 2 means a transient network/CDN problem that must
# stay retryable. Caching a 2 would suppress a legitimate retry for LIDARR_ART_RECHECK_DAYS.
download_if_valid() {
local url="$1"
local dest="$2"
@@ -214,7 +263,67 @@ download_if_valid() {
done
warn "Failed to fetch: $(basename "$dest")"
return 1
return 2
}
# ── Negative art cache ────────────────────────────────────────────────────────────────────────
# Same shape as arr_corruption_scan.sh's clean-file skip cache: a flat TSV of "<key>\t<epoch>"
# in DATA_DIR, slurped into an assoc array once at startup.
#
# Keyed per (entity, artwork filename) so an album that got cover.jpg from iTunes but has no
# cdart upstream still caches the cdart miss alone. MBID is the key where present because it
# survives a Lidarr DB rebuild; albums with no MBID fall back to the Lidarr id.
art_key() {
local mbid="$1" fallback_id="$2"
if [[ -n "$mbid" && "$mbid" != "null" ]]; then echo "$mbid"; else echo "lidarrid:$fallback_id"; fi
}
art_miss_fresh() {
local stamp="${ART_MISS[$1]:-}"
[[ -z "$stamp" ]] && return 1
(( NOW - stamp < ART_RECHECK_SECS ))
}
# True when every artwork target still missing from disk is a known-fresh upstream miss —
# i.e. this entity cannot possibly gain anything from another round of API calls right now.
# This is the check that skips the fanart.tv request entirely, which is where the time goes.
art_all_cached() {
local dir="$1"; local key="$2"; shift 2
local target
[[ "$ART_REFRESH" == true ]] && return 1
for target in "$@"; do
[[ -f "$dir/$target" ]] && continue
art_miss_fresh "${key}:${target}" || return 1
done
return 0
}
# Merges this run's fresh misses into the persistent cache, newest wins per key (fresh file is
# read first so awk's first-seen is always the newer stamp). Entries past the recheck window are
# dropped rather than carried: they would be re-queried on the next run anyway, so expiring them
# here is free and keeps the file from growing without bound as albums leave the library.
merge_art_misses() {
local fresh="$1" tmp
[[ "$DRY_RUN" == true ]] && return 0
[[ -s "$fresh" ]] || return 0
tmp=$(mktemp)
cat "$fresh" "$LIDARR_ART_MISS_CACHE" 2>/dev/null |
awk -F'\t' -v cutoff="$(( NOW - ART_RECHECK_SECS ))" \
'NF==2 && !seen[$1]++ && $2 >= cutoff' | sort > "$tmp"
mv "$tmp" "$LIDARR_ART_MISS_CACHE"
}
# Runs one single-source artwork target and classifies the outcome. Must be called from inside
# a fetch subshell — it updates that subshell's _fetches/_nourl/_dlfail/_miss_keys directly.
try_single() {
local dest="$1" url="$2" key="$3" rc
[[ -f "$dest" ]] && return 0
download_if_valid "$url" "$dest"; rc=$?
case "$rc" in
0) (( _fetches++ )) ;;
2) (( _dlfail++ )) ;;
*) (( _nourl++ )); _miss_keys+="${key}"$'\t'"${NOW}"$'\n' ;;
esac
}
deezer_artist_image() {
@@ -250,9 +359,11 @@ START=$(date +%s)
ALBUMS_CHECKED=0
ALBUMS_COMPLETE=0
ALBUMS_CACHED=0
ARTISTS_CHECKED=0
ARTISTS_COMPLETE=0
ARTISTS_CACHED=0
# ==============================================================================================
# ━━━ Build Album Directory Map ━━━
@@ -333,10 +444,18 @@ while IFS=$'\t' read -r mbid artist_name album_name album_id; do
continue
fi
akey=$(art_key "$mbid" "$album_id")
if art_all_cached "$local_path" "$akey" cover.jpg cdart.png back.jpg; then
(( ALBUMS_CACHED++ ))
log " every missing target is a known upstream miss — skipping"
continue
fi
wait_for_slot
(
_fetches=0 _fails=0
_fetches=0 _nourl=0 _dlfail=0 _miss_keys=""
JSON=""
if [[ -n "$mbid" && "$mbid" != "null" ]]; then
@@ -344,44 +463,56 @@ while IFS=$'\t' read -r mbid artist_name album_name album_id; do
sleep "$LIDARR_ART_SLEEP_BETWEEN"
fi
# cover.jpg has a two-source chain, so it tracks whether *any* source offered a URL:
# only a clean no-URL-anywhere result is cacheable.
if [[ ! -f "$local_path/cover.jpg" ]]; then
_saw_url=false
IMG=$(echo "$JSON" | jq -r '.albums[].albumcover[0].url // empty' 2>/dev/null)
if download_if_valid "$IMG" "$local_path/cover.jpg"; then
download_if_valid "$IMG" "$local_path/cover.jpg"; _rc=$?
(( _rc == 2 )) && _saw_url=true
if (( _rc == 0 )); then
(( _fetches++ ))
else
query=$(printf "%s %s" "$artist_name" "$album_name" | sed 's/ /+/g')
itunes=$(curl_json "https://itunes.apple.com/search?term=$query&entity=album&limit=1" |
jq -r '.results[0].artworkUrl100 // empty' 2>/dev/null | sed 's/100x100/600x600/')
if download_if_valid "$itunes" "$local_path/cover.jpg"; then
download_if_valid "$itunes" "$local_path/cover.jpg"; _rc=$?
(( _rc == 2 )) && _saw_url=true
if (( _rc == 0 )); then
(( _fetches++ ))
elif [[ "$_saw_url" == true ]]; then
(( _dlfail++ ))
else
(( _fails++ ))
(( _nourl++ )); _miss_keys+="${akey}:cover.jpg"$'\t'"${NOW}"$'\n'
fi
fi
fi
if [[ ! -f "$local_path/cdart.png" ]]; then
IMG=$(echo "$JSON" | jq -r '.albums[].cdart[0].url // empty' 2>/dev/null)
if download_if_valid "$IMG" "$local_path/cdart.png"; then (( _fetches++ )); else (( _fails++ )); fi
fi
try_single "$local_path/cdart.png" \
"$(echo "$JSON" | jq -r '.albums[].cdart[0].url // empty' 2>/dev/null)" \
"${akey}:cdart.png"
if [[ ! -f "$local_path/back.jpg" ]]; then
IMG=$(echo "$JSON" | jq -r '.albums[].albumback[0].url // empty' 2>/dev/null)
if download_if_valid "$IMG" "$local_path/back.jpg"; then (( _fetches++ )); else (( _fails++ )); fi
fi
try_single "$local_path/back.jpg" \
"$(echo "$JSON" | jq -r '.albums[].albumback[0].url // empty' 2>/dev/null)" \
"${akey}:back.jpg"
(( _fetches > 0 )) && printf '1\n' >> "$LIDARR_TMP/album_fetches"
(( _fails > 0 )) && printf '1\n' >> "$LIDARR_TMP/album_fails"
(( _nourl > 0 )) && printf '1\n' >> "$LIDARR_TMP/album_nourl"
(( _dlfail > 0 )) && printf '1\n' >> "$LIDARR_TMP/album_dlfail"
[[ -n "$_miss_keys" ]] && printf '%s' "$_miss_keys" >> "$LIDARR_TMP/album_misses"
) &
done < <(echo "$albums" | jq -r '.[] | [(.foreignAlbumId // ""), (.artist.artistName // ""), (.title // ""), (.id | tostring)] | @tsv')
wait
merge_art_misses "$LIDARR_TMP/album_misses"
ALBUM_FETCHED=$(wc -l < "$LIDARR_TMP/album_fetches" 2>/dev/null || echo 0)
ALBUM_FAILED=$(wc -l < "$LIDARR_TMP/album_fails" 2>/dev/null || echo 0)
ALBUM_NOART=$(wc -l < "$LIDARR_TMP/album_nourl" 2>/dev/null || echo 0)
ALBUM_DLFAIL=$(wc -l < "$LIDARR_TMP/album_dlfail" 2>/dev/null || echo 0)
ALBUM_MISSING=$(( ALBUMS_CHECKED - ALBUMS_COMPLETE ))
info "Checked: $ALBUMS_CHECKED | Complete: $ALBUMS_COMPLETE | Needed art: $ALBUM_MISSING | Fetched: $ALBUM_FETCHED | Failed: $ALBUM_FAILED"
info "Checked: $ALBUMS_CHECKED | Complete: $ALBUMS_COMPLETE | Needed art: $ALBUM_MISSING | Cached-skip: $ALBUMS_CACHED | Fetched: $ALBUM_FETCHED | No art upstream: $ALBUM_NOART | Fetch failed: $ALBUM_DLFAIL"
# ==============================================================================================
# ━━━ Artists ━━━
@@ -420,65 +551,96 @@ while IFS=$'\t' read -r local_path mbid name; do
continue
fi
if art_all_cached "$local_path" "$mbid" folder.jpg fanart.jpg clearlogo.png banner.jpg; then
(( ARTISTS_CACHED++ ))
log " every missing target is a known upstream miss — skipping"
continue
fi
wait_for_slot
(
_fetches=0 _fails=0
_fetches=0 _nourl=0 _dlfail=0 _miss_keys=""
JSON=$(curl_json "http://webservice.fanart.tv/v3/music/$mbid?api_key=$FANART_API_KEY")
sleep "$LIDARR_ART_SLEEP_BETWEEN"
# folder.jpg walks fanart → Deezer → Last.fm; only a no-URL result from all three is
# cacheable, same reasoning as the album cover chain.
if [[ ! -f "$local_path/folder.jpg" ]]; then
_saw_url=false
IMG=$(echo "$JSON" | jq -r '.artistthumb[0].url // empty' 2>/dev/null)
if download_if_valid "$IMG" "$local_path/folder.jpg"; then
download_if_valid "$IMG" "$local_path/folder.jpg"; _rc=$?
(( _rc == 2 )) && _saw_url=true
if (( _rc == 0 )); then
(( _fetches++ ))
else
IMG=$(deezer_artist_image "$name")
if download_if_valid "$IMG" "$local_path/folder.jpg"; then
download_if_valid "$IMG" "$local_path/folder.jpg"; _rc=$?
(( _rc == 2 )) && _saw_url=true
if (( _rc == 0 )); then
(( _fetches++ ))
else
IMG=$(lastfm_artist_image "$name")
if download_if_valid "$IMG" "$local_path/folder.jpg"; then
download_if_valid "$IMG" "$local_path/folder.jpg"; _rc=$?
(( _rc == 2 )) && _saw_url=true
if (( _rc == 0 )); then
(( _fetches++ ))
elif [[ "$_saw_url" == true ]]; then
(( _dlfail++ ))
else
(( _fails++ ))
(( _nourl++ )); _miss_keys+="${mbid}:folder.jpg"$'\t'"${NOW}"$'\n'
fi
fi
fi
fi
if [[ ! -f "$local_path/fanart.jpg" ]]; then
_saw_url=false
IMG=$(echo "$JSON" | jq -r '.artistbackground[0].url // empty' 2>/dev/null)
if download_if_valid "$IMG" "$local_path/fanart.jpg"; then
download_if_valid "$IMG" "$local_path/fanart.jpg"; _rc=$?
(( _rc == 2 )) && _saw_url=true
if (( _rc == 0 )); then
(( _fetches++ ))
else
IMG=$(deezer_artist_image "$name")
if download_if_valid "$IMG" "$local_path/fanart.jpg"; then (( _fetches++ )); else (( _fails++ )); fi
download_if_valid "$IMG" "$local_path/fanart.jpg"; _rc=$?
(( _rc == 2 )) && _saw_url=true
if (( _rc == 0 )); then
(( _fetches++ ))
elif [[ "$_saw_url" == true ]]; then
(( _dlfail++ ))
else
(( _nourl++ )); _miss_keys+="${mbid}:fanart.jpg"$'\t'"${NOW}"$'\n'
fi
fi
fi
if [[ ! -f "$local_path/clearlogo.png" ]]; then
IMG=$(echo "$JSON" | jq -r '.hdmusiclogo[0].url // empty' 2>/dev/null)
if download_if_valid "$IMG" "$local_path/clearlogo.png"; then (( _fetches++ )); else (( _fails++ )); fi
fi
try_single "$local_path/clearlogo.png" \
"$(echo "$JSON" | jq -r '.hdmusiclogo[0].url // empty' 2>/dev/null)" \
"${mbid}:clearlogo.png"
if [[ ! -f "$local_path/banner.jpg" ]]; then
IMG=$(echo "$JSON" | jq -r '.musicbanner[0].url // empty' 2>/dev/null)
if download_if_valid "$IMG" "$local_path/banner.jpg"; then (( _fetches++ )); else (( _fails++ )); fi
fi
try_single "$local_path/banner.jpg" \
"$(echo "$JSON" | jq -r '.musicbanner[0].url // empty' 2>/dev/null)" \
"${mbid}:banner.jpg"
(( _fetches > 0 )) && printf '1\n' >> "$LIDARR_TMP/artist_fetches"
(( _fails > 0 )) && printf '1\n' >> "$LIDARR_TMP/artist_fails"
(( _nourl > 0 )) && printf '1\n' >> "$LIDARR_TMP/artist_nourl"
(( _dlfail > 0 )) && printf '1\n' >> "$LIDARR_TMP/artist_dlfail"
[[ -n "$_miss_keys" ]] && printf '%s' "$_miss_keys" >> "$LIDARR_TMP/artist_misses"
) &
done < <(echo "$artists" | jq -r '.[] | [.path, .foreignArtistId, .artistName] | @tsv')
wait
merge_art_misses "$LIDARR_TMP/artist_misses"
ARTIST_FETCHED=$(wc -l < "$LIDARR_TMP/artist_fetches" 2>/dev/null || echo 0)
ARTIST_FAILED=$(wc -l < "$LIDARR_TMP/artist_fails" 2>/dev/null || echo 0)
ARTIST_NOART=$(wc -l < "$LIDARR_TMP/artist_nourl" 2>/dev/null || echo 0)
ARTIST_DLFAIL=$(wc -l < "$LIDARR_TMP/artist_dlfail" 2>/dev/null || echo 0)
ARTIST_MISSING=$(( ARTISTS_CHECKED - ARTISTS_COMPLETE ))
info "Checked: $ARTISTS_CHECKED | Complete: $ARTISTS_COMPLETE | Needed art: $ARTIST_MISSING | Fetched: $ARTIST_FETCHED | Failed: $ARTIST_FAILED"
info "Checked: $ARTISTS_CHECKED | Complete: $ARTISTS_COMPLETE | Needed art: $ARTIST_MISSING | Cached-skip: $ARTISTS_CACHED | Fetched: $ARTIST_FETCHED | No art upstream: $ARTIST_NOART | Fetch failed: $ARTIST_DLFAIL"
END=$(date +%s)
@@ -489,8 +651,11 @@ echo ""
echo "━━━━━ $ICON_SUMMARY LIDARR MISSING ART SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_EMBY Albums: $ALBUMS_CHECKED checked | $ALBUMS_COMPLETE complete | $ALBUM_FETCHED fetched | $ALBUM_FAILED failed"
echo "$ICON_EMBY Artists: $ARTISTS_CHECKED checked | $ARTISTS_COMPLETE complete | $ARTIST_FETCHED fetched | $ARTIST_FAILED failed"
echo "$ICON_EMBY Albums: $ALBUMS_CHECKED checked | $ALBUMS_COMPLETE complete | $ALBUM_FETCHED fetched"
echo "$ICON_SKIP Albums: $ALBUMS_CACHED skipped (cached miss) | $ALBUM_NOART no art upstream | $ALBUM_DLFAIL fetch failed"
echo "$ICON_EMBY Artists: $ARTISTS_CHECKED checked | $ARTISTS_COMPLETE complete | $ARTIST_FETCHED fetched"
echo "$ICON_SKIP Artists: $ARTISTS_CACHED skipped (cached miss) | $ARTIST_NOART no art upstream | $ARTIST_DLFAIL fetch failed"
echo "$ICON_GEAR Cache: $([[ -f "$LIDARR_ART_MISS_CACHE" ]] && wc -l < "$LIDARR_ART_MISS_CACHE" || echo 0) known upstream misses | recheck every ${LIDARR_ART_RECHECK_DAYS}d"
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no files written"