diff --git a/Arrs_Stack/lidarr_missing_art.sh b/Arrs_Stack/lidarr_missing_art.sh index b2ce49f..58bbe1c 100755 --- a/Arrs_Stack/lidarr_missing_art.sh +++ b/Arrs_Stack/lidarr_missing_art.sh @@ -32,6 +32,14 @@ # walk for its own cleanup decisions), falling back to its own live per-artist walk only if # that cache is missing or from outside the current window. # +# Negative art cache (2026-07-28). fanart.tv has no cdart/back art for most of the long tail — +# roughly 80% of this library — so the nightly run was re-asking about the same ~11.8K albums +# every night and getting the same nothing back. Measured 2h41m for 3 images fetched. A flat TSV +# in DATA_DIR now remembers "upstream has no for " and skips the API call +# entirely until LIDARR_ART_RECHECK_DAYS has passed, so new fanart.tv contributions are still +# picked up, just monthly instead of nightly. Only genuine no-art-upstream results are cached — +# a failed download of a URL that did exist stays retryable on the next run. +# # ============================================================================================== # DESIGN PRINCIPLES # ============================================================================================== @@ -64,6 +72,8 @@ # Parallel job cap — LIDARR_ART_MAX_PARALLEL — avoids hammering external APIs # Download retries — LIDARR_ART_RETRIES attempts per image before giving up # Rate limiting — LIDARR_ART_SLEEP_BETWEEN between fanart.tv API calls +# Negative cache — skips entities whose every missing target is a known upstream miss +# Cache expiry on merge — entries past the recheck window are dropped, file stays bounded # # ============================================================================================== # CONFIGURATION @@ -82,6 +92,8 @@ # LIDARR_ART_MAX_PARALLEL — concurrent background download jobs # LIDARR_ART_RETRIES — download retry attempts per image # LIDARR_ART_SLEEP_BETWEEN — seconds between fanart.tv API calls +# LIDARR_ART_RECHECK_DAYS — days before re-querying art upstream didn't have +# LIDARR_ART_MISS_CACHE — path to the negative cache TSV # # ============================================================================================== # RUNTIME MODES @@ -90,6 +102,7 @@ # lidarr_missing_art.sh — fetch all missing artwork # lidarr_missing_art.sh --dry-run — preview without downloading # lidarr_missing_art.sh --log — verbose per-item output +# lidarr_missing_art.sh --refresh — ignore the negative cache, re-query everything # lidarr_missing_art.sh --status — show config and exit # # ============================================================================================== @@ -100,6 +113,15 @@ source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" +# --refresh ignores the negative cache for this run — use after fanart.tv has had time to gain +# new contributions, or to re-prove a miss set by hand. Does not clear the cache; misses found +# this run simply overwrite their old stamps. +ART_REFRESH=false +for _arg in "${PARSED_ARGS[@]}"; do + [[ "$_arg" == "--refresh" ]] && ART_REFRESH=true +done +unset _arg + # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== @@ -138,6 +160,13 @@ unset _key info "$MY_ID ($LOCAL_SERVER_NAME) — tools OK" log "$ICON_GEAR Config: url=${LIDARR_URL}" +# Defaulted here rather than relying solely on master.conf: Configurations/master.conf is +# gitignored, so these reach a node through conf_upgrade seeding them from +# Deployment/master.conf.template — which lands in the same window as this script, not before +# it. Same guard style as arr_corruption_scan.sh's state file. +LIDARR_ART_RECHECK_DAYS="${LIDARR_ART_RECHECK_DAYS:-30}" +LIDARR_ART_MISS_CACHE="${LIDARR_ART_MISS_CACHE:-$DATA_DIR/lidarr_art_miss_cache.tsv}" + # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== @@ -152,6 +181,8 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "$ICON_GEAR Parallel: $LIDARR_ART_MAX_PARALLEL jobs" echo "$ICON_RETRY Retries: $LIDARR_ART_RETRIES" echo "$ICON_TIME API sleep: ${LIDARR_ART_SLEEP_BETWEEN}s" + echo "$ICON_GEAR Miss cache: $LIDARR_ART_MISS_CACHE ($([[ -f "$LIDARR_ART_MISS_CACHE" ]] && wc -l < "$LIDARR_ART_MISS_CACHE" || echo 0) entries)" + echo "$ICON_TIME Recheck: every ${LIDARR_ART_RECHECK_DAYS}d" echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)" echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "━━━━━━━━━━━━━━━━━━━━━━━" @@ -163,8 +194,21 @@ fi # ── Temp dir for subshell fetch/fail counters ───────────────────────────────────────────────── LIDARR_TMP=$(mktemp -d) trap '_release_all_locks; rm -rf "$LIDARR_TMP"' EXIT -touch "$LIDARR_TMP/album_fetches" "$LIDARR_TMP/album_fails" \ - "$LIDARR_TMP/artist_fetches" "$LIDARR_TMP/artist_fails" +touch "$LIDARR_TMP/album_fetches" "$LIDARR_TMP/album_nourl" "$LIDARR_TMP/album_dlfail" \ + "$LIDARR_TMP/artist_fetches" "$LIDARR_TMP/artist_nourl" "$LIDARR_TMP/artist_dlfail" \ + "$LIDARR_TMP/album_misses" "$LIDARR_TMP/artist_misses" + +# ── Negative art cache state ────────────────────────────────────────────────────────────────── +NOW=$(date +%s) +ART_RECHECK_SECS=$(( LIDARR_ART_RECHECK_DAYS * 86400 )) +mkdir -p "$(dirname "$LIDARR_ART_MISS_CACHE")" +touch "$LIDARR_ART_MISS_CACHE" + +declare -A ART_MISS +while IFS=$'\t' read -r _m_key _m_stamp; do + [[ -n "$_m_key" ]] && ART_MISS["$_m_key"]="$_m_stamp" +done < "$LIDARR_ART_MISS_CACHE" +unset _m_key _m_stamp # ============================================================================================== # ── FUNCTIONS ───────────────────────────────────────────────────────────────────────────────── @@ -185,7 +229,12 @@ wait_for_slot() { } # Downloads URL to dest only if dest doesn't exist and downloaded size >= MIN_SIZE. -# Returns 0 on success or skip (file already exists), 1 on failure. +# Returns 0 on success or skip (file already exists), 1 when the source had no URL to offer, +# 2 when a URL existed but every download attempt failed. +# +# The 1-vs-2 split is what makes the negative cache safe: 1 means upstream genuinely has no +# such artwork and is worth remembering, 2 means a transient network/CDN problem that must +# stay retryable. Caching a 2 would suppress a legitimate retry for LIDARR_ART_RECHECK_DAYS. download_if_valid() { local url="$1" local dest="$2" @@ -214,7 +263,67 @@ download_if_valid() { done warn "Failed to fetch: $(basename "$dest")" - return 1 + return 2 +} + +# ── Negative art cache ──────────────────────────────────────────────────────────────────────── +# Same shape as arr_corruption_scan.sh's clean-file skip cache: a flat TSV of "\t" +# in DATA_DIR, slurped into an assoc array once at startup. +# +# Keyed per (entity, artwork filename) so an album that got cover.jpg from iTunes but has no +# cdart upstream still caches the cdart miss alone. MBID is the key where present because it +# survives a Lidarr DB rebuild; albums with no MBID fall back to the Lidarr id. +art_key() { + local mbid="$1" fallback_id="$2" + if [[ -n "$mbid" && "$mbid" != "null" ]]; then echo "$mbid"; else echo "lidarrid:$fallback_id"; fi +} + +art_miss_fresh() { + local stamp="${ART_MISS[$1]:-}" + [[ -z "$stamp" ]] && return 1 + (( NOW - stamp < ART_RECHECK_SECS )) +} + +# True when every artwork target still missing from disk is a known-fresh upstream miss — +# i.e. this entity cannot possibly gain anything from another round of API calls right now. +# This is the check that skips the fanart.tv request entirely, which is where the time goes. +art_all_cached() { + local dir="$1"; local key="$2"; shift 2 + local target + [[ "$ART_REFRESH" == true ]] && return 1 + for target in "$@"; do + [[ -f "$dir/$target" ]] && continue + art_miss_fresh "${key}:${target}" || return 1 + done + return 0 +} + +# Merges this run's fresh misses into the persistent cache, newest wins per key (fresh file is +# read first so awk's first-seen is always the newer stamp). Entries past the recheck window are +# dropped rather than carried: they would be re-queried on the next run anyway, so expiring them +# here is free and keeps the file from growing without bound as albums leave the library. +merge_art_misses() { + local fresh="$1" tmp + [[ "$DRY_RUN" == true ]] && return 0 + [[ -s "$fresh" ]] || return 0 + tmp=$(mktemp) + cat "$fresh" "$LIDARR_ART_MISS_CACHE" 2>/dev/null | + awk -F'\t' -v cutoff="$(( NOW - ART_RECHECK_SECS ))" \ + 'NF==2 && !seen[$1]++ && $2 >= cutoff' | sort > "$tmp" + mv "$tmp" "$LIDARR_ART_MISS_CACHE" +} + +# Runs one single-source artwork target and classifies the outcome. Must be called from inside +# a fetch subshell — it updates that subshell's _fetches/_nourl/_dlfail/_miss_keys directly. +try_single() { + local dest="$1" url="$2" key="$3" rc + [[ -f "$dest" ]] && return 0 + download_if_valid "$url" "$dest"; rc=$? + case "$rc" in + 0) (( _fetches++ )) ;; + 2) (( _dlfail++ )) ;; + *) (( _nourl++ )); _miss_keys+="${key}"$'\t'"${NOW}"$'\n' ;; + esac } deezer_artist_image() { @@ -250,9 +359,11 @@ START=$(date +%s) ALBUMS_CHECKED=0 ALBUMS_COMPLETE=0 +ALBUMS_CACHED=0 ARTISTS_CHECKED=0 ARTISTS_COMPLETE=0 +ARTISTS_CACHED=0 # ============================================================================================== # ━━━ Build Album Directory Map ━━━ @@ -333,10 +444,18 @@ while IFS=$'\t' read -r mbid artist_name album_name album_id; do continue fi + akey=$(art_key "$mbid" "$album_id") + + if art_all_cached "$local_path" "$akey" cover.jpg cdart.png back.jpg; then + (( ALBUMS_CACHED++ )) + log " every missing target is a known upstream miss — skipping" + continue + fi + wait_for_slot ( - _fetches=0 _fails=0 + _fetches=0 _nourl=0 _dlfail=0 _miss_keys="" JSON="" if [[ -n "$mbid" && "$mbid" != "null" ]]; then @@ -344,44 +463,56 @@ while IFS=$'\t' read -r mbid artist_name album_name album_id; do sleep "$LIDARR_ART_SLEEP_BETWEEN" fi + # cover.jpg has a two-source chain, so it tracks whether *any* source offered a URL: + # only a clean no-URL-anywhere result is cacheable. if [[ ! -f "$local_path/cover.jpg" ]]; then + _saw_url=false IMG=$(echo "$JSON" | jq -r '.albums[].albumcover[0].url // empty' 2>/dev/null) - if download_if_valid "$IMG" "$local_path/cover.jpg"; then + download_if_valid "$IMG" "$local_path/cover.jpg"; _rc=$? + (( _rc == 2 )) && _saw_url=true + if (( _rc == 0 )); then (( _fetches++ )) else query=$(printf "%s %s" "$artist_name" "$album_name" | sed 's/ /+/g') itunes=$(curl_json "https://itunes.apple.com/search?term=$query&entity=album&limit=1" | jq -r '.results[0].artworkUrl100 // empty' 2>/dev/null | sed 's/100x100/600x600/') - if download_if_valid "$itunes" "$local_path/cover.jpg"; then + download_if_valid "$itunes" "$local_path/cover.jpg"; _rc=$? + (( _rc == 2 )) && _saw_url=true + if (( _rc == 0 )); then (( _fetches++ )) + elif [[ "$_saw_url" == true ]]; then + (( _dlfail++ )) else - (( _fails++ )) + (( _nourl++ )); _miss_keys+="${akey}:cover.jpg"$'\t'"${NOW}"$'\n' fi fi fi - if [[ ! -f "$local_path/cdart.png" ]]; then - IMG=$(echo "$JSON" | jq -r '.albums[].cdart[0].url // empty' 2>/dev/null) - if download_if_valid "$IMG" "$local_path/cdart.png"; then (( _fetches++ )); else (( _fails++ )); fi - fi + try_single "$local_path/cdart.png" \ + "$(echo "$JSON" | jq -r '.albums[].cdart[0].url // empty' 2>/dev/null)" \ + "${akey}:cdart.png" - if [[ ! -f "$local_path/back.jpg" ]]; then - IMG=$(echo "$JSON" | jq -r '.albums[].albumback[0].url // empty' 2>/dev/null) - if download_if_valid "$IMG" "$local_path/back.jpg"; then (( _fetches++ )); else (( _fails++ )); fi - fi + try_single "$local_path/back.jpg" \ + "$(echo "$JSON" | jq -r '.albums[].albumback[0].url // empty' 2>/dev/null)" \ + "${akey}:back.jpg" (( _fetches > 0 )) && printf '1\n' >> "$LIDARR_TMP/album_fetches" - (( _fails > 0 )) && printf '1\n' >> "$LIDARR_TMP/album_fails" + (( _nourl > 0 )) && printf '1\n' >> "$LIDARR_TMP/album_nourl" + (( _dlfail > 0 )) && printf '1\n' >> "$LIDARR_TMP/album_dlfail" + [[ -n "$_miss_keys" ]] && printf '%s' "$_miss_keys" >> "$LIDARR_TMP/album_misses" ) & done < <(echo "$albums" | jq -r '.[] | [(.foreignAlbumId // ""), (.artist.artistName // ""), (.title // ""), (.id | tostring)] | @tsv') wait +merge_art_misses "$LIDARR_TMP/album_misses" + ALBUM_FETCHED=$(wc -l < "$LIDARR_TMP/album_fetches" 2>/dev/null || echo 0) -ALBUM_FAILED=$(wc -l < "$LIDARR_TMP/album_fails" 2>/dev/null || echo 0) +ALBUM_NOART=$(wc -l < "$LIDARR_TMP/album_nourl" 2>/dev/null || echo 0) +ALBUM_DLFAIL=$(wc -l < "$LIDARR_TMP/album_dlfail" 2>/dev/null || echo 0) ALBUM_MISSING=$(( ALBUMS_CHECKED - ALBUMS_COMPLETE )) -info "Checked: $ALBUMS_CHECKED | Complete: $ALBUMS_COMPLETE | Needed art: $ALBUM_MISSING | Fetched: $ALBUM_FETCHED | Failed: $ALBUM_FAILED" +info "Checked: $ALBUMS_CHECKED | Complete: $ALBUMS_COMPLETE | Needed art: $ALBUM_MISSING | Cached-skip: $ALBUMS_CACHED | Fetched: $ALBUM_FETCHED | No art upstream: $ALBUM_NOART | Fetch failed: $ALBUM_DLFAIL" # ============================================================================================== # ━━━ Artists ━━━ @@ -420,65 +551,96 @@ while IFS=$'\t' read -r local_path mbid name; do continue fi + if art_all_cached "$local_path" "$mbid" folder.jpg fanart.jpg clearlogo.png banner.jpg; then + (( ARTISTS_CACHED++ )) + log " every missing target is a known upstream miss — skipping" + continue + fi + wait_for_slot ( - _fetches=0 _fails=0 + _fetches=0 _nourl=0 _dlfail=0 _miss_keys="" JSON=$(curl_json "http://webservice.fanart.tv/v3/music/$mbid?api_key=$FANART_API_KEY") sleep "$LIDARR_ART_SLEEP_BETWEEN" + # folder.jpg walks fanart → Deezer → Last.fm; only a no-URL result from all three is + # cacheable, same reasoning as the album cover chain. if [[ ! -f "$local_path/folder.jpg" ]]; then + _saw_url=false IMG=$(echo "$JSON" | jq -r '.artistthumb[0].url // empty' 2>/dev/null) - if download_if_valid "$IMG" "$local_path/folder.jpg"; then + download_if_valid "$IMG" "$local_path/folder.jpg"; _rc=$? + (( _rc == 2 )) && _saw_url=true + if (( _rc == 0 )); then (( _fetches++ )) else IMG=$(deezer_artist_image "$name") - if download_if_valid "$IMG" "$local_path/folder.jpg"; then + download_if_valid "$IMG" "$local_path/folder.jpg"; _rc=$? + (( _rc == 2 )) && _saw_url=true + if (( _rc == 0 )); then (( _fetches++ )) else IMG=$(lastfm_artist_image "$name") - if download_if_valid "$IMG" "$local_path/folder.jpg"; then + download_if_valid "$IMG" "$local_path/folder.jpg"; _rc=$? + (( _rc == 2 )) && _saw_url=true + if (( _rc == 0 )); then (( _fetches++ )) + elif [[ "$_saw_url" == true ]]; then + (( _dlfail++ )) else - (( _fails++ )) + (( _nourl++ )); _miss_keys+="${mbid}:folder.jpg"$'\t'"${NOW}"$'\n' fi fi fi fi if [[ ! -f "$local_path/fanart.jpg" ]]; then + _saw_url=false IMG=$(echo "$JSON" | jq -r '.artistbackground[0].url // empty' 2>/dev/null) - if download_if_valid "$IMG" "$local_path/fanart.jpg"; then + download_if_valid "$IMG" "$local_path/fanart.jpg"; _rc=$? + (( _rc == 2 )) && _saw_url=true + if (( _rc == 0 )); then (( _fetches++ )) else IMG=$(deezer_artist_image "$name") - if download_if_valid "$IMG" "$local_path/fanart.jpg"; then (( _fetches++ )); else (( _fails++ )); fi + download_if_valid "$IMG" "$local_path/fanart.jpg"; _rc=$? + (( _rc == 2 )) && _saw_url=true + if (( _rc == 0 )); then + (( _fetches++ )) + elif [[ "$_saw_url" == true ]]; then + (( _dlfail++ )) + else + (( _nourl++ )); _miss_keys+="${mbid}:fanart.jpg"$'\t'"${NOW}"$'\n' + fi fi fi - if [[ ! -f "$local_path/clearlogo.png" ]]; then - IMG=$(echo "$JSON" | jq -r '.hdmusiclogo[0].url // empty' 2>/dev/null) - if download_if_valid "$IMG" "$local_path/clearlogo.png"; then (( _fetches++ )); else (( _fails++ )); fi - fi + try_single "$local_path/clearlogo.png" \ + "$(echo "$JSON" | jq -r '.hdmusiclogo[0].url // empty' 2>/dev/null)" \ + "${mbid}:clearlogo.png" - if [[ ! -f "$local_path/banner.jpg" ]]; then - IMG=$(echo "$JSON" | jq -r '.musicbanner[0].url // empty' 2>/dev/null) - if download_if_valid "$IMG" "$local_path/banner.jpg"; then (( _fetches++ )); else (( _fails++ )); fi - fi + try_single "$local_path/banner.jpg" \ + "$(echo "$JSON" | jq -r '.musicbanner[0].url // empty' 2>/dev/null)" \ + "${mbid}:banner.jpg" (( _fetches > 0 )) && printf '1\n' >> "$LIDARR_TMP/artist_fetches" - (( _fails > 0 )) && printf '1\n' >> "$LIDARR_TMP/artist_fails" + (( _nourl > 0 )) && printf '1\n' >> "$LIDARR_TMP/artist_nourl" + (( _dlfail > 0 )) && printf '1\n' >> "$LIDARR_TMP/artist_dlfail" + [[ -n "$_miss_keys" ]] && printf '%s' "$_miss_keys" >> "$LIDARR_TMP/artist_misses" ) & done < <(echo "$artists" | jq -r '.[] | [.path, .foreignArtistId, .artistName] | @tsv') wait +merge_art_misses "$LIDARR_TMP/artist_misses" + ARTIST_FETCHED=$(wc -l < "$LIDARR_TMP/artist_fetches" 2>/dev/null || echo 0) -ARTIST_FAILED=$(wc -l < "$LIDARR_TMP/artist_fails" 2>/dev/null || echo 0) +ARTIST_NOART=$(wc -l < "$LIDARR_TMP/artist_nourl" 2>/dev/null || echo 0) +ARTIST_DLFAIL=$(wc -l < "$LIDARR_TMP/artist_dlfail" 2>/dev/null || echo 0) ARTIST_MISSING=$(( ARTISTS_CHECKED - ARTISTS_COMPLETE )) -info "Checked: $ARTISTS_CHECKED | Complete: $ARTISTS_COMPLETE | Needed art: $ARTIST_MISSING | Fetched: $ARTIST_FETCHED | Failed: $ARTIST_FAILED" +info "Checked: $ARTISTS_CHECKED | Complete: $ARTISTS_COMPLETE | Needed art: $ARTIST_MISSING | Cached-skip: $ARTISTS_CACHED | Fetched: $ARTIST_FETCHED | No art upstream: $ARTIST_NOART | Fetch failed: $ARTIST_DLFAIL" END=$(date +%s) @@ -489,8 +651,11 @@ echo "" echo "━━━━━ $ICON_SUMMARY LIDARR MISSING ART SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -echo "$ICON_EMBY Albums: $ALBUMS_CHECKED checked | $ALBUMS_COMPLETE complete | $ALBUM_FETCHED fetched | $ALBUM_FAILED failed" -echo "$ICON_EMBY Artists: $ARTISTS_CHECKED checked | $ARTISTS_COMPLETE complete | $ARTIST_FETCHED fetched | $ARTIST_FAILED failed" +echo "$ICON_EMBY Albums: $ALBUMS_CHECKED checked | $ALBUMS_COMPLETE complete | $ALBUM_FETCHED fetched" +echo "$ICON_SKIP Albums: $ALBUMS_CACHED skipped (cached miss) | $ALBUM_NOART no art upstream | $ALBUM_DLFAIL fetch failed" +echo "$ICON_EMBY Artists: $ARTISTS_CHECKED checked | $ARTISTS_COMPLETE complete | $ARTIST_FETCHED fetched" +echo "$ICON_SKIP Artists: $ARTISTS_CACHED skipped (cached miss) | $ARTIST_NOART no art upstream | $ARTIST_DLFAIL fetch failed" +echo "$ICON_GEAR Cache: $([[ -f "$LIDARR_ART_MISS_CACHE" ]] && wc -l < "$LIDARR_ART_MISS_CACHE" || echo 0) known upstream misses | recheck every ${LIDARR_ART_RECHECK_DAYS}d" if [[ "$DRY_RUN" == true ]]; then echo "$ICON_WARN Status: DRY RUN — no files written" diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 224ff80..c9d5797 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -1099,6 +1099,8 @@ LIDARR_ART_MAX_PARALLEL=4 # concurrent background download jobs LIDARR_ART_RETRIES=2 # download retry attempts per image LIDARR_ART_SLEEP_BETWEEN=0.2 # seconds between fanart.tv API calls + LIDARR_ART_RECHECK_DAYS=30 # days before re-querying art that upstream didn't have + LIDARR_ART_MISS_CACHE="${DATA_DIR}/lidarr_art_miss_cache.tsv" # negative cache — art upstream has never had # HOST*_FANART_API_KEY / HOST*_LASTFM_API_KEY — set in host*.conf # Lidarr discovery settings (playback_aware_lidarr_discovery.sh)