From 59cee06f45b20d2d43f726b18e3f93a2a90a1685 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Thu, 16 Jul 2026 14:02:35 -0400 Subject: [PATCH] Add Lidarr tracked-data cache + duplicate artist cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shared cache (lidarr_get_tracked_data() in common.sh) so scripts stop hitting Lidarr's live API for tracked counts every run, and stop treating a mid-rescan dip as a genuine problem — a whole-library RescanFolders legitimately makes trackFileCount read far below normal while it re-verifies every file (confirmed 2026-07-16: 22% of normal mid-scan). Cache reads fresh-if-recent, waits out an active rescan (calibrated to that command's own historical duration, tracked per command name since RescanFolders and DownloadedAlbumsScan take wildly different amounts of time), then falls back to a stale cache rather than hard-failing after a few strikes. lidarr_cleanup.sh: no longer stacks a fresh DownloadedAlbumsScan on top of one already running, and the tracked-count floor check now waits out a genuine rescan instead of aborting on every overlap. lidarr_duplicate_artist_cleanup.sh (new): finds case-insensitive duplicate artist entries — same display name, different MusicBrainz ID, added when a search/list-sync matches the wrong same-named artist. Deletes the empty phantom side and blocks it from Import List Exclusions, leaves genuinely-different-real-artists alone (checked by album title overlap, deduped per-artist first so a legitimate reissue under an artist's own catalog doesn't false-flag as cross-artist overlap), and only notifies for the rare case where both sides have real, overlapping content. lidarr_cache_prefill.sh (new): warms the cache at array start so nothing reads it cold after boot. lidarr_missing_art.sh, lidarr_release_fixer.sh: write-through the cache as a side effect of fetches they already needed for their own purposes. --- Arrs_Stack/lidarr_cache_prefill.sh | 81 +++++ Arrs_Stack/lidarr_cleanup.sh | 116 +++++-- Arrs_Stack/lidarr_duplicate_artist_cleanup.sh | 313 ++++++++++++++++++ Arrs_Stack/lidarr_missing_art.sh | 3 + Arrs_Stack/lidarr_release_fixer.sh | 3 + Deployment/master.conf.template | 10 + common.sh | 167 +++++++++- 7 files changed, 667 insertions(+), 26 deletions(-) create mode 100755 Arrs_Stack/lidarr_cache_prefill.sh create mode 100755 Arrs_Stack/lidarr_duplicate_artist_cleanup.sh diff --git a/Arrs_Stack/lidarr_cache_prefill.sh b/Arrs_Stack/lidarr_cache_prefill.sh new file mode 100755 index 0000000..8b84976 --- /dev/null +++ b/Arrs_Stack/lidarr_cache_prefill.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# ============================================================================================== +# ================================= Lidarr Cache Prefill ======================================== +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Populates the shared Lidarr tracked-data cache (see lidarr_get_tracked_data() in common.sh) +# once at array start, before any other script needs it. Without this, the cache stays cold +# from boot until whichever Lidarr-touching script happens to run first and write through — +# which could be hours, depending on the daily schedule. One-shot: runs and exits. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Lidarr's container may still be starting when this fires (array just started) — retries +# reaching the API for up to LIDARR_PREFILL_WAIT_MINUTES before giving up. Not fatal if it +# never comes up in time; the cache just stays cold until the next script writes through +# naturally, exactly as it would without this script existing at all. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# host*.conf +# HOST1_LIDARR_URL / HOST1_LIDARR_API_KEY — aliased by detect_hosts() +# +# master.conf +# LIDARR_PREFILL_WAIT_MINUTES — how long to retry reaching Lidarr before giving up (default 10) +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../load_config.sh" + +parse_args "$@" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + warn "jq not found — skipping Lidarr cache prefill" + exit 0 +fi + +acquire_lock + +detect_hosts + +if [[ -z "${LIDARR_URL:-}" ]] || [[ -z "${LIDARR_API_KEY:-}" ]]; then + info "Lidarr not configured on $MY_ID ($LOCAL_SERVER_NAME) — nothing to prefill" + exit 0 +fi + +WAIT_MINUTES="${LIDARR_PREFILL_WAIT_MINUTES:-10}" +WAITED=0 +until check_api "$LIDARR_URL" "Lidarr" 5 >/dev/null 2>&1; do + if [[ "$WAITED" -ge $(( WAIT_MINUTES * 60 )) ]]; then + warn "Lidarr not reachable after ${WAIT_MINUTES}m — leaving cache cold, next script will write through" + exit 0 + fi + sleep 15 + (( WAITED += 15 )) +done + +ARTISTS=$(arr_api "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "artist" "Lidarr") || { + warn "Could not fetch artists for cache prefill — leaving cache cold" + exit 0 +} + +if lidarr_cache_write "$ARTISTS"; then + log "$ICON_DONE Lidarr cache prefilled ($(echo "$ARTISTS" | jq 'length') artists)" +else + warn "Failed to write Lidarr cache prefill" +fi + +exit 0 diff --git a/Arrs_Stack/lidarr_cleanup.sh b/Arrs_Stack/lidarr_cleanup.sh index 87dd6bc..ed29c0f 100755 --- a/Arrs_Stack/lidarr_cleanup.sh +++ b/Arrs_Stack/lidarr_cleanup.sh @@ -10,6 +10,16 @@ # that is old enough to be past the import window. Triggers an Emby library # clean after runs where files were deleted so ghost entries disappear immediately. # +# Pre-flight and the tracked-count floor check are both rescan-aware: a library-wide +# rescan legitimately makes tracked counts read low mid-scan (confirmed 2026-07-16 — +# 22% of normal during an active RescanFolders), which used to trigger this script's own +# hard abort every time it overlapped with a real rescan. Now it checks for an active +# rescan-type command first — if one's running, it waits (calibrated to that command's +# own historical duration via lidarr_get_rescan_duration(), up to 3 strikes) and re-fetches +# rather than either stacking a duplicate scan or crying wolf on a normal, if slow, state. +# Only escalates to the scary abort-and-notify when the count is genuinely low AND nothing +# is actively rescanning. +# # ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== @@ -246,16 +256,32 @@ for _cp in "${!ARR_PATH_MAP[@]}"; do done unset _cp -if [[ -n "$LIDARR_CONTAINER_ROOT" ]]; then +# Don't stack a fresh scan on top of one already running — confirmed 2026-07-16 that +# repeated runs each firing their own DownloadedAlbumsScan piled up in Lidarr's command +# queue behind each other rather than replacing/coalescing, contributing to a multi-hour +# backlog. If something's already scanning, just wait for that one instead. +_already_active=$(lidarr_active_rescan_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1") + +if [[ -n "$_already_active" ]]; then + info "$_already_active already in progress — waiting for it instead of starting a new scan" + _wait=$(( $(lidarr_get_rescan_duration "$_already_active" "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}") )) + _polled=0 + while [[ "$_polled" -lt "$_wait" ]]; do + [[ -z "$(lidarr_active_rescan_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1")" ]] && break + sleep 15 + (( _polled += 15 )) + [[ $(( _polled % 60 )) -eq 0 ]] && log " Still waiting on $_already_active... (${_polled}s elapsed)" + done +elif [[ -n "$LIDARR_CONTAINER_ROOT" ]]; then info "Triggering DownloadedAlbumsScan on: $LIDARR_CONTAINER_ROOT" SCAN_PAYLOAD="{\"name\": \"DownloadedAlbumsScan\", \"path\": \"$LIDARR_CONTAINER_ROOT\"}" + trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$SCAN_PAYLOAD" "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}" else info "No path map match — triggering DownloadedAlbumsScan (all root folders)" SCAN_PAYLOAD='{"name": "DownloadedAlbumsScan"}' + trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$SCAN_PAYLOAD" "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}" fi -trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$SCAN_PAYLOAD" "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}" - # ============================================================================================== # ━━━ Fetch Lidarr Tracked Files ━━━ # ============================================================================================== @@ -295,30 +321,38 @@ fi info "Found $ARTIST_COUNT artists — fetching track files..." TRACKED_FILE="$TMP_DIR/tracked_paths.txt" -> "$TRACKED_FILE" -while IFS= read -r artist_id; do - [[ -z "$artist_id" ]] && continue - ARTIST_TRACKS=$(arr_api "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "trackFile?artistId=${artist_id}" "Lidarr" 2>/dev/null) - if [[ -n "$ARTIST_TRACKS" ]]; then - while IFS= read -r api_path; do - [[ -z "$api_path" ]] && continue - translate_path "$api_path" >> "$TRACKED_FILE" - done < <(echo "$ARTIST_TRACKS" | jq -r '.[].path' 2>/dev/null) - fi -done <<< "$ARTIST_IDS" +# Fetches every artist's track-file paths fresh into TRACKED_FILE/TRACKED_MAP/TRACKED_COUNT. +# Pulled into a function so the rescan-aware retry below can re-fetch after waiting without +# duplicating this whole loop inline. +_fetch_tracked_files() { + > "$TRACKED_FILE" + while IFS= read -r artist_id; do + [[ -z "$artist_id" ]] && continue + ARTIST_TRACKS=$(arr_api "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "trackFile?artistId=${artist_id}" "Lidarr" 2>/dev/null) + if [[ -n "$ARTIST_TRACKS" ]]; then + while IFS= read -r api_path; do + [[ -z "$api_path" ]] && continue + translate_path "$api_path" >> "$TRACKED_FILE" + done < <(echo "$ARTIST_TRACKS" | jq -r '.[].path' 2>/dev/null) + fi + done <<< "$ARTIST_IDS" -sort -u "$TRACKED_FILE" -o "$TRACKED_FILE" + sort -u "$TRACKED_FILE" -o "$TRACKED_FILE" -# Build in-memory lookup map — O(1) per lookup vs O(n) grep per file -# Eliminates the main performance bottleneck for large libraries -declare -A TRACKED_MAP -while IFS= read -r _tracked_path; do - [[ -n "$_tracked_path" ]] && TRACKED_MAP["$_tracked_path"]=1 -done < "$TRACKED_FILE" -unset _tracked_path + # Build in-memory lookup map — O(1) per lookup vs O(n) grep per file + # Eliminates the main performance bottleneck for large libraries + unset TRACKED_MAP + declare -gA TRACKED_MAP + while IFS= read -r _tracked_path; do + [[ -n "$_tracked_path" ]] && TRACKED_MAP["$_tracked_path"]=1 + done < "$TRACKED_FILE" + unset _tracked_path + TRACKED_COUNT=$(wc -l < "$TRACKED_FILE") +} + +_fetch_tracked_files info "Built in-memory lookup map: ${#TRACKED_MAP[@]} tracked paths" -TRACKED_COUNT=$(wc -l < "$TRACKED_FILE") # Safety Layer 5 — tracked count > 0 if [[ "$TRACKED_COUNT" -eq 0 ]]; then @@ -330,7 +364,41 @@ fi info "$ARTIST_COUNT artists | $TRACKED_COUNT tracked files" -# Safety Layer 6 — percentage drop vs last known count +# Safety Layer 6 — percentage drop vs last known count, with rescan-aware retry. +# A library-wide rescan legitimately makes tracked counts read low mid-scan — sometimes +# dramatically (confirmed 2026-07-16: 22% of normal during an active RescanFolders). +# That's not "something's wrong," it's Lidarr actively re-verifying every file. Wait it out +# (calibrated to that command's own historical duration) before treating a drop as a genuine +# problem worth the scary abort-and-notify. Only escalates to the hard abort in +# check_tracked_count_floor if the count is still low AND nothing is actively rescanning — +# that combination is the actually-suspicious case the floor check exists to catch. +_last_known=$(cat "$LIDARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0) +if [[ "$_last_known" -gt 0 ]]; then + _strike=1 + while [[ "$_strike" -le 3 ]]; do + _pct=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $_last_known) * 100}") + [[ "$_pct" -ge "${LIDARR_MIN_TRACKED_PCT:-50}" ]] && break + + _active_cmd=$(lidarr_active_rescan_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1") + [[ -z "$_active_cmd" ]] && break # low count, nothing rescanning — genuine, don't retry + + _wait=$(( $(lidarr_get_rescan_duration "$_active_cmd" 300) / 2 )) + [[ "$_wait" -lt 30 ]] && _wait=30 + warn "Tracked count ${_pct}% of last run, but $_active_cmd active — waiting ${_wait}s (strike ${_strike}/3)" + sleep "$_wait" + _fetch_tracked_files + (( _strike++ )) + done + + if [[ "$_strike" -gt 3 ]]; then + _active_cmd=$(lidarr_active_rescan_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1") + if [[ -n "$_active_cmd" ]]; then + warn "Lidarr still busy ($_active_cmd) after 3 strikes — deferring to next scheduled run" + exit 0 + fi + fi +fi + check_tracked_count_floor "$TRACKED_COUNT" "$LIDARR_TRACKED_COUNT_FILE" "$LIDARR_MIN_TRACKED_PCT" "Lidarr Cleanup" # ============================================================================================== diff --git a/Arrs_Stack/lidarr_duplicate_artist_cleanup.sh b/Arrs_Stack/lidarr_duplicate_artist_cleanup.sh new file mode 100755 index 0000000..6a7de5e --- /dev/null +++ b/Arrs_Stack/lidarr_duplicate_artist_cleanup.sh @@ -0,0 +1,313 @@ +#!/bin/bash +# ============================================================================================== +# ======================= Lidarr Duplicate Artist Cleanup ======================================= +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Detects and resolves duplicate artist entries in Lidarr's library — cases where the +# same display name (case-insensitive) is backed by two different MusicBrainz artist +# IDs. This happens when a search or list sync matches a same-named-but-different real +# artist and adds it alongside the one already in the library. From that point on, +# every completed download for that display name throws MultipleArtistsFoundException +# and can never import — Lidarr correctly refuses to guess which of the two it means. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# For each case-insensitive duplicate artist name found: +# +# Only one side has any tracked files +# → The zero-file side is a phantom — delete it (deleteFiles=false, nothing on disk +# to lose) and add its MusicBrainz ID to Lidarr's Import List Exclusions so it +# can't be silently re-added by a future list sync. Then trigger a refresh on the +# surviving artist so anything that was stuck on this exact ambiguity resolves +# immediately instead of waiting for Lidarr's own next check cycle. +# +# Both sides have files, but their album titles don't overlap at all +# → Genuinely two different real artists sharing a name (e.g. a band's classic +# lineup vs. a later solo era with the same stage name). Not a bug — left alone. +# +# Both sides have files AND overlapping album titles +# → The one case actually risky to automate: could mean real content is split +# across two entries and needs an actual merge, not a delete. Untouched, +# notified for manual review. +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Never Delete Real Content +# Only the zero-tracked-file side of a pair is ever deleted. Anything with files is +# either left alone (disjoint albums) or flagged for a human (overlapping albums) — +# never auto-removed. +# +# Block Re-Addition At The Source +# A phantom that keeps coming back is worse than one that was never cleaned — +# Import List Exclusion is Lidarr's own mechanism for "never auto-add this again." +# +# Silent When Clean +# No duplicates found, or all duplicates are the simple phantom case — minimal +# output. Only ambiguous (overlapping-album) pairs produce a notification. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# host*.conf +# +# HOST1_LIDARR_URL / HOST1_LIDARR_API_KEY — aliased by detect_hosts() +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# lidarr_duplicate_artist_cleanup.sh — normal run +# lidarr_duplicate_artist_cleanup.sh --dry-run — preview, no deletions or exclusions +# lidarr_duplicate_artist_cleanup.sh --log — verbose per-pair output +# lidarr_duplicate_artist_cleanup.sh --status — show config and exit +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../load_config.sh" + +parse_args "$@" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + error "jq not found — required for JSON parsing" + notify "lidarr_duplicate_artist_cleanup failed on $(hostname) — jq not installed" \ + "Lidarr Duplicate Cleanup" "warning" + exit 1 +fi + +acquire_lock "wait" + +# detect_hosts() sets MY_ID and aliases LIDARR_URL, LIDARR_API_KEY +detect_hosts + +if [[ -z "${LIDARR_URL:-}" ]] || [[ -z "${LIDARR_API_KEY:-}" ]]; then + info "Lidarr not configured on $MY_ID ($LOCAL_SERVER_NAME) — skipping" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no artists will be deleted or excluded" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_GEAR Lidarr URL: $LIDARR_URL" + echo "$ICON_GEAR Lidarr ver: v${LIDARR_VERSION_MAJOR:-3} expected" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Safety Checks ━━━ +# ============================================================================================== +check_api "$LIDARR_URL" "Lidarr" 10 || { + notify "Lidarr duplicate cleanup aborted on $(hostname) — API unreachable" \ + "Lidarr Duplicate Cleanup" "warning" + exit 1 +} +check_arr_version "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "${LIDARR_VERSION_MAJOR:-3}" "Lidarr" || exit 1 + +TMP_DIR=$(mktemp -d) +trap 'rm -rf "$TMP_DIR"' EXIT + +# ============================================================================================== +# ━━━ Fetch Artists (cache-aware — waits out an active rescan rather than trusting a +# mid-scan number, falls back to cache if Lidarr's still busy after the strike limit) ━━━ +# ============================================================================================== +ARTISTS=$(lidarr_get_tracked_data "$LIDARR_URL" "$LIDARR_API_KEY" "v1") +if [[ -z "$ARTISTS" ]]; then + warn "Lidarr busy and no usable cache — deferring to next scheduled run" + exit 0 +fi + +ARTIST_COUNT=$(echo "$ARTISTS" | jq 'length' 2>/dev/null) +if [[ -z "$ARTIST_COUNT" || "$ARTIST_COUNT" -eq 0 ]]; then + error "API returned 0 artists — aborting to avoid acting on empty data" + notify "Lidarr duplicate cleanup aborted on $(hostname) — 0 artists returned" \ + "Lidarr Duplicate Cleanup" "warning" + exit 1 +fi +log "$ICON_GEAR Fetched $ARTIST_COUNT artists" + +# Safety Layer — same shared baseline as lidarr_cleanup.sh. A library-wide desync (e.g. mid +# full-rescan) can make trackFileCount read far lower than reality for many artists at once — +# confirmed 2026-07-16, where this exact scenario would have made the script see both sides +# of a genuinely-real duplicate (ROMES) as 0-file phantoms and delete the wrong thing entirely. +# Reuses lidarr_cleanup.sh's own baseline file — one shared "is Lidarr's data trustworthy +# right now" answer for every script that depends on tracked counts, not a separate opinion +# per script. +TOTAL_TRACKED=$(echo "$ARTISTS" | jq '[.[].statistics.trackFileCount] | add' 2>/dev/null) +check_tracked_count_floor "${TOTAL_TRACKED:-0}" "$LIDARR_TRACKED_COUNT_FILE" "${LIDARR_MIN_TRACKED_PCT:-50}" "Lidarr Duplicate Cleanup" + +# ============================================================================================== +# ━━━ Find Case-Insensitive Duplicate Names ━━━ +# ============================================================================================== +DUP_NAMES=$(echo "$ARTISTS" | jq -r '.[].artistName' | tr '[:upper:]' '[:lower:]' | sort | uniq -d) + +if [[ -z "$DUP_NAMES" ]]; then + echo "Lidarr — clean ✅ no duplicate artist names" + exit 0 +fi + +DUP_COUNT=$(echo "$DUP_NAMES" | grep -c .) +warn "Found $DUP_COUNT duplicate artist name(s)" + +DELETED=0 +EXCLUDED=0 +LEFT_ALONE=0 +FLAGGED=0 +FLAGGED_NAMES=() + +# ============================================================================================== +# ━━━ Resolve Each Duplicate ━━━ +# ============================================================================================== +while IFS= read -r lname; do + [[ -z "$lname" ]] && continue + + MEMBERS=$(echo "$ARTISTS" | jq -c --arg n "$lname" '[.[] | select((.artistName|ascii_downcase)==$n)]') + + NONZERO_IDS=() + ZERO_MEMBERS_FILE="$TMP_DIR/zero_${RANDOM}.jsonl" + : > "$ZERO_MEMBERS_FILE" + + while IFS= read -r member; do + [[ -z "$member" ]] && continue + fc=$(echo "$member" | jq -r '.statistics.trackFileCount // 0') + if [[ "$fc" -gt 0 ]]; then + NONZERO_IDS+=("$(echo "$member" | jq -r '.id')") + else + echo "$member" >> "$ZERO_MEMBERS_FILE" + fi + done < <(echo "$MEMBERS" | jq -c '.[]') + + if [[ "${#NONZERO_IDS[@]}" -le 1 ]]; then + # Simple phantom case — delete every zero-file member, keep the real one (if any) + while IFS= read -r zmember; do + [[ -z "$zmember" ]] && continue + zid=$(echo "$zmember" | jq -r '.id') + zname=$(echo "$zmember" | jq -r '.artistName') + zmbid=$(echo "$zmember" | jq -r '.foreignArtistId') + + if [[ "$DRY_RUN" == true ]]; then + warn " DRY RUN — would delete phantom: $zname ($zid) and exclude MBID $zmbid" + continue + fi + + if curl -sf --max-time 15 -X DELETE \ + "${LIDARR_URL}/api/v1/artist/${zid}?deleteFiles=false" \ + -H "X-Api-Key: $LIDARR_API_KEY" >/dev/null 2>&1; then + (( DELETED++ )) + log " $ICON_TRASH Deleted phantom: $zname ($zid)" + else + warn " Failed to delete phantom: $zname ($zid)" + continue + fi + + EXCL_PAYLOAD=$(jq -c -n --arg fid "$zmbid" --arg name "$zname" \ + '{foreignId:$fid, artistName:$name}') + if curl -sf --max-time 15 -X POST \ + "${LIDARR_URL}/api/v1/importlistexclusion" \ + -H "X-Api-Key: $LIDARR_API_KEY" -H "Content-Type: application/json" \ + -d "$EXCL_PAYLOAD" >/dev/null 2>&1; then + (( EXCLUDED++ )) + log " Added to import list exclusions: $zmbid" + else + warn " Could not add exclusion for $zname ($zmbid) — may already exist" + fi + done < "$ZERO_MEMBERS_FILE" + + # Nudge the surviving real artist so anything stuck on this ambiguity + # resolves now rather than waiting for Lidarr's own next check cycle. + if [[ "${#NONZERO_IDS[@]}" -eq 1 && "$DRY_RUN" != true ]]; then + curl -sf --max-time 15 -X POST \ + "${LIDARR_URL}/api/v1/command" \ + -H "X-Api-Key: $LIDARR_API_KEY" -H "Content-Type: application/json" \ + -d "{\"name\":\"RefreshArtist\",\"artistId\":${NONZERO_IDS[0]}}" >/dev/null 2>&1 + fi + else + # 2+ members have real content — same name, need to know if it's the same + # artist actually split (album overlap) or genuinely different acts (no overlap). + # + # Titles must be deduped WITHIN each artist before checking overlap ACROSS artists — + # a single artist can legitimately list the same album title twice (a reissue, a + # deluxe edition under an unchanged title). Confirmed 2026-07-16: Alice Cooper's own + # catalog has "Lace and Whiskey" and "School's Out" each listed twice under one + # artist ID — treating that as "overlap" false-flagged a genuinely disjoint pair + # (band-era vs. solo-era) as ambiguous when it wasn't. + OVERLAP=false + declare -A GLOBAL_TITLES + for nid in "${NONZERO_IDS[@]}"; do + declare -A THIS_ARTIST_TITLES + while IFS= read -r title; do + [[ -z "$title" ]] && continue + THIS_ARTIST_TITLES["$title"]=1 + done < <(curl -sf --max-time 15 "${LIDARR_URL}/api/v1/album?artistId=${nid}" \ + -H "X-Api-Key: $LIDARR_API_KEY" 2>/dev/null | jq -r '.[].title') + + for title in "${!THIS_ARTIST_TITLES[@]}"; do + [[ -n "${GLOBAL_TITLES[$title]:-}" ]] && OVERLAP=true + GLOBAL_TITLES["$title"]=1 + done + unset THIS_ARTIST_TITLES + done + unset GLOBAL_TITLES + + if [[ "$OVERLAP" == true ]]; then + warn " $ICON_WARN Ambiguous duplicate needs manual review: $lname (both have files, albums overlap)" + FLAGGED_NAMES+=("$lname") + (( FLAGGED++ )) + else + log " $lname — different real artists sharing a name, no album overlap, no action" + (( LEFT_ALONE++ )) + fi + fi +done <<< "$DUP_NAMES" + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== +echo "" +echo "━━━━━ $ICON_SUMMARY LIDARR DUPLICATE ARTIST CLEANUP SUMMARY ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_TRASH Deleted: $DELETED phantom artist(s)" +echo "$ICON_GEAR Excluded: $EXCLUDED (blocked from future re-add)" +echo "$ICON_SKIP Left alone: $LEFT_ALONE (different real artists, no overlap)" +echo "$ICON_WARN Flagged: $FLAGGED (needs manual review)" +for n in "${FLAGGED_NAMES[@]}"; do + echo " - $n" +done +echo "" + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — no changes made" +elif [[ "$FLAGGED" -gt 0 ]]; then + notify "Lidarr duplicate cleanup on $(hostname) — $DELETED phantom(s) removed, $FLAGGED artist(s) need manual review: ${FLAGGED_NAMES[*]}" \ + "Lidarr Duplicate Cleanup" "warning" +elif [[ "$DELETED" -gt 0 ]]; then + echo "$ICON_DONE Status: cleaned $DELETED phantom artist(s), nothing needs review" +else + echo "$ICON_DONE Status: no action needed" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +exit 0 diff --git a/Arrs_Stack/lidarr_missing_art.sh b/Arrs_Stack/lidarr_missing_art.sh index 4e44082..2d45ff2 100755 --- a/Arrs_Stack/lidarr_missing_art.sh +++ b/Arrs_Stack/lidarr_missing_art.sh @@ -255,6 +255,9 @@ echo "━━━ $ICON_SYNC Building Album Directory Map ━━━" declare -A ALBUM_DIR_MAP _artist_list=$(curl_json "$LIDARR_URL/api/v1/artist?apikey=$LIDARR_API_KEY") +# Write-through — keeps the shared tracked-data cache fresh as a side effect of a fetch +# this script already needed for its own purposes. See lidarr_get_tracked_data() in common.sh. +lidarr_cache_write "$_artist_list" _map_artist_count=$(echo "$_artist_list" | jq '. | length') info "Fetching track files for $_map_artist_count artists..." diff --git a/Arrs_Stack/lidarr_release_fixer.sh b/Arrs_Stack/lidarr_release_fixer.sh index 47deee9..a8d362d 100755 --- a/Arrs_Stack/lidarr_release_fixer.sh +++ b/Arrs_Stack/lidarr_release_fixer.sh @@ -306,6 +306,9 @@ ALL_ARTISTS=$(lidarr_api "artist") || { error "Failed to fetch artists" exit 1 } +# Write-through — keeps the shared tracked-data cache fresh as a side effect of a fetch +# this script already needed for its own purposes. See lidarr_get_tracked_data() in common.sh. +lidarr_cache_write "$ALL_ARTISTS" while IFS= read -r artist; do aid=$(echo "$artist" | jq -r '.id') diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 58938bd..fccd953 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -300,6 +300,7 @@ "unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning "unRAID_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers + "Arrs_Stack/lidarr_cache_prefill.sh" # warm Lidarr tracked-data cache before anything reads it cold "Arrs_Stack/start_webhook_listener.sh" # arr upgrade webhook listener — continuous "Fallback/fallback.sh" # mutual failover — continuous ) @@ -387,6 +388,7 @@ "Media/media_cleaner.sh anime" # remove junk from anime shares "Media/media_cleaner.sh media" # remove junk from media shares "Arrs_Stack/lidarr_release_fixer.sh" # fix wrong release editions before cleanup runs + "Arrs_Stack/lidarr_duplicate_artist_cleanup.sh" # resolve duplicate MusicBrainz artist entries before cleanup runs "Arrs_Stack/lidarr_cleanup.sh" # remove orphaned music files "Arrs_Stack/sonarr_cleanup.sh" # remove orphaned TV files "Arrs_Stack/radarr_cleanup.sh" # remove orphaned movie files @@ -1089,6 +1091,14 @@ # protects against API returning partial data on a bad day LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count" LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan + +# Lidarr tracked-data cache — shared by lidarr_cleanup.sh, lidarr_duplicate_artist_cleanup.sh, +# lidarr_missing_art.sh, lidarr_release_fixer.sh, and lidarr_cache_prefill.sh. See +# lidarr_get_tracked_data() in common.sh for the fresh/stale/rescan-active branching logic. + LIDARR_CACHE_FILE="$DATA_DIR/lidarr_tracked_cache.json" + LIDARR_RESCAN_DURATION_DB="$DATA_DIR/lidarr_rescan_duration.db" + LIDARR_CACHE_MAX_AGE_DAYS=1 # force a live refresh (or rescan-aware wait) past this age + LIDARR_PREFILL_WAIT_MINUTES=10 # array-start prefill: how long to retry reaching Lidarr LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") LIDARR_PROTECTED_PATTERNS=( # Metadata diff --git a/common.sh b/common.sh index 464a618..59824f6 100755 --- a/common.sh +++ b/common.sh @@ -2174,11 +2174,22 @@ arr_api() { # Triggers an arr "command" endpoint with the given JSON payload, then polls every 10s until # completed/failed/timeout, logging progress every 60s. Best-effort pre-flight — never treats # an unreachable/timed-out scan as fatal, matching original per-script behavior. +# +# On a genuine "completed" observation, records the actual elapsed duration keyed by the +# command's own name via lidarr_record_rescan_duration() — this is the one place in the +# codebase that reliably watches a command from trigger to completion, so it's the natural +# spot to build up real historical duration data for the wait-calibration logic in +# lidarr_get_tracked_data(). A timeout doesn't record anything — we only know a lower bound, +# not the true duration, and recording that would corrupt future wait calculations downward. +# # Usage: trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" \ # '{"name": "DownloadedAlbumsScan"}' "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}" trigger_and_await_command() { local base_url="$1" api_key="$2" api_version="$3" payload="$4" poll_timeout="${5:-600}" + local cmd_name + cmd_name=$(echo "$payload" | jq -r '.name // empty' 2>/dev/null) + local scan_response scan_cmd_id scan_response=$(curl -sf --max-time 30 -X POST \ -H "X-Api-Key: $api_key" \ @@ -2194,6 +2205,7 @@ trigger_and_await_command() { fi info "Import scan queued (command ID: $scan_cmd_id) — waiting for completion..." + local start_epoch=$(date +%s) local polled=0 scan_status while [[ "$polled" -lt "$poll_timeout" ]]; do scan_status=$(curl -sf --max-time 10 \ @@ -2201,8 +2213,15 @@ trigger_and_await_command() { "${base_url}/api/${api_version}/command/${scan_cmd_id}" 2>/dev/null | \ jq -r '.status // empty' 2>/dev/null) case "$scan_status" in - completed) info "Import scan complete ✅"; return 0 ;; - failed) warn "Import scan reported failed — proceeding anyway"; return 0 ;; + completed) + info "Import scan complete ✅" + [[ -n "$cmd_name" ]] && lidarr_record_rescan_duration "$cmd_name" "$(( $(date +%s) - start_epoch ))" + return 0 + ;; + failed) + warn "Import scan reported failed — proceeding anyway" + return 0 + ;; esac sleep 10 (( polled += 10 )) @@ -2212,6 +2231,150 @@ trigger_and_await_command() { return 0 } +# ============================================================================================== +# ── LIDARR TRACKED-DATA CACHE ───────────────────────────────────────────────────────────────── +# ============================================================================================== +# Shared cache for Lidarr's tracked-artist/file-count data. Multiple scripts (lidarr_cleanup.sh, +# lidarr_duplicate_artist_cleanup.sh, lidarr_missing_art.sh, lidarr_release_fixer.sh) all need a +# reasonably-current snapshot of "what does Lidarr think it has tracked." Hitting the live API +# fresh every time is wasteful, and during an active rescan the live number is actively +# misleading — tracked counts dip and recover as files are detached/re-verified one by one +# (confirmed 2026-07-16: a whole-library RescanFolders made trackFileCount read 22% of normal +# mid-scan, which is exactly the false-alarm lidarr_cleanup.sh's count-drop guard is meant to +# catch, but a genuine rescan isn't the "something's actually wrong" case that guard exists for). +# +# Write-through: any script that already does a live artist-list fetch for its own purposes +# writes the result here as a side effect via lidarr_cache_write() — no dedicated polling timer +# needed. Arrs_Stack/lidarr_cache_prefill.sh closes the cold-boot gap by populating the cache +# once at array start, before anything else needs it. +# +# Consumers should call lidarr_get_tracked_data() — never read the cache file directly. It +# handles the fresh/stale-no-rescan/stale-rescan-active branching so no script reimplements it. +# ============================================================================================== + +LIDARR_CACHE_FILE="${LIDARR_CACHE_FILE:-$DATA_DIR/lidarr_tracked_cache.json}" +LIDARR_RESCAN_DURATION_DB="${LIDARR_RESCAN_DURATION_DB:-$DATA_DIR/lidarr_rescan_duration.db}" + +# Writes the current artist list + total tracked count to the shared cache. +# Args: artists_json (the full /api/v1/artist response body) +lidarr_cache_write() { + local artists_json="$1" + local total_files + total_files=$(echo "$artists_json" | jq '[.[].statistics.trackFileCount] | add' 2>/dev/null) + [[ -z "$total_files" || "$total_files" == "null" ]] && return 1 + + mkdir -p "$(dirname "$LIDARR_CACHE_FILE")" 2>/dev/null + jq -c -n --argjson artists "$artists_json" --argjson total "$total_files" --argjson ts "$(date +%s)" \ + '{ts:$ts, totalTrackedFiles:$total, artists:$artists}' > "${LIDARR_CACHE_FILE}.tmp" 2>/dev/null \ + && mv "${LIDARR_CACHE_FILE}.tmp" "$LIDARR_CACHE_FILE" +} + +# Echoes the cached artists JSON array if a cache file exists, regardless of age — staleness +# is lidarr_get_tracked_data()'s decision, not this raw reader's. +lidarr_cache_read_raw() { + [[ -f "$LIDARR_CACHE_FILE" ]] || return 1 + jq -c '.artists // empty' "$LIDARR_CACHE_FILE" 2>/dev/null +} + +# Echoes the cache's age in seconds, or a very large number if no cache exists (so age +# comparisons naturally treat "no cache" the same as "very stale cache"). +lidarr_cache_age_seconds() { + [[ -f "$LIDARR_CACHE_FILE" ]] || { echo 999999999; return; } + local ts + ts=$(jq -r '.ts // 0' "$LIDARR_CACHE_FILE" 2>/dev/null) + echo $(( $(date +%s) - ${ts:-0} )) +} + +# Records how long a rescan-type command actually took, keyed by command name, so future +# waits can be calibrated per command type instead of guessed or blended across very +# different operations — a whole-library RescanFolders takes vastly longer than a targeted +# DownloadedAlbumsScan, and averaging them would miscalibrate the wait for both. +# Args: command_name, duration_seconds +lidarr_record_rescan_duration() { + local cmd_name="$1" duration="$2" + [[ -z "$cmd_name" || -z "$duration" ]] && return 1 + mkdir -p "$(dirname "$LIDARR_RESCAN_DURATION_DB")" 2>/dev/null + local tmp + tmp=$(mktemp) + [[ -f "$LIDARR_RESCAN_DURATION_DB" ]] && grep -v "^${cmd_name}|" "$LIDARR_RESCAN_DURATION_DB" > "$tmp" 2>/dev/null + echo "${cmd_name}|${duration}" >> "$tmp" + mv "$tmp" "$LIDARR_RESCAN_DURATION_DB" +} + +# Echoes the last recorded duration (seconds) for a given command name, or a fallback +# default if none has ever been recorded. +# Args: command_name, fallback_default_seconds +lidarr_get_rescan_duration() { + local cmd_name="$1" fallback="${2:-300}" + [[ -f "$LIDARR_RESCAN_DURATION_DB" ]] || { echo "$fallback"; return; } + local val + val=$(grep "^${cmd_name}|" "$LIDARR_RESCAN_DURATION_DB" 2>/dev/null | tail -1 | cut -d'|' -f2) + echo "${val:-$fallback}" +} + +# Checks Lidarr's command queue for any currently-active (started or queued) rescan-type +# command. Echoes the command name of the first one found (so the caller can look up its +# specific historical duration), empty if none active. +# Args: url, api_key, api_version +lidarr_active_rescan_command() { + local url="$1" api_key="$2" api_version="$3" + curl -sf --max-time 15 -H "X-Api-Key: $api_key" \ + "${url}/api/${api_version}/command" 2>/dev/null | \ + jq -r '[.[] | select( + (.status=="started" or .status=="queued") and + (.name=="RescanFolders" or .name=="DownloadedAlbumsScan" or .name=="RefreshArtist") + )] | .[0].name // empty' 2>/dev/null +} + +# Main entry point — the only function consuming scripts should call for tracked artist data. +# Handles fresh/stale-no-rescan/stale-rescan-active branching and always writes through on any +# live fetch it performs. Echoes the artists JSON array on success, returns 1 if no usable data +# (no cache and live fetch impossible) could be obtained. +# Args: url, api_key, api_version, max_age_days (default 1), max_wait_strikes (default 3) +lidarr_get_tracked_data() { + local url="$1" api_key="$2" api_version="$3" + local max_age_days="${4:-${LIDARR_CACHE_MAX_AGE_DAYS:-1}}" max_strikes="${5:-3}" + local max_age_seconds=$(( max_age_days * 86400 )) + local age + age=$(lidarr_cache_age_seconds) + + if [[ "$age" -lt "$max_age_seconds" ]]; then + lidarr_cache_read_raw && return 0 + fi + + # Cache missing or stale — check for an active rescan before deciding how to proceed. + local active_cmd + active_cmd=$(lidarr_active_rescan_command "$url" "$api_key" "$api_version") + + if [[ -n "$active_cmd" ]]; then + local wait_duration strike + wait_duration=$(( $(lidarr_get_rescan_duration "$active_cmd" 300) / 2 )) + [[ "$wait_duration" -lt 30 ]] && wait_duration=30 + for (( strike=1; strike<=max_strikes; strike++ )); do + # Redirected to stderr — this function's stdout is a data channel (callers + # capture it via command substitution), never mix log/warn output into it. + warn " Lidarr busy ($active_cmd) — waiting ${wait_duration}s (strike ${strike}/${max_strikes})" >&2 + sleep "$wait_duration" + active_cmd=$(lidarr_active_rescan_command "$url" "$api_key" "$api_version") + [[ -z "$active_cmd" ]] && break + done + if [[ -n "$active_cmd" ]]; then + warn " Lidarr still busy ($active_cmd) after ${max_strikes} strikes — using cache if present, else skipping" >&2 + lidarr_cache_read_raw && return 0 + return 1 + fi + fi + + # No active rescan (or it just finished mid-wait) — safe to fetch live and refresh cache. + local fresh + fresh=$(arr_api "$url" "$api_key" "$api_version" "artist" "Lidarr") || { + lidarr_cache_read_raw && return 0 + return 1 + } + lidarr_cache_write "$fresh" + echo "$fresh" +} + # ============================================================================================== # ── ARR VERSION CHECK ───────────────────────────────────────────────────────────────────────── # ==============================================================================================