Add Lidarr tracked-data cache + duplicate artist cleanup
Shared cache (lidarr_get_tracked_data() in common.sh) so scripts stop hitting Lidarr's live API for tracked counts every run, and stop treating a mid-rescan dip as a genuine problem — a whole-library RescanFolders legitimately makes trackFileCount read far below normal while it re-verifies every file (confirmed 2026-07-16: 22% of normal mid-scan). Cache reads fresh-if-recent, waits out an active rescan (calibrated to that command's own historical duration, tracked per command name since RescanFolders and DownloadedAlbumsScan take wildly different amounts of time), then falls back to a stale cache rather than hard-failing after a few strikes. lidarr_cleanup.sh: no longer stacks a fresh DownloadedAlbumsScan on top of one already running, and the tracked-count floor check now waits out a genuine rescan instead of aborting on every overlap. lidarr_duplicate_artist_cleanup.sh (new): finds case-insensitive duplicate artist entries — same display name, different MusicBrainz ID, added when a search/list-sync matches the wrong same-named artist. Deletes the empty phantom side and blocks it from Import List Exclusions, leaves genuinely-different-real-artists alone (checked by album title overlap, deduped per-artist first so a legitimate reissue under an artist's own catalog doesn't false-flag as cross-artist overlap), and only notifies for the rare case where both sides have real, overlapping content. lidarr_cache_prefill.sh (new): warms the cache at array start so nothing reads it cold after boot. lidarr_missing_art.sh, lidarr_release_fixer.sh: write-through the cache as a side effect of fetches they already needed for their own purposes.
This commit is contained in:
Gmer4Lfe
Executable
+81
@@ -0,0 +1,81 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Lidarr Cache Prefill ========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Populates the shared Lidarr tracked-data cache (see lidarr_get_tracked_data() in common.sh)
|
||||
# once at array start, before any other script needs it. Without this, the cache stays cold
|
||||
# from boot until whichever Lidarr-touching script happens to run first and write through —
|
||||
# which could be hours, depending on the daily schedule. One-shot: runs and exits.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Lidarr's container may still be starting when this fires (array just started) — retries
|
||||
# reaching the API for up to LIDARR_PREFILL_WAIT_MINUTES before giving up. Not fatal if it
|
||||
# never comes up in time; the cache just stays cold until the next script writes through
|
||||
# naturally, exactly as it would without this script existing at all.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
# HOST1_LIDARR_URL / HOST1_LIDARR_API_KEY — aliased by detect_hosts()
|
||||
#
|
||||
# master.conf
|
||||
# LIDARR_PREFILL_WAIT_MINUTES — how long to retry reaching Lidarr before giving up (default 10)
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
warn "jq not found — skipping Lidarr cache prefill"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
if [[ -z "${LIDARR_URL:-}" ]] || [[ -z "${LIDARR_API_KEY:-}" ]]; then
|
||||
info "Lidarr not configured on $MY_ID ($LOCAL_SERVER_NAME) — nothing to prefill"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
WAIT_MINUTES="${LIDARR_PREFILL_WAIT_MINUTES:-10}"
|
||||
WAITED=0
|
||||
until check_api "$LIDARR_URL" "Lidarr" 5 >/dev/null 2>&1; do
|
||||
if [[ "$WAITED" -ge $(( WAIT_MINUTES * 60 )) ]]; then
|
||||
warn "Lidarr not reachable after ${WAIT_MINUTES}m — leaving cache cold, next script will write through"
|
||||
exit 0
|
||||
fi
|
||||
sleep 15
|
||||
(( WAITED += 15 ))
|
||||
done
|
||||
|
||||
ARTISTS=$(arr_api "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "artist" "Lidarr") || {
|
||||
warn "Could not fetch artists for cache prefill — leaving cache cold"
|
||||
exit 0
|
||||
}
|
||||
|
||||
if lidarr_cache_write "$ARTISTS"; then
|
||||
log "$ICON_DONE Lidarr cache prefilled ($(echo "$ARTISTS" | jq 'length') artists)"
|
||||
else
|
||||
warn "Failed to write Lidarr cache prefill"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@@ -10,6 +10,16 @@
|
||||
# that is old enough to be past the import window. Triggers an Emby library
|
||||
# clean after runs where files were deleted so ghost entries disappear immediately.
|
||||
#
|
||||
# Pre-flight and the tracked-count floor check are both rescan-aware: a library-wide
|
||||
# rescan legitimately makes tracked counts read low mid-scan (confirmed 2026-07-16 —
|
||||
# 22% of normal during an active RescanFolders), which used to trigger this script's own
|
||||
# hard abort every time it overlapped with a real rescan. Now it checks for an active
|
||||
# rescan-type command first — if one's running, it waits (calibrated to that command's
|
||||
# own historical duration via lidarr_get_rescan_duration(), up to 3 strikes) and re-fetches
|
||||
# rather than either stacking a duplicate scan or crying wolf on a normal, if slow, state.
|
||||
# Only escalates to the scary abort-and-notify when the count is genuinely low AND nothing
|
||||
# is actively rescanning.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
@@ -246,15 +256,31 @@ for _cp in "${!ARR_PATH_MAP[@]}"; do
|
||||
done
|
||||
unset _cp
|
||||
|
||||
if [[ -n "$LIDARR_CONTAINER_ROOT" ]]; then
|
||||
# Don't stack a fresh scan on top of one already running — confirmed 2026-07-16 that
|
||||
# repeated runs each firing their own DownloadedAlbumsScan piled up in Lidarr's command
|
||||
# queue behind each other rather than replacing/coalescing, contributing to a multi-hour
|
||||
# backlog. If something's already scanning, just wait for that one instead.
|
||||
_already_active=$(lidarr_active_rescan_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1")
|
||||
|
||||
if [[ -n "$_already_active" ]]; then
|
||||
info "$_already_active already in progress — waiting for it instead of starting a new scan"
|
||||
_wait=$(( $(lidarr_get_rescan_duration "$_already_active" "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}") ))
|
||||
_polled=0
|
||||
while [[ "$_polled" -lt "$_wait" ]]; do
|
||||
[[ -z "$(lidarr_active_rescan_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1")" ]] && break
|
||||
sleep 15
|
||||
(( _polled += 15 ))
|
||||
[[ $(( _polled % 60 )) -eq 0 ]] && log " Still waiting on $_already_active... (${_polled}s elapsed)"
|
||||
done
|
||||
elif [[ -n "$LIDARR_CONTAINER_ROOT" ]]; then
|
||||
info "Triggering DownloadedAlbumsScan on: $LIDARR_CONTAINER_ROOT"
|
||||
SCAN_PAYLOAD="{\"name\": \"DownloadedAlbumsScan\", \"path\": \"$LIDARR_CONTAINER_ROOT\"}"
|
||||
trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$SCAN_PAYLOAD" "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}"
|
||||
else
|
||||
info "No path map match — triggering DownloadedAlbumsScan (all root folders)"
|
||||
SCAN_PAYLOAD='{"name": "DownloadedAlbumsScan"}'
|
||||
fi
|
||||
|
||||
trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$SCAN_PAYLOAD" "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Fetch Lidarr Tracked Files ━━━
|
||||
@@ -295,8 +321,12 @@ fi
|
||||
info "Found $ARTIST_COUNT artists — fetching track files..."
|
||||
|
||||
TRACKED_FILE="$TMP_DIR/tracked_paths.txt"
|
||||
> "$TRACKED_FILE"
|
||||
|
||||
# Fetches every artist's track-file paths fresh into TRACKED_FILE/TRACKED_MAP/TRACKED_COUNT.
|
||||
# Pulled into a function so the rescan-aware retry below can re-fetch after waiting without
|
||||
# duplicating this whole loop inline.
|
||||
_fetch_tracked_files() {
|
||||
> "$TRACKED_FILE"
|
||||
while IFS= read -r artist_id; do
|
||||
[[ -z "$artist_id" ]] && continue
|
||||
ARTIST_TRACKS=$(arr_api "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "trackFile?artistId=${artist_id}" "Lidarr" 2>/dev/null)
|
||||
@@ -312,13 +342,17 @@ sort -u "$TRACKED_FILE" -o "$TRACKED_FILE"
|
||||
|
||||
# Build in-memory lookup map — O(1) per lookup vs O(n) grep per file
|
||||
# Eliminates the main performance bottleneck for large libraries
|
||||
declare -A TRACKED_MAP
|
||||
unset TRACKED_MAP
|
||||
declare -gA TRACKED_MAP
|
||||
while IFS= read -r _tracked_path; do
|
||||
[[ -n "$_tracked_path" ]] && TRACKED_MAP["$_tracked_path"]=1
|
||||
done < "$TRACKED_FILE"
|
||||
unset _tracked_path
|
||||
info "Built in-memory lookup map: ${#TRACKED_MAP[@]} tracked paths"
|
||||
TRACKED_COUNT=$(wc -l < "$TRACKED_FILE")
|
||||
}
|
||||
|
||||
_fetch_tracked_files
|
||||
info "Built in-memory lookup map: ${#TRACKED_MAP[@]} tracked paths"
|
||||
|
||||
# Safety Layer 5 — tracked count > 0
|
||||
if [[ "$TRACKED_COUNT" -eq 0 ]]; then
|
||||
@@ -330,7 +364,41 @@ fi
|
||||
|
||||
info "$ARTIST_COUNT artists | $TRACKED_COUNT tracked files"
|
||||
|
||||
# Safety Layer 6 — percentage drop vs last known count
|
||||
# Safety Layer 6 — percentage drop vs last known count, with rescan-aware retry.
|
||||
# A library-wide rescan legitimately makes tracked counts read low mid-scan — sometimes
|
||||
# dramatically (confirmed 2026-07-16: 22% of normal during an active RescanFolders).
|
||||
# That's not "something's wrong," it's Lidarr actively re-verifying every file. Wait it out
|
||||
# (calibrated to that command's own historical duration) before treating a drop as a genuine
|
||||
# problem worth the scary abort-and-notify. Only escalates to the hard abort in
|
||||
# check_tracked_count_floor if the count is still low AND nothing is actively rescanning —
|
||||
# that combination is the actually-suspicious case the floor check exists to catch.
|
||||
_last_known=$(cat "$LIDARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0)
|
||||
if [[ "$_last_known" -gt 0 ]]; then
|
||||
_strike=1
|
||||
while [[ "$_strike" -le 3 ]]; do
|
||||
_pct=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $_last_known) * 100}")
|
||||
[[ "$_pct" -ge "${LIDARR_MIN_TRACKED_PCT:-50}" ]] && break
|
||||
|
||||
_active_cmd=$(lidarr_active_rescan_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1")
|
||||
[[ -z "$_active_cmd" ]] && break # low count, nothing rescanning — genuine, don't retry
|
||||
|
||||
_wait=$(( $(lidarr_get_rescan_duration "$_active_cmd" 300) / 2 ))
|
||||
[[ "$_wait" -lt 30 ]] && _wait=30
|
||||
warn "Tracked count ${_pct}% of last run, but $_active_cmd active — waiting ${_wait}s (strike ${_strike}/3)"
|
||||
sleep "$_wait"
|
||||
_fetch_tracked_files
|
||||
(( _strike++ ))
|
||||
done
|
||||
|
||||
if [[ "$_strike" -gt 3 ]]; then
|
||||
_active_cmd=$(lidarr_active_rescan_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1")
|
||||
if [[ -n "$_active_cmd" ]]; then
|
||||
warn "Lidarr still busy ($_active_cmd) after 3 strikes — deferring to next scheduled run"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
check_tracked_count_floor "$TRACKED_COUNT" "$LIDARR_TRACKED_COUNT_FILE" "$LIDARR_MIN_TRACKED_PCT" "Lidarr Cleanup"
|
||||
|
||||
# ==============================================================================================
|
||||
|
||||
+313
@@ -0,0 +1,313 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ======================= Lidarr Duplicate Artist Cleanup =======================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Detects and resolves duplicate artist entries in Lidarr's library — cases where the
|
||||
# same display name (case-insensitive) is backed by two different MusicBrainz artist
|
||||
# IDs. This happens when a search or list sync matches a same-named-but-different real
|
||||
# artist and adds it alongside the one already in the library. From that point on,
|
||||
# every completed download for that display name throws MultipleArtistsFoundException
|
||||
# and can never import — Lidarr correctly refuses to guess which of the two it means.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# For each case-insensitive duplicate artist name found:
|
||||
#
|
||||
# Only one side has any tracked files
|
||||
# → The zero-file side is a phantom — delete it (deleteFiles=false, nothing on disk
|
||||
# to lose) and add its MusicBrainz ID to Lidarr's Import List Exclusions so it
|
||||
# can't be silently re-added by a future list sync. Then trigger a refresh on the
|
||||
# surviving artist so anything that was stuck on this exact ambiguity resolves
|
||||
# immediately instead of waiting for Lidarr's own next check cycle.
|
||||
#
|
||||
# Both sides have files, but their album titles don't overlap at all
|
||||
# → Genuinely two different real artists sharing a name (e.g. a band's classic
|
||||
# lineup vs. a later solo era with the same stage name). Not a bug — left alone.
|
||||
#
|
||||
# Both sides have files AND overlapping album titles
|
||||
# → The one case actually risky to automate: could mean real content is split
|
||||
# across two entries and needs an actual merge, not a delete. Untouched,
|
||||
# notified for manual review.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Never Delete Real Content
|
||||
# Only the zero-tracked-file side of a pair is ever deleted. Anything with files is
|
||||
# either left alone (disjoint albums) or flagged for a human (overlapping albums) —
|
||||
# never auto-removed.
|
||||
#
|
||||
# Block Re-Addition At The Source
|
||||
# A phantom that keeps coming back is worse than one that was never cleaned —
|
||||
# Import List Exclusion is Lidarr's own mechanism for "never auto-add this again."
|
||||
#
|
||||
# Silent When Clean
|
||||
# No duplicates found, or all duplicates are the simple phantom case — minimal
|
||||
# output. Only ambiguous (overlapping-album) pairs produce a notification.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST1_LIDARR_URL / HOST1_LIDARR_API_KEY — aliased by detect_hosts()
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# lidarr_duplicate_artist_cleanup.sh — normal run
|
||||
# lidarr_duplicate_artist_cleanup.sh --dry-run — preview, no deletions or exclusions
|
||||
# lidarr_duplicate_artist_cleanup.sh --log — verbose per-pair output
|
||||
# lidarr_duplicate_artist_cleanup.sh --status — show config and exit
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
error "jq not found — required for JSON parsing"
|
||||
notify "lidarr_duplicate_artist_cleanup failed on $(hostname) — jq not installed" \
|
||||
"Lidarr Duplicate Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock "wait"
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases LIDARR_URL, LIDARR_API_KEY
|
||||
detect_hosts
|
||||
|
||||
if [[ -z "${LIDARR_URL:-}" ]] || [[ -z "${LIDARR_API_KEY:-}" ]]; then
|
||||
info "Lidarr not configured on $MY_ID ($LOCAL_SERVER_NAME) — skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no artists will be deleted or excluded"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Lidarr URL: $LIDARR_URL"
|
||||
echo "$ICON_GEAR Lidarr ver: v${LIDARR_VERSION_MAJOR:-3} expected"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Checks ━━━
|
||||
# ==============================================================================================
|
||||
check_api "$LIDARR_URL" "Lidarr" 10 || {
|
||||
notify "Lidarr duplicate cleanup aborted on $(hostname) — API unreachable" \
|
||||
"Lidarr Duplicate Cleanup" "warning"
|
||||
exit 1
|
||||
}
|
||||
check_arr_version "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "${LIDARR_VERSION_MAJOR:-3}" "Lidarr" || exit 1
|
||||
|
||||
TMP_DIR=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Fetch Artists (cache-aware — waits out an active rescan rather than trusting a
|
||||
# mid-scan number, falls back to cache if Lidarr's still busy after the strike limit) ━━━
|
||||
# ==============================================================================================
|
||||
ARTISTS=$(lidarr_get_tracked_data "$LIDARR_URL" "$LIDARR_API_KEY" "v1")
|
||||
if [[ -z "$ARTISTS" ]]; then
|
||||
warn "Lidarr busy and no usable cache — deferring to next scheduled run"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
ARTIST_COUNT=$(echo "$ARTISTS" | jq 'length' 2>/dev/null)
|
||||
if [[ -z "$ARTIST_COUNT" || "$ARTIST_COUNT" -eq 0 ]]; then
|
||||
error "API returned 0 artists — aborting to avoid acting on empty data"
|
||||
notify "Lidarr duplicate cleanup aborted on $(hostname) — 0 artists returned" \
|
||||
"Lidarr Duplicate Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
log "$ICON_GEAR Fetched $ARTIST_COUNT artists"
|
||||
|
||||
# Safety Layer — same shared baseline as lidarr_cleanup.sh. A library-wide desync (e.g. mid
|
||||
# full-rescan) can make trackFileCount read far lower than reality for many artists at once —
|
||||
# confirmed 2026-07-16, where this exact scenario would have made the script see both sides
|
||||
# of a genuinely-real duplicate (ROMES) as 0-file phantoms and delete the wrong thing entirely.
|
||||
# Reuses lidarr_cleanup.sh's own baseline file — one shared "is Lidarr's data trustworthy
|
||||
# right now" answer for every script that depends on tracked counts, not a separate opinion
|
||||
# per script.
|
||||
TOTAL_TRACKED=$(echo "$ARTISTS" | jq '[.[].statistics.trackFileCount] | add' 2>/dev/null)
|
||||
check_tracked_count_floor "${TOTAL_TRACKED:-0}" "$LIDARR_TRACKED_COUNT_FILE" "${LIDARR_MIN_TRACKED_PCT:-50}" "Lidarr Duplicate Cleanup"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Find Case-Insensitive Duplicate Names ━━━
|
||||
# ==============================================================================================
|
||||
DUP_NAMES=$(echo "$ARTISTS" | jq -r '.[].artistName' | tr '[:upper:]' '[:lower:]' | sort | uniq -d)
|
||||
|
||||
if [[ -z "$DUP_NAMES" ]]; then
|
||||
echo "Lidarr — clean ✅ no duplicate artist names"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
DUP_COUNT=$(echo "$DUP_NAMES" | grep -c .)
|
||||
warn "Found $DUP_COUNT duplicate artist name(s)"
|
||||
|
||||
DELETED=0
|
||||
EXCLUDED=0
|
||||
LEFT_ALONE=0
|
||||
FLAGGED=0
|
||||
FLAGGED_NAMES=()
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Resolve Each Duplicate ━━━
|
||||
# ==============================================================================================
|
||||
while IFS= read -r lname; do
|
||||
[[ -z "$lname" ]] && continue
|
||||
|
||||
MEMBERS=$(echo "$ARTISTS" | jq -c --arg n "$lname" '[.[] | select((.artistName|ascii_downcase)==$n)]')
|
||||
|
||||
NONZERO_IDS=()
|
||||
ZERO_MEMBERS_FILE="$TMP_DIR/zero_${RANDOM}.jsonl"
|
||||
: > "$ZERO_MEMBERS_FILE"
|
||||
|
||||
while IFS= read -r member; do
|
||||
[[ -z "$member" ]] && continue
|
||||
fc=$(echo "$member" | jq -r '.statistics.trackFileCount // 0')
|
||||
if [[ "$fc" -gt 0 ]]; then
|
||||
NONZERO_IDS+=("$(echo "$member" | jq -r '.id')")
|
||||
else
|
||||
echo "$member" >> "$ZERO_MEMBERS_FILE"
|
||||
fi
|
||||
done < <(echo "$MEMBERS" | jq -c '.[]')
|
||||
|
||||
if [[ "${#NONZERO_IDS[@]}" -le 1 ]]; then
|
||||
# Simple phantom case — delete every zero-file member, keep the real one (if any)
|
||||
while IFS= read -r zmember; do
|
||||
[[ -z "$zmember" ]] && continue
|
||||
zid=$(echo "$zmember" | jq -r '.id')
|
||||
zname=$(echo "$zmember" | jq -r '.artistName')
|
||||
zmbid=$(echo "$zmember" | jq -r '.foreignArtistId')
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn " DRY RUN — would delete phantom: $zname ($zid) and exclude MBID $zmbid"
|
||||
continue
|
||||
fi
|
||||
|
||||
if curl -sf --max-time 15 -X DELETE \
|
||||
"${LIDARR_URL}/api/v1/artist/${zid}?deleteFiles=false" \
|
||||
-H "X-Api-Key: $LIDARR_API_KEY" >/dev/null 2>&1; then
|
||||
(( DELETED++ ))
|
||||
log " $ICON_TRASH Deleted phantom: $zname ($zid)"
|
||||
else
|
||||
warn " Failed to delete phantom: $zname ($zid)"
|
||||
continue
|
||||
fi
|
||||
|
||||
EXCL_PAYLOAD=$(jq -c -n --arg fid "$zmbid" --arg name "$zname" \
|
||||
'{foreignId:$fid, artistName:$name}')
|
||||
if curl -sf --max-time 15 -X POST \
|
||||
"${LIDARR_URL}/api/v1/importlistexclusion" \
|
||||
-H "X-Api-Key: $LIDARR_API_KEY" -H "Content-Type: application/json" \
|
||||
-d "$EXCL_PAYLOAD" >/dev/null 2>&1; then
|
||||
(( EXCLUDED++ ))
|
||||
log " Added to import list exclusions: $zmbid"
|
||||
else
|
||||
warn " Could not add exclusion for $zname ($zmbid) — may already exist"
|
||||
fi
|
||||
done < "$ZERO_MEMBERS_FILE"
|
||||
|
||||
# Nudge the surviving real artist so anything stuck on this ambiguity
|
||||
# resolves now rather than waiting for Lidarr's own next check cycle.
|
||||
if [[ "${#NONZERO_IDS[@]}" -eq 1 && "$DRY_RUN" != true ]]; then
|
||||
curl -sf --max-time 15 -X POST \
|
||||
"${LIDARR_URL}/api/v1/command" \
|
||||
-H "X-Api-Key: $LIDARR_API_KEY" -H "Content-Type: application/json" \
|
||||
-d "{\"name\":\"RefreshArtist\",\"artistId\":${NONZERO_IDS[0]}}" >/dev/null 2>&1
|
||||
fi
|
||||
else
|
||||
# 2+ members have real content — same name, need to know if it's the same
|
||||
# artist actually split (album overlap) or genuinely different acts (no overlap).
|
||||
#
|
||||
# Titles must be deduped WITHIN each artist before checking overlap ACROSS artists —
|
||||
# a single artist can legitimately list the same album title twice (a reissue, a
|
||||
# deluxe edition under an unchanged title). Confirmed 2026-07-16: Alice Cooper's own
|
||||
# catalog has "Lace and Whiskey" and "School's Out" each listed twice under one
|
||||
# artist ID — treating that as "overlap" false-flagged a genuinely disjoint pair
|
||||
# (band-era vs. solo-era) as ambiguous when it wasn't.
|
||||
OVERLAP=false
|
||||
declare -A GLOBAL_TITLES
|
||||
for nid in "${NONZERO_IDS[@]}"; do
|
||||
declare -A THIS_ARTIST_TITLES
|
||||
while IFS= read -r title; do
|
||||
[[ -z "$title" ]] && continue
|
||||
THIS_ARTIST_TITLES["$title"]=1
|
||||
done < <(curl -sf --max-time 15 "${LIDARR_URL}/api/v1/album?artistId=${nid}" \
|
||||
-H "X-Api-Key: $LIDARR_API_KEY" 2>/dev/null | jq -r '.[].title')
|
||||
|
||||
for title in "${!THIS_ARTIST_TITLES[@]}"; do
|
||||
[[ -n "${GLOBAL_TITLES[$title]:-}" ]] && OVERLAP=true
|
||||
GLOBAL_TITLES["$title"]=1
|
||||
done
|
||||
unset THIS_ARTIST_TITLES
|
||||
done
|
||||
unset GLOBAL_TITLES
|
||||
|
||||
if [[ "$OVERLAP" == true ]]; then
|
||||
warn " $ICON_WARN Ambiguous duplicate needs manual review: $lname (both have files, albums overlap)"
|
||||
FLAGGED_NAMES+=("$lname")
|
||||
(( FLAGGED++ ))
|
||||
else
|
||||
log " $lname — different real artists sharing a name, no album overlap, no action"
|
||||
(( LEFT_ALONE++ ))
|
||||
fi
|
||||
fi
|
||||
done <<< "$DUP_NAMES"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY LIDARR DUPLICATE ARTIST CLEANUP SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TRASH Deleted: $DELETED phantom artist(s)"
|
||||
echo "$ICON_GEAR Excluded: $EXCLUDED (blocked from future re-add)"
|
||||
echo "$ICON_SKIP Left alone: $LEFT_ALONE (different real artists, no overlap)"
|
||||
echo "$ICON_WARN Flagged: $FLAGGED (needs manual review)"
|
||||
for n in "${FLAGGED_NAMES[@]}"; do
|
||||
echo " - $n"
|
||||
done
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$FLAGGED" -gt 0 ]]; then
|
||||
notify "Lidarr duplicate cleanup on $(hostname) — $DELETED phantom(s) removed, $FLAGGED artist(s) need manual review: ${FLAGGED_NAMES[*]}" \
|
||||
"Lidarr Duplicate Cleanup" "warning"
|
||||
elif [[ "$DELETED" -gt 0 ]]; then
|
||||
echo "$ICON_DONE Status: cleaned $DELETED phantom artist(s), nothing needs review"
|
||||
else
|
||||
echo "$ICON_DONE Status: no action needed"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
exit 0
|
||||
@@ -255,6 +255,9 @@ echo "━━━ $ICON_SYNC Building Album Directory Map ━━━"
|
||||
|
||||
declare -A ALBUM_DIR_MAP
|
||||
_artist_list=$(curl_json "$LIDARR_URL/api/v1/artist?apikey=$LIDARR_API_KEY")
|
||||
# Write-through — keeps the shared tracked-data cache fresh as a side effect of a fetch
|
||||
# this script already needed for its own purposes. See lidarr_get_tracked_data() in common.sh.
|
||||
lidarr_cache_write "$_artist_list"
|
||||
_map_artist_count=$(echo "$_artist_list" | jq '. | length')
|
||||
info "Fetching track files for $_map_artist_count artists..."
|
||||
|
||||
|
||||
@@ -306,6 +306,9 @@ ALL_ARTISTS=$(lidarr_api "artist") || {
|
||||
error "Failed to fetch artists"
|
||||
exit 1
|
||||
}
|
||||
# Write-through — keeps the shared tracked-data cache fresh as a side effect of a fetch
|
||||
# this script already needed for its own purposes. See lidarr_get_tracked_data() in common.sh.
|
||||
lidarr_cache_write "$ALL_ARTISTS"
|
||||
|
||||
while IFS= read -r artist; do
|
||||
aid=$(echo "$artist" | jq -r '.id')
|
||||
|
||||
@@ -300,6 +300,7 @@
|
||||
"unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning
|
||||
"unRAID_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted
|
||||
"Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers
|
||||
"Arrs_Stack/lidarr_cache_prefill.sh" # warm Lidarr tracked-data cache before anything reads it cold
|
||||
"Arrs_Stack/start_webhook_listener.sh" # arr upgrade webhook listener — continuous
|
||||
"Fallback/fallback.sh" # mutual failover — continuous
|
||||
)
|
||||
@@ -387,6 +388,7 @@
|
||||
"Media/media_cleaner.sh anime" # remove junk from anime shares
|
||||
"Media/media_cleaner.sh media" # remove junk from media shares
|
||||
"Arrs_Stack/lidarr_release_fixer.sh" # fix wrong release editions before cleanup runs
|
||||
"Arrs_Stack/lidarr_duplicate_artist_cleanup.sh" # resolve duplicate MusicBrainz artist entries before cleanup runs
|
||||
"Arrs_Stack/lidarr_cleanup.sh" # remove orphaned music files
|
||||
"Arrs_Stack/sonarr_cleanup.sh" # remove orphaned TV files
|
||||
"Arrs_Stack/radarr_cleanup.sh" # remove orphaned movie files
|
||||
@@ -1089,6 +1091,14 @@
|
||||
# protects against API returning partial data on a bad day
|
||||
LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
|
||||
LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||
|
||||
# Lidarr tracked-data cache — shared by lidarr_cleanup.sh, lidarr_duplicate_artist_cleanup.sh,
|
||||
# lidarr_missing_art.sh, lidarr_release_fixer.sh, and lidarr_cache_prefill.sh. See
|
||||
# lidarr_get_tracked_data() in common.sh for the fresh/stale/rescan-active branching logic.
|
||||
LIDARR_CACHE_FILE="$DATA_DIR/lidarr_tracked_cache.json"
|
||||
LIDARR_RESCAN_DURATION_DB="$DATA_DIR/lidarr_rescan_duration.db"
|
||||
LIDARR_CACHE_MAX_AGE_DAYS=1 # force a live refresh (or rescan-aware wait) past this age
|
||||
LIDARR_PREFILL_WAIT_MINUTES=10 # array-start prefill: how long to retry reaching Lidarr
|
||||
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
|
||||
LIDARR_PROTECTED_PATTERNS=(
|
||||
# Metadata
|
||||
|
||||
@@ -2174,11 +2174,22 @@ arr_api() {
|
||||
# Triggers an arr "command" endpoint with the given JSON payload, then polls every 10s until
|
||||
# completed/failed/timeout, logging progress every 60s. Best-effort pre-flight — never treats
|
||||
# an unreachable/timed-out scan as fatal, matching original per-script behavior.
|
||||
#
|
||||
# On a genuine "completed" observation, records the actual elapsed duration keyed by the
|
||||
# command's own name via lidarr_record_rescan_duration() — this is the one place in the
|
||||
# codebase that reliably watches a command from trigger to completion, so it's the natural
|
||||
# spot to build up real historical duration data for the wait-calibration logic in
|
||||
# lidarr_get_tracked_data(). A timeout doesn't record anything — we only know a lower bound,
|
||||
# not the true duration, and recording that would corrupt future wait calculations downward.
|
||||
#
|
||||
# Usage: trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" \
|
||||
# '{"name": "DownloadedAlbumsScan"}' "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}"
|
||||
trigger_and_await_command() {
|
||||
local base_url="$1" api_key="$2" api_version="$3" payload="$4" poll_timeout="${5:-600}"
|
||||
|
||||
local cmd_name
|
||||
cmd_name=$(echo "$payload" | jq -r '.name // empty' 2>/dev/null)
|
||||
|
||||
local scan_response scan_cmd_id
|
||||
scan_response=$(curl -sf --max-time 30 -X POST \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
@@ -2194,6 +2205,7 @@ trigger_and_await_command() {
|
||||
fi
|
||||
|
||||
info "Import scan queued (command ID: $scan_cmd_id) — waiting for completion..."
|
||||
local start_epoch=$(date +%s)
|
||||
local polled=0 scan_status
|
||||
while [[ "$polled" -lt "$poll_timeout" ]]; do
|
||||
scan_status=$(curl -sf --max-time 10 \
|
||||
@@ -2201,8 +2213,15 @@ trigger_and_await_command() {
|
||||
"${base_url}/api/${api_version}/command/${scan_cmd_id}" 2>/dev/null | \
|
||||
jq -r '.status // empty' 2>/dev/null)
|
||||
case "$scan_status" in
|
||||
completed) info "Import scan complete ✅"; return 0 ;;
|
||||
failed) warn "Import scan reported failed — proceeding anyway"; return 0 ;;
|
||||
completed)
|
||||
info "Import scan complete ✅"
|
||||
[[ -n "$cmd_name" ]] && lidarr_record_rescan_duration "$cmd_name" "$(( $(date +%s) - start_epoch ))"
|
||||
return 0
|
||||
;;
|
||||
failed)
|
||||
warn "Import scan reported failed — proceeding anyway"
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
sleep 10
|
||||
(( polled += 10 ))
|
||||
@@ -2212,6 +2231,150 @@ trigger_and_await_command() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── LIDARR TRACKED-DATA CACHE ─────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Shared cache for Lidarr's tracked-artist/file-count data. Multiple scripts (lidarr_cleanup.sh,
|
||||
# lidarr_duplicate_artist_cleanup.sh, lidarr_missing_art.sh, lidarr_release_fixer.sh) all need a
|
||||
# reasonably-current snapshot of "what does Lidarr think it has tracked." Hitting the live API
|
||||
# fresh every time is wasteful, and during an active rescan the live number is actively
|
||||
# misleading — tracked counts dip and recover as files are detached/re-verified one by one
|
||||
# (confirmed 2026-07-16: a whole-library RescanFolders made trackFileCount read 22% of normal
|
||||
# mid-scan, which is exactly the false-alarm lidarr_cleanup.sh's count-drop guard is meant to
|
||||
# catch, but a genuine rescan isn't the "something's actually wrong" case that guard exists for).
|
||||
#
|
||||
# Write-through: any script that already does a live artist-list fetch for its own purposes
|
||||
# writes the result here as a side effect via lidarr_cache_write() — no dedicated polling timer
|
||||
# needed. Arrs_Stack/lidarr_cache_prefill.sh closes the cold-boot gap by populating the cache
|
||||
# once at array start, before anything else needs it.
|
||||
#
|
||||
# Consumers should call lidarr_get_tracked_data() — never read the cache file directly. It
|
||||
# handles the fresh/stale-no-rescan/stale-rescan-active branching so no script reimplements it.
|
||||
# ==============================================================================================
|
||||
|
||||
LIDARR_CACHE_FILE="${LIDARR_CACHE_FILE:-$DATA_DIR/lidarr_tracked_cache.json}"
|
||||
LIDARR_RESCAN_DURATION_DB="${LIDARR_RESCAN_DURATION_DB:-$DATA_DIR/lidarr_rescan_duration.db}"
|
||||
|
||||
# Writes the current artist list + total tracked count to the shared cache.
|
||||
# Args: artists_json (the full /api/v1/artist response body)
|
||||
lidarr_cache_write() {
|
||||
local artists_json="$1"
|
||||
local total_files
|
||||
total_files=$(echo "$artists_json" | jq '[.[].statistics.trackFileCount] | add' 2>/dev/null)
|
||||
[[ -z "$total_files" || "$total_files" == "null" ]] && return 1
|
||||
|
||||
mkdir -p "$(dirname "$LIDARR_CACHE_FILE")" 2>/dev/null
|
||||
jq -c -n --argjson artists "$artists_json" --argjson total "$total_files" --argjson ts "$(date +%s)" \
|
||||
'{ts:$ts, totalTrackedFiles:$total, artists:$artists}' > "${LIDARR_CACHE_FILE}.tmp" 2>/dev/null \
|
||||
&& mv "${LIDARR_CACHE_FILE}.tmp" "$LIDARR_CACHE_FILE"
|
||||
}
|
||||
|
||||
# Echoes the cached artists JSON array if a cache file exists, regardless of age — staleness
|
||||
# is lidarr_get_tracked_data()'s decision, not this raw reader's.
|
||||
lidarr_cache_read_raw() {
|
||||
[[ -f "$LIDARR_CACHE_FILE" ]] || return 1
|
||||
jq -c '.artists // empty' "$LIDARR_CACHE_FILE" 2>/dev/null
|
||||
}
|
||||
|
||||
# Echoes the cache's age in seconds, or a very large number if no cache exists (so age
|
||||
# comparisons naturally treat "no cache" the same as "very stale cache").
|
||||
lidarr_cache_age_seconds() {
|
||||
[[ -f "$LIDARR_CACHE_FILE" ]] || { echo 999999999; return; }
|
||||
local ts
|
||||
ts=$(jq -r '.ts // 0' "$LIDARR_CACHE_FILE" 2>/dev/null)
|
||||
echo $(( $(date +%s) - ${ts:-0} ))
|
||||
}
|
||||
|
||||
# Records how long a rescan-type command actually took, keyed by command name, so future
|
||||
# waits can be calibrated per command type instead of guessed or blended across very
|
||||
# different operations — a whole-library RescanFolders takes vastly longer than a targeted
|
||||
# DownloadedAlbumsScan, and averaging them would miscalibrate the wait for both.
|
||||
# Args: command_name, duration_seconds
|
||||
lidarr_record_rescan_duration() {
|
||||
local cmd_name="$1" duration="$2"
|
||||
[[ -z "$cmd_name" || -z "$duration" ]] && return 1
|
||||
mkdir -p "$(dirname "$LIDARR_RESCAN_DURATION_DB")" 2>/dev/null
|
||||
local tmp
|
||||
tmp=$(mktemp)
|
||||
[[ -f "$LIDARR_RESCAN_DURATION_DB" ]] && grep -v "^${cmd_name}|" "$LIDARR_RESCAN_DURATION_DB" > "$tmp" 2>/dev/null
|
||||
echo "${cmd_name}|${duration}" >> "$tmp"
|
||||
mv "$tmp" "$LIDARR_RESCAN_DURATION_DB"
|
||||
}
|
||||
|
||||
# Echoes the last recorded duration (seconds) for a given command name, or a fallback
|
||||
# default if none has ever been recorded.
|
||||
# Args: command_name, fallback_default_seconds
|
||||
lidarr_get_rescan_duration() {
|
||||
local cmd_name="$1" fallback="${2:-300}"
|
||||
[[ -f "$LIDARR_RESCAN_DURATION_DB" ]] || { echo "$fallback"; return; }
|
||||
local val
|
||||
val=$(grep "^${cmd_name}|" "$LIDARR_RESCAN_DURATION_DB" 2>/dev/null | tail -1 | cut -d'|' -f2)
|
||||
echo "${val:-$fallback}"
|
||||
}
|
||||
|
||||
# Checks Lidarr's command queue for any currently-active (started or queued) rescan-type
|
||||
# command. Echoes the command name of the first one found (so the caller can look up its
|
||||
# specific historical duration), empty if none active.
|
||||
# Args: url, api_key, api_version
|
||||
lidarr_active_rescan_command() {
|
||||
local url="$1" api_key="$2" api_version="$3"
|
||||
curl -sf --max-time 15 -H "X-Api-Key: $api_key" \
|
||||
"${url}/api/${api_version}/command" 2>/dev/null | \
|
||||
jq -r '[.[] | select(
|
||||
(.status=="started" or .status=="queued") and
|
||||
(.name=="RescanFolders" or .name=="DownloadedAlbumsScan" or .name=="RefreshArtist")
|
||||
)] | .[0].name // empty' 2>/dev/null
|
||||
}
|
||||
|
||||
# Main entry point — the only function consuming scripts should call for tracked artist data.
|
||||
# Handles fresh/stale-no-rescan/stale-rescan-active branching and always writes through on any
|
||||
# live fetch it performs. Echoes the artists JSON array on success, returns 1 if no usable data
|
||||
# (no cache and live fetch impossible) could be obtained.
|
||||
# Args: url, api_key, api_version, max_age_days (default 1), max_wait_strikes (default 3)
|
||||
lidarr_get_tracked_data() {
|
||||
local url="$1" api_key="$2" api_version="$3"
|
||||
local max_age_days="${4:-${LIDARR_CACHE_MAX_AGE_DAYS:-1}}" max_strikes="${5:-3}"
|
||||
local max_age_seconds=$(( max_age_days * 86400 ))
|
||||
local age
|
||||
age=$(lidarr_cache_age_seconds)
|
||||
|
||||
if [[ "$age" -lt "$max_age_seconds" ]]; then
|
||||
lidarr_cache_read_raw && return 0
|
||||
fi
|
||||
|
||||
# Cache missing or stale — check for an active rescan before deciding how to proceed.
|
||||
local active_cmd
|
||||
active_cmd=$(lidarr_active_rescan_command "$url" "$api_key" "$api_version")
|
||||
|
||||
if [[ -n "$active_cmd" ]]; then
|
||||
local wait_duration strike
|
||||
wait_duration=$(( $(lidarr_get_rescan_duration "$active_cmd" 300) / 2 ))
|
||||
[[ "$wait_duration" -lt 30 ]] && wait_duration=30
|
||||
for (( strike=1; strike<=max_strikes; strike++ )); do
|
||||
# Redirected to stderr — this function's stdout is a data channel (callers
|
||||
# capture it via command substitution), never mix log/warn output into it.
|
||||
warn " Lidarr busy ($active_cmd) — waiting ${wait_duration}s (strike ${strike}/${max_strikes})" >&2
|
||||
sleep "$wait_duration"
|
||||
active_cmd=$(lidarr_active_rescan_command "$url" "$api_key" "$api_version")
|
||||
[[ -z "$active_cmd" ]] && break
|
||||
done
|
||||
if [[ -n "$active_cmd" ]]; then
|
||||
warn " Lidarr still busy ($active_cmd) after ${max_strikes} strikes — using cache if present, else skipping" >&2
|
||||
lidarr_cache_read_raw && return 0
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# No active rescan (or it just finished mid-wait) — safe to fetch live and refresh cache.
|
||||
local fresh
|
||||
fresh=$(arr_api "$url" "$api_key" "$api_version" "artist" "Lidarr") || {
|
||||
lidarr_cache_read_raw && return 0
|
||||
return 1
|
||||
}
|
||||
lidarr_cache_write "$fresh"
|
||||
echo "$fresh"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ARR VERSION CHECK ─────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
Reference in New Issue
Block a user