FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Drain the orphan backlog safest-first instead of aborting, so a run over budget still makes progress

Browse Source
This commit is contained in:
Gmer4Lfe
2026-08-22 16:37:31 -04:00
parent 61b7d2a96a
commit 879109e55d
1 changed files with 164 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Arrs_Stack/arr_download_orphan_cleaner.sh
+164 -51
View File
@@ -146,10 +146,21 @@
# (default: 7) # (default: 7)
# #
# DOWNLOAD_ORPHAN_MIN_VIDEO_MB # DOWNLOAD_ORPHAN_MIN_VIDEO_MB
# An entry with no video file above this size is JUNK (default: 50) # An entry with no video file above this size is JUNK (default: 50). Sonarr/Radarr only.
#
# DOWNLOAD_ORPHAN_MIN_AUDIO_MB
# The same test for Lidarr (default: 2). Separate because a 50M floor would mark
# every album folder as JUNK — single tracks rarely reach it.
#
# DOWNLOAD_ORPHAN_KEEP_MARKER
# A file with this name inside a download folder pins it — the folder is never
# classified or deleted (default: .vv-keep). For lossless rips the library holds
# only at lower quality, which REDUNDANT would otherwise sweep.
# #
# DOWNLOAD_ORPHAN_MAX_DELETE_GB # DOWNLOAD_ORPHAN_MAX_DELETE_GB
# Abort the delete pass if the run total exceeds this (default: 100) # Per-run delete budget in GB (default: 100). A backlog above this is drained
# safest-first (JUNK, then REDUNDANT, then UNMATCHED) up to the budget, and the
# remainder is deferred to the next run rather than aborting the pass.
# #
# SONARR_EXTENSIONS / RADARR_EXTENSIONS # SONARR_EXTENSIONS / RADARR_EXTENSIONS
# Video extensions used to decide whether an entry contains real media # Video extensions used to decide whether an entry contains real media
@@ -161,7 +172,7 @@
# arr_download_orphan_cleaner.sh — daily orchestrator entry # arr_download_orphan_cleaner.sh — daily orchestrator entry
# arr_download_orphan_cleaner.sh --dry-run — classify and report only # arr_download_orphan_cleaner.sh --dry-run — classify and report only
# arr_download_orphan_cleaner.sh --status — show config and exit # arr_download_orphan_cleaner.sh --status — show config and exit
# arr_download_orphan_cleaner.sh --i-know-what-im-doing — bypass MAX_DELETE_GB cap # arr_download_orphan_cleaner.sh --i-know-what-im-doing — bypass MAX_DELETE_GB budget
# #
# ============================================================================================== # ==============================================================================================
@@ -197,6 +208,8 @@ fi
DOWNLOAD_ORPHAN_AGE="${DOWNLOAD_ORPHAN_AGE:-7}" DOWNLOAD_ORPHAN_AGE="${DOWNLOAD_ORPHAN_AGE:-7}"
DOWNLOAD_ORPHAN_MIN_VIDEO_MB="${DOWNLOAD_ORPHAN_MIN_VIDEO_MB:-50}" DOWNLOAD_ORPHAN_MIN_VIDEO_MB="${DOWNLOAD_ORPHAN_MIN_VIDEO_MB:-50}"
DOWNLOAD_ORPHAN_MIN_AUDIO_MB="${DOWNLOAD_ORPHAN_MIN_AUDIO_MB:-2}"
DOWNLOAD_ORPHAN_KEEP_MARKER="${DOWNLOAD_ORPHAN_KEEP_MARKER:-.vv-keep}"
DOWNLOAD_ORPHAN_MAX_DELETE_GB="${DOWNLOAD_ORPHAN_MAX_DELETE_GB:-100}" DOWNLOAD_ORPHAN_MAX_DELETE_GB="${DOWNLOAD_ORPHAN_MAX_DELETE_GB:-100}"
if [[ "${SHOW_STATUS:-false}" == true ]]; then if [[ "${SHOW_STATUS:-false}" == true ]]; then
@@ -204,9 +217,10 @@ if [[ "${SHOW_STATUS:-false}" == true ]]; then
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_GEAR Enabled: ${DOWNLOAD_ORPHAN_CLEANER_ENABLED}" echo "$ICON_GEAR Enabled: ${DOWNLOAD_ORPHAN_CLEANER_ENABLED}"
echo "$ICON_TIME Age gate: ${DOWNLOAD_ORPHAN_AGE}d" echo "$ICON_TIME Age gate: ${DOWNLOAD_ORPHAN_AGE}d"
echo "$ICON_DISK Junk threshold: ${DOWNLOAD_ORPHAN_MIN_VIDEO_MB}M" echo "$ICON_DISK Junk threshold: ${DOWNLOAD_ORPHAN_MIN_VIDEO_MB}M video / ${DOWNLOAD_ORPHAN_MIN_AUDIO_MB}M audio"
echo "$ICON_SHIELD Delete cap: ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G" echo "$ICON_SHIELD Delete cap: ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G"
for arr in SONARR RADARR; do echo "$ICON_SHIELD Keep marker: ${DOWNLOAD_ORPHAN_KEEP_MARKER}"
for arr in SONARR RADARR LIDARR; do
dir_var="${MY_ID}_${arr}_DOWNLOAD_DIR" dir_var="${MY_ID}_${arr}_DOWNLOAD_DIR"
echo "$ICON_CLEAN ${arr}: ${!dir_var:-<not configured>}" echo "$ICON_CLEAN ${arr}: ${!dir_var:-<not configured>}"
done done
@@ -221,13 +235,16 @@ AGE_CUTOFF=$(( $(date +%s) - DOWNLOAD_ORPHAN_AGE * 86400 ))
TOTAL_DELETED=0 TOTAL_DELETED=0
TOTAL_DELETED_MB=0 TOTAL_DELETED_MB=0
TOTAL_HELD=0 TOTAL_HELD=0
TOTAL_DEFERRED=0
TOTAL_KEPT=0
TOTAL_SCANS=0 TOTAL_SCANS=0
echo "━━━━━ $ICON_CLEAN DOWNLOAD ORPHAN CLEANER ━━━━━" echo "━━━━━ $ICON_CLEAN DOWNLOAD ORPHAN CLEANER ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
[[ "$DRY_RUN" == true ]] && echo "$ICON_SKIP DRY RUN — nothing will be deleted or imported" [[ "$DRY_RUN" == true ]] && echo "$ICON_SKIP DRY RUN — nothing will be deleted or imported"
for arr in sonarr radarr; do for arr in sonarr radarr lidarr; do
api_ver="v3"; [[ "$arr" == "lidarr" ]] && api_ver="v1"
url_var="${arr^^}_URL"; key_var="${arr^^}_API_KEY" url_var="${arr^^}_URL"; key_var="${arr^^}_API_KEY"
arr_url="${!url_var:-}"; arr_key="${!key_var:-}" arr_url="${!url_var:-}"; arr_key="${!key_var:-}"
dir_var="${MY_ID}_${arr^^}_DOWNLOAD_DIR" dir_var="${MY_ID}_${arr^^}_DOWNLOAD_DIR"
@@ -247,24 +264,38 @@ for arr in sonarr radarr; do
echo "━━━ $ICON_SYNC ${arr^} — $dl_dir ━━━" echo "━━━ $ICON_SYNC ${arr^} — $dl_dir ━━━"
ver_var="${arr^^}_VERSION_MAJOR" ver_var="${arr^^}_VERSION_MAJOR"
check_arr_version "$arr_url" "$arr_key" "v3" "${!ver_var}" "${arr^}" || { check_arr_version "$arr_url" "$arr_key" "$api_ver" "${!ver_var}" "${arr^}" || {
warn "${arr^} version check failed — skipping this arr" warn "${arr^} version check failed — skipping this arr"
continue continue
} }
# min_mb is per-arr because the JUNK test is "contains no real media file". A 50MB floor
# is right for video and catastrophic for audio — most single tracks never reach it, so
# every music folder would classify as JUNK and be deleted regardless of import state.
case "$arr" in
sonarr)
queue_endpoint="queue?pageSize=1000&includeUnknownSeriesItems=true"
exts_var="SONARR_EXTENSIONS"
scan_command="DownloadedEpisodesScan"
library_endpoint="series"
min_mb="$DOWNLOAD_ORPHAN_MIN_VIDEO_MB"
;;
radarr)
queue_endpoint="queue?pageSize=1000&includeUnknownMovieItems=true"
exts_var="RADARR_EXTENSIONS"
scan_command="DownloadedMoviesScan"
library_endpoint="movie"
min_mb="$DOWNLOAD_ORPHAN_MIN_VIDEO_MB"
;;
lidarr)
queue_endpoint="queue?pageSize=1000&includeUnknownArtistItems=true"
exts_var="LIDARR_EXTENSIONS"
scan_command="DownloadedAlbumsScan"
library_endpoint="artist"
min_mb="$DOWNLOAD_ORPHAN_MIN_AUDIO_MB"
;;
esac
if [[ "$arr" == "sonarr" ]]; then QUEUE_JSON=$(arr_api "$arr_url" "$arr_key" "$api_ver" "$queue_endpoint" "${arr^}") || {
queue_endpoint="queue?pageSize=1000&includeUnknownSeriesItems=true"
exts_var="SONARR_EXTENSIONS"
scan_command="DownloadedEpisodesScan"
library_endpoint="series"
else
queue_endpoint="queue?pageSize=1000&includeUnknownMovieItems=true"
exts_var="RADARR_EXTENSIONS"
scan_command="DownloadedMoviesScan"
library_endpoint="movie"
fi
QUEUE_JSON=$(arr_api "$arr_url" "$arr_key" "v3" "$queue_endpoint" "${arr^}") || {
error "${arr^} queue fetch failed — cannot tell tracked from orphaned, skipping this arr" error "${arr^} queue fetch failed — cannot tell tracked from orphaned, skipping this arr"
continue continue
} }
@@ -282,11 +313,21 @@ for arr in sonarr radarr; do
SCAN_PATHS=() SCAN_PATHS=()
UNMATCHED_PATHS=() UNMATCHED_PATHS=()
UNMATCHED_SIZES=() UNMATCHED_SIZES=()
arr_tracked=0; arr_recent=0; arr_held=0; arr_delete_mb=0 arr_tracked=0; arr_recent=0; arr_held=0; arr_delete_mb=0; arr_kept=0
while IFS= read -r entry; do while IFS= read -r entry; do
base="${entry##*/}" base="${entry##*/}"
# An operator keep-marker outranks every verdict below. Needed because REDUNDANT only
# asks "does the library hold this album", not "at what quality" — a lossless rip whose
# library copy is MP3 is redundant by that test and would be swept on the next run.
# The marker is a file inside the folder rather than a conf list so it survives renames
# and cannot drift out of sync with what is actually on disk.
if [[ -e "$entry/$DOWNLOAD_ORPHAN_KEEP_MARKER" ]]; then
arr_kept=$((arr_kept + 1))
continue
fi
if [[ -n "${PROTECTED[$base]:-}" ]]; then if [[ -n "${PROTECTED[$base]:-}" ]]; then
arr_tracked=$((arr_tracked + 1)) arr_tracked=$((arr_tracked + 1))
continue continue
@@ -298,18 +339,31 @@ for arr in sonarr radarr; do
continue continue
fi fi
has_video=false # JUNK means "holds no real media". That verdict is only as good as the extension
# list, and a missing extension turns real content into a delete — 2026-08-21 the
# audio list had no "wv", which classified 23 folders of WavPack lossless (1.5G per
# file) as junk. So a folder with large files that are merely *unrecognised* is held
# for review, never deleted; only a folder with nothing big in it at all is junk.
has_media=false
big_unknown=0
while IFS= read -r f; do while IFS= read -r f; do
if has_extension "$f" "${arr_exts[@]}"; then if has_extension "$f" "${arr_exts[@]}"; then
has_video=true has_media=true
break break
fi fi
done < <(find "$entry" -type f -size +"${DOWNLOAD_ORPHAN_MIN_VIDEO_MB}"M 2>/dev/null) big_unknown=$((big_unknown + 1))
done < <(find "$entry" -type f -size +"${min_mb}"M 2>/dev/null)
size_mb=$(du -sm "$entry" 2>/dev/null | cut -f1) size_mb=$(du -sm "$entry" 2>/dev/null | cut -f1)
size_mb=${size_mb:-0} size_mb=${size_mb:-0}
if [[ "$has_video" == false ]]; then if [[ "$has_media" == false ]] && (( big_unknown > 0 )); then
warn " no recognised media, but $big_unknown large file(s) of unknown type — holding: $base"
arr_held=$((arr_held + 1))
continue
fi
if [[ "$has_media" == false ]]; then
DELETE_PATHS+=("$entry") DELETE_PATHS+=("$entry")
DELETE_SIZES+=("$size_mb") DELETE_SIZES+=("$size_mb")
DELETE_LABELS+=("JUNK") DELETE_LABELS+=("JUNK")
@@ -318,20 +372,43 @@ for arr in sonarr radarr; do
fi fi
enc_title=$(jq -rn --arg t "$base" '$t|@uri') enc_title=$(jq -rn --arg t "$base" '$t|@uri')
parse=$(arr_api "$arr_url" "$arr_key" "v3" "parse?title=${enc_title}" "${arr^}") || { parse=$(arr_api "$arr_url" "$arr_key" "$api_ver" "parse?title=${enc_title}" "${arr^}") || {
warn " parse failed for: $base — holding" warn " parse failed for: $base — holding"
arr_held=$((arr_held + 1)) arr_held=$((arr_held + 1))
continue continue
} }
if [[ "$arr" == "sonarr" ]]; then case "$arr" in
matched=$(echo "$parse" | jq '(.series != null) and ((.episodes | length) > 0)') sonarr)
missing=$(echo "$parse" | jq '[.episodes[]? | select(.hasFile == false)] | length') matched=$(echo "$parse" | jq '(.series != null) and ((.episodes | length) > 0)')
else missing=$(echo "$parse" | jq '[.episodes[]? | select(.hasFile == false)] | length')
# Radarr's parse never populates hasFile — movieFileId is the reliable signal ;;
matched=$(echo "$parse" | jq '.movie != null') radarr)
missing=$(echo "$parse" | jq 'if (.movie.movieFileId // 0) > 0 then 0 else 1 end') # Radarr's parse never populates hasFile — movieFileId is the reliable signal
fi matched=$(echo "$parse" | jq '.movie != null')
missing=$(echo "$parse" | jq 'if (.movie.movieFileId // 0) > 0 then 0 else 1 end')
;;
lidarr)
# Lidarr's parse returns albums with statistics:null, so the track count has
# to be read back from album/{id} — the same shape of gap as Radarr's hasFile.
matched=$(echo "$parse" | jq '(.artist != null) and ((.albums | length) > 0)')
missing=1
if [[ "$matched" == true ]]; then
album_id=$(echo "$parse" | jq -r '.albums[0].id // empty')
if [[ -z "$album_id" ]]; then
warn " parse matched but returned no album id: $base — holding"
arr_held=$((arr_held + 1))
continue
fi
album_json=$(arr_api "$arr_url" "$arr_key" "$api_ver" "album/$album_id" "${arr^}") || {
warn " album lookup failed for: $base — holding"
arr_held=$((arr_held + 1))
continue
}
missing=$(echo "$album_json" | jq 'if ((.statistics.trackFileCount // 0) > 0) then 0 else 1 end')
fi
;;
esac
if [[ "$matched" != true ]]; then if [[ "$matched" != true ]]; then
UNMATCHED_PATHS+=("$entry") UNMATCHED_PATHS+=("$entry")
@@ -354,7 +431,7 @@ for arr in sonarr radarr; do
# fails on a small batch that is legitimately all-unmatched, which is the normal case # fails on a small batch that is legitimately all-unmatched, which is the normal case
# once daily runs have caught up. # once daily runs have caught up.
if (( ${#UNMATCHED_PATHS[@]} > 0 )); then if (( ${#UNMATCHED_PATHS[@]} > 0 )); then
library_count=$(arr_api "$arr_url" "$arr_key" "v3" "$library_endpoint" "${arr^}" | jq 'length' 2>/dev/null) library_count=$(arr_api "$arr_url" "$arr_key" "$api_ver" "$library_endpoint" "${arr^}" | jq 'length' 2>/dev/null)
if [[ ! "$library_count" =~ ^[0-9]+$ ]] || (( library_count == 0 )); then if [[ ! "$library_count" =~ ^[0-9]+$ ]] || (( library_count == 0 )); then
warn " ${arr^}: library reports ${library_count:-no} titles — cannot trust 'no match', holding ${#UNMATCHED_PATHS[@]} unmatched" warn " ${arr^}: library reports ${library_count:-no} titles — cannot trust 'no match', holding ${#UNMATCHED_PATHS[@]} unmatched"
arr_held=$((arr_held + ${#UNMATCHED_PATHS[@]})) arr_held=$((arr_held + ${#UNMATCHED_PATHS[@]}))
@@ -368,26 +445,59 @@ for arr in sonarr radarr; do
fi fi
fi fi
if (( arr_delete_mb / 1024 > DOWNLOAD_ORPHAN_MAX_DELETE_GB )) && [[ "$I_KNOW" != true ]]; then # The cap is a per-run risk budget, not a reason to do nothing. Aborting the whole pass
error "${arr^}: delete total $((arr_delete_mb / 1024))G exceeds cap of ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G — aborting delete pass" # once the backlog exceeds it is self-defeating: the backlog can never shrink below the
notify "${arr^} download orphan delete total $((arr_delete_mb / 1024))G exceeds ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G cap on $(hostname) — possible partial queue data, nothing deleted. Re-run with --i-know-what-im-doing if legitimate." \ # cap on its own, so every later run aborts too and the pool fills anyway (exactly how
# 347G accumulated here by 2026-08-21). Delete in ascending order of risk instead, stop
# at the cap, and defer the rest to the next run so a backlog drains over days.
#
# Live downloads are already protected by DOWNLOAD_ORPHAN_AGE, not by this cap — anything
# in flight is younger than the age gate and never reaches classification. That is what
# makes draining safe: the partial-queue-data case the cap was written for cannot put a
# still-downloading entry in these arrays.
cap_mb=$((DOWNLOAD_ORPHAN_MAX_DELETE_GB * 1024))
cap_active=true
[[ "$I_KNOW" == true || "$DRY_RUN" == true ]] && cap_active=false
arr_deferred=0; arr_deferred_mb=0; arr_run_mb=0
if [[ "$cap_active" == true ]] && (( arr_delete_mb > cap_mb )); then
warn " ${arr^}: $((arr_delete_mb / 1024))G classified vs ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G cap — deleting safest-first up to the cap, deferring the rest"
notify "${arr^} download orphan backlog is $((arr_delete_mb / 1024))G on $(hostname), above the ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G per-run cap. Draining safest-first; the remainder follows on later runs. Re-run with --i-know-what-im-doing to clear it in one pass." \
"Download Orphan Cleaner" "warning" "Download Orphan Cleaner" "warning"
unset PROTECTED
continue
fi fi
for i in "${!DELETE_PATHS[@]}"; do # JUNK first (no media at all), then REDUNDANT (parse-verified already in the library),
entry="${DELETE_PATHS[$i]}" # then UNMATCHED last — it rests on "the arr does not know this title", the weakest of
if [[ "$DRY_RUN" == true ]]; then # the three signals, so it is the first thing the cap defers.
echo " $ICON_SKIP would delete [${DELETE_LABELS[$i]}]: ${entry##*/} (${DELETE_SIZES[$i]}M)" for pass in JUNK REDUNDANT UNMATCHED; do
else for i in "${!DELETE_PATHS[@]}"; do
echo " $ICON_TRASH deleting [${DELETE_LABELS[$i]}]: ${entry##*/} (${DELETE_SIZES[$i]}M)" [[ "${DELETE_LABELS[$i]}" == "$pass" ]] || continue
rm -rf "$entry" entry="${DELETE_PATHS[$i]}"
fi
TOTAL_DELETED=$((TOTAL_DELETED + 1)) if [[ "$cap_active" == true ]] && (( arr_run_mb + DELETE_SIZES[i] > cap_mb )); then
TOTAL_DELETED_MB=$((TOTAL_DELETED_MB + DELETE_SIZES[i])) arr_deferred=$((arr_deferred + 1))
arr_deferred_mb=$((arr_deferred_mb + DELETE_SIZES[i]))
continue
fi
if [[ "$DRY_RUN" == true ]]; then
echo " $ICON_SKIP would delete [${DELETE_LABELS[$i]}]: ${entry##*/} (${DELETE_SIZES[$i]}M)"
else
echo " $ICON_TRASH deleting [${DELETE_LABELS[$i]}]: ${entry##*/} (${DELETE_SIZES[$i]}M)"
rm -rf "$entry"
fi
arr_run_mb=$((arr_run_mb + DELETE_SIZES[i]))
TOTAL_DELETED=$((TOTAL_DELETED + 1))
TOTAL_DELETED_MB=$((TOTAL_DELETED_MB + DELETE_SIZES[i]))
done
done done
if (( arr_deferred > 0 )); then
echo " $ICON_WARN ${arr^}: deferred $arr_deferred entries ($((arr_deferred_mb / 1024))G) to the next run — cap reached"
TOTAL_DEFERRED=$((TOTAL_DEFERRED + arr_deferred))
fi
for base in "${SCAN_PATHS[@]}"; do for base in "${SCAN_PATHS[@]}"; do
if [[ -z "$container_dir" ]]; then if [[ -z "$container_dir" ]]; then
echo " $ICON_WARN IMPORTABLE but ${cdir_var} not set — holding: $base" echo " $ICON_WARN IMPORTABLE but ${cdir_var} not set — holding: $base"
@@ -406,8 +516,9 @@ for arr in sonarr radarr; do
fi fi
done done
echo " $ICON_SUMMARY ${arr^}: $arr_tracked tracked, $arr_recent recent, ${#DELETE_PATHS[@]} deleted ($((arr_delete_mb / 1024))G), ${#SCAN_PATHS[@]} import scans, $arr_held held" echo " $ICON_SUMMARY ${arr^}: $arr_tracked tracked, $arr_kept kept, $arr_recent recent, $((${#DELETE_PATHS[@]} - arr_deferred)) deleted ($((arr_run_mb / 1024))G), $arr_deferred deferred ($((arr_deferred_mb / 1024))G), ${#SCAN_PATHS[@]} import scans, $arr_held held"
TOTAL_HELD=$((TOTAL_HELD + arr_held)) TOTAL_HELD=$((TOTAL_HELD + arr_held))
TOTAL_KEPT=$((TOTAL_KEPT + arr_kept))
unset PROTECTED unset PROTECTED
done done
@@ -416,6 +527,8 @@ echo "━━━━━ $ICON_DONE SUMMARY ━━━━━"
echo "$ICON_TRASH Deleted: $TOTAL_DELETED ($((TOTAL_DELETED_MB / 1024))G)" echo "$ICON_TRASH Deleted: $TOTAL_DELETED ($((TOTAL_DELETED_MB / 1024))G)"
echo "$ICON_RUN Import scans: $TOTAL_SCANS" echo "$ICON_RUN Import scans: $TOTAL_SCANS"
echo "$ICON_WARN Held: $TOTAL_HELD" echo "$ICON_WARN Held: $TOTAL_HELD"
echo "$ICON_SKIP Deferred: $TOTAL_DEFERRED"
echo "$ICON_SHIELD Kept (marker): $TOTAL_KEPT"
# Held alone never notifies — there is always something awaiting review, and on a daily # Held alone never notifies — there is always something awaiting review, and on a daily
# schedule that would be a notification every morning saying nothing happened. # schedule that would be a notification every morning saying nothing happened.