From 895347f20f87a81748c76253499ead59658faf3d Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sun, 26 Jul 2026 09:55:17 -0400 Subject: [PATCH] =?UTF-8?q?Add=20weekly=20download=20orphan=20cleaner=20?= =?UTF-8?q?=E2=80=94=20nothing=20covered=20the=20SAB=20Completed=20folders?= =?UTF-8?q?=20and=20755G=20of=20orphans=20accumulated=20since=202022?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Arrs_Stack/arr_download_orphan_cleaner.sh | 299 ++++++++++++++++++++++ Deployment/host.conf.template | 4 + Deployment/master.conf.template | 14 + 3 files changed, 317 insertions(+) create mode 100755 Arrs_Stack/arr_download_orphan_cleaner.sh diff --git a/Arrs_Stack/arr_download_orphan_cleaner.sh b/Arrs_Stack/arr_download_orphan_cleaner.sh new file mode 100755 index 0000000..ca4f547 --- /dev/null +++ b/Arrs_Stack/arr_download_orphan_cleaner.sh @@ -0,0 +1,299 @@ +#!/bin/bash +# ============================================================================================== +# ========================== Arr Download Orphan Cleaner ======================================= +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Clears orphaned completed downloads out of the SABnzbd Completed folders that Sonarr and +# Radarr import from. Anything sitting there that the arr's queue no longer references is an +# orphan — the arr will never touch it again on its own, so without this script the folder +# only ever grows. +# +# Built 2026-07-26 after exactly that: 755G of orphaned completed TV downloads (oldest from +# 2022) had silently accumulated and filled the cache pool to 89%. Every existing cleaner +# covers the library side (sonarr_cleanup.sh walks SONARR_TV_ROOT etc.) — nothing covered +# the download side. This is that missing piece, using the same triage that recovered the +# pool that day. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Runs Sonarr then Radarr, sequentially. Per arr, every top-level entry in the configured +# download dir is classified: +# +# TRACKED — basename matches a queue record's outputPath or title → leave it alone, +# the arr still knows about it +# RECENT — mtime under DOWNLOAD_ORPHAN_AGE days → skip, may be mid-import +# JUNK — no video file over DOWNLOAD_ORPHAN_MIN_VIDEO_MB → delete (par2 debris, +# samples, failed/never-extracted archives — the arr can't import these, +# and if the content is still wanted its own missing-search re-grabs it) +# REDUNDANT — parse API matches it AND the library already has every episode / the +# movie file → delete (the arr already refused it as not-an-upgrade) +# IMPORTABLE — parse API matches it but the library is missing episodes / the movie +# → trigger DownloadedEpisodesScan/DownloadedMoviesScan on the folder and +# leave it; whatever imports gets swept as REDUNDANT next week, whatever +# the arr rejects (XEM-blocked, season-span files) stays HELD for a human +# HELD — parse API can't match it (series/movie not in the arr) → report only, +# deciding whether to add the series or bin the files is a human call +# +# The queue fetch is a hard gate: if it fails, the whole arr is skipped — with no queue +# there is no way to tell tracked from orphaned, and guessing means deleting active imports. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# 1. DOWNLOAD_ORPHAN_CLEANER_ENABLED master toggle +# 2. Download dir must exist and live under /mnt/ — refuses to walk anything else +# 3. Queue fetch must succeed (see above) +# 4. Age gate — nothing under DOWNLOAD_ORPHAN_AGE days is touched +# 5. Deletion only for JUNK and parse-verified REDUNDANT — never for HELD/IMPORTABLE +# 6. Run total over DOWNLOAD_ORPHAN_MAX_DELETE_GB aborts the delete pass and notifies — +# a queue fetch that returned partial data would classify live downloads as orphans, +# and an abnormally large delete total is the visible symptom of exactly that +# (--i-know-what-im-doing overrides, e.g. for a first run against a known backlog) +# +# ============================================================================================== +# USAGE +# ============================================================================================== +# +# arr_download_orphan_cleaner.sh — weekly orchestrator entry +# arr_download_orphan_cleaner.sh --dry-run — classify and report only +# arr_download_orphan_cleaner.sh --status — show config and exit +# arr_download_orphan_cleaner.sh --i-know-what-im-doing — bypass MAX_DELETE_GB cap +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../load_config.sh" + +parse_destructive_flags "$@" + +parse_args "${FILTERED_ARGS[@]}" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +for cmd in curl jq; do + if ! command -v "$cmd" >/dev/null 2>&1; then + error "$cmd not found — required" + exit 1 + fi +done + +detect_hosts + +if [[ "${DOWNLOAD_ORPHAN_CLEANER_ENABLED:-false}" != true ]]; then + info "DOWNLOAD_ORPHAN_CLEANER_ENABLED=false — skipping" + exit 0 +fi + +DOWNLOAD_ORPHAN_AGE="${DOWNLOAD_ORPHAN_AGE:-7}" +DOWNLOAD_ORPHAN_MIN_VIDEO_MB="${DOWNLOAD_ORPHAN_MIN_VIDEO_MB:-50}" +DOWNLOAD_ORPHAN_MAX_DELETE_GB="${DOWNLOAD_ORPHAN_MAX_DELETE_GB:-100}" + +if [[ "${SHOW_STATUS:-false}" == true ]]; then + echo "━━━━━ $ICON_SUMMARY DOWNLOAD ORPHAN CLEANER STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_GEAR Enabled: ${DOWNLOAD_ORPHAN_CLEANER_ENABLED}" + echo "$ICON_TIME Age gate: ${DOWNLOAD_ORPHAN_AGE}d" + echo "$ICON_DISK Junk threshold: ${DOWNLOAD_ORPHAN_MIN_VIDEO_MB}M" + echo "$ICON_SHIELD Delete cap: ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G" + for arr in SONARR RADARR; do + dir_var="${MY_ID}_${arr}_DOWNLOAD_DIR" + echo "$ICON_CLEAN ${arr}: ${!dir_var:-}" + done + exit 0 +fi + +acquire_lock "wait" +trap "_release_all_locks" EXIT + +AGE_CUTOFF=$(( $(date +%s) - DOWNLOAD_ORPHAN_AGE * 86400 )) + +TOTAL_DELETED=0 +TOTAL_DELETED_MB=0 +TOTAL_HELD=0 +TOTAL_SCANS=0 + +echo "━━━━━ $ICON_CLEAN DOWNLOAD ORPHAN CLEANER ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +[[ "$DRY_RUN" == true ]] && echo "$ICON_SKIP DRY RUN — nothing will be deleted or imported" + +for arr in sonarr radarr; do + url_var="${arr^^}_URL"; key_var="${arr^^}_API_KEY" + arr_url="${!url_var:-}"; arr_key="${!key_var:-}" + dir_var="${MY_ID}_${arr^^}_DOWNLOAD_DIR" + cdir_var="${MY_ID}_${arr^^}_DOWNLOAD_CONTAINER_DIR" + dl_dir="${!dir_var:-}"; container_dir="${!cdir_var:-}" + + if [[ -z "$arr_url" || -z "$arr_key" || -z "$dl_dir" ]]; then + info "${arr^} download cleanup not configured on $MY_ID — skipping" + continue + fi + if [[ "$dl_dir" != /mnt/* || ! -d "$dl_dir" ]]; then + warn "${arr^} download dir invalid or missing: $dl_dir — skipping" + continue + fi + + echo "" + echo "━━━ $ICON_SYNC ${arr^} — $dl_dir ━━━" + + ver_var="${arr^^}_VERSION_MAJOR" + check_arr_version "$arr_url" "$arr_key" "v3" "${!ver_var}" "${arr^}" || { + warn "${arr^} version check failed — skipping this arr" + continue + } + + if [[ "$arr" == "sonarr" ]]; then + queue_endpoint="queue?pageSize=1000&includeUnknownSeriesItems=true" + exts_var="SONARR_EXTENSIONS" + scan_command="DownloadedEpisodesScan" + else + queue_endpoint="queue?pageSize=1000&includeUnknownMovieItems=true" + exts_var="RADARR_EXTENSIONS" + scan_command="DownloadedMoviesScan" + fi + + QUEUE_JSON=$(arr_api "$arr_url" "$arr_key" "v3" "$queue_endpoint" "${arr^}") || { + error "${arr^} queue fetch failed — cannot tell tracked from orphaned, skipping this arr" + continue + } + + declare -A PROTECTED=() + while IFS= read -r name; do + [[ -n "$name" ]] && PROTECTED["$name"]=1 + done < <(echo "$QUEUE_JSON" | jq -r '.records[] | (.outputPath // empty | split("/") | last), (.title // empty)') + + eval "arr_exts=(\"\${${exts_var}[@]}\")" + + DELETE_PATHS=() + DELETE_SIZES=() + DELETE_LABELS=() + SCAN_PATHS=() + arr_tracked=0; arr_recent=0; arr_held=0; arr_delete_mb=0 + + while IFS= read -r entry; do + base="${entry##*/}" + + if [[ -n "${PROTECTED[$base]:-}" ]]; then + arr_tracked=$((arr_tracked + 1)) + continue + fi + + mtime=$(stat -c %Y "$entry" 2>/dev/null) || continue + if (( mtime > AGE_CUTOFF )); then + arr_recent=$((arr_recent + 1)) + continue + fi + + has_video=false + while IFS= read -r f; do + if has_extension "$f" "${arr_exts[@]}"; then + has_video=true + break + fi + done < <(find "$entry" -type f -size +"${DOWNLOAD_ORPHAN_MIN_VIDEO_MB}"M 2>/dev/null) + + size_mb=$(du -sm "$entry" 2>/dev/null | cut -f1) + size_mb=${size_mb:-0} + + if [[ "$has_video" == false ]]; then + DELETE_PATHS+=("$entry") + DELETE_SIZES+=("$size_mb") + DELETE_LABELS+=("JUNK") + arr_delete_mb=$((arr_delete_mb + size_mb)) + continue + fi + + enc_title=$(jq -rn --arg t "$base" '$t|@uri') + parse=$(arr_api "$arr_url" "$arr_key" "v3" "parse?title=${enc_title}" "${arr^}") || { + warn " parse failed for: $base — holding" + arr_held=$((arr_held + 1)) + continue + } + + if [[ "$arr" == "sonarr" ]]; then + matched=$(echo "$parse" | jq '(.series != null) and ((.episodes | length) > 0)') + missing=$(echo "$parse" | jq '[.episodes[]? | select(.hasFile == false)] | length') + else + # Radarr's parse never populates hasFile — movieFileId is the reliable signal + matched=$(echo "$parse" | jq '.movie != null') + missing=$(echo "$parse" | jq 'if (.movie.movieFileId // 0) > 0 then 0 else 1 end') + fi + + if [[ "$matched" != true ]]; then + echo " $ICON_WARN HELD (no match in ${arr^}): $base (${size_mb}M)" + arr_held=$((arr_held + 1)) + elif (( missing == 0 )); then + DELETE_PATHS+=("$entry") + DELETE_SIZES+=("$size_mb") + DELETE_LABELS+=("REDUNDANT") + arr_delete_mb=$((arr_delete_mb + size_mb)) + else + SCAN_PATHS+=("$base") + arr_held=$((arr_held + 1)) + fi + done < <(find "$dl_dir" -mindepth 1 -maxdepth 1 2>/dev/null) + + if (( arr_delete_mb / 1024 > DOWNLOAD_ORPHAN_MAX_DELETE_GB )) && [[ "$I_KNOW" != true ]]; then + error "${arr^}: delete total $((arr_delete_mb / 1024))G exceeds cap of ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G — aborting delete pass" + notify "${arr^} download orphan delete total $((arr_delete_mb / 1024))G exceeds ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G cap on $(hostname) — possible partial queue data, nothing deleted. Re-run with --i-know-what-im-doing if legitimate." \ + "Download Orphan Cleaner" "warning" + unset PROTECTED + continue + fi + + for i in "${!DELETE_PATHS[@]}"; do + entry="${DELETE_PATHS[$i]}" + if [[ "$DRY_RUN" == true ]]; then + echo " $ICON_SKIP would delete [${DELETE_LABELS[$i]}]: ${entry##*/} (${DELETE_SIZES[$i]}M)" + else + echo " $ICON_TRASH deleting [${DELETE_LABELS[$i]}]: ${entry##*/} (${DELETE_SIZES[$i]}M)" + rm -rf "$entry" + fi + TOTAL_DELETED=$((TOTAL_DELETED + 1)) + TOTAL_DELETED_MB=$((TOTAL_DELETED_MB + DELETE_SIZES[i])) + done + + for base in "${SCAN_PATHS[@]}"; do + if [[ -z "$container_dir" ]]; then + echo " $ICON_WARN IMPORTABLE but ${cdir_var} not set — holding: $base" + continue + fi + if [[ "$DRY_RUN" == true ]]; then + echo " $ICON_SKIP would trigger $scan_command: $base" + else + echo " $ICON_RUN triggering $scan_command: $base" + payload=$(jq -nc --arg n "$scan_command" --arg p "$container_dir/$base" '{name: $n, path: $p, importMode: "Move"}') + curl -sf --max-time 30 -X POST \ + -H "X-Api-Key: $arr_key" -H "Content-Type: application/json" \ + -d "$payload" "$arr_url/api/v3/command" >/dev/null \ + || warn " $scan_command trigger failed for: $base" + TOTAL_SCANS=$((TOTAL_SCANS + 1)) + fi + done + + echo " $ICON_SUMMARY ${arr^}: $arr_tracked tracked, $arr_recent recent, ${#DELETE_PATHS[@]} deleted ($((arr_delete_mb / 1024))G), ${#SCAN_PATHS[@]} import scans, $arr_held held" + TOTAL_HELD=$((TOTAL_HELD + arr_held)) + unset PROTECTED +done + +echo "" +echo "━━━━━ $ICON_DONE SUMMARY ━━━━━" +echo "$ICON_TRASH Deleted: $TOTAL_DELETED ($((TOTAL_DELETED_MB / 1024))G)" +echo "$ICON_RUN Import scans: $TOTAL_SCANS" +echo "$ICON_WARN Held: $TOTAL_HELD" + +if [[ "$DRY_RUN" != true ]] && (( TOTAL_DELETED > 0 || TOTAL_HELD > 0 )); then + notify "Download orphan cleaner on $(hostname): deleted $TOTAL_DELETED orphans ($((TOTAL_DELETED_MB / 1024))G), triggered $TOTAL_SCANS import scans, $TOTAL_HELD held for review" \ + "Download Orphan Cleaner" "normal" +fi diff --git a/Deployment/host.conf.template b/Deployment/host.conf.template index 30d725f..ec05091 100644 --- a/Deployment/host.conf.template +++ b/Deployment/host.conf.template @@ -439,6 +439,8 @@ HOSTN_SONARR_GENERAL_ROOT="" # rootFolderPath literal for the general root (e.g. "/tv") — target for reverse-kids-leak moves; leave blank to disable HOSTN_SONARR_KIDS_ROOT="" # rootFolderPath literal, as reported by Sonarr API — leave blank if no dedicated kids root HOSTN_SONARR_ANIME_ROOT="" # rootFolderPath literal, as reported by Sonarr API — leave blank if no dedicated anime root + HOSTN_SONARR_DOWNLOAD_DIR="" # host path of the completed-downloads folder Sonarr imports from (e.g. "/mnt/cache/Temp_Storage/SABnzbd/Completed/Tv Shows") — blank disables the download orphan cleaner for Sonarr + HOSTN_SONARR_DOWNLOAD_CONTAINER_DIR="" # same folder as Sonarr's container sees it (e.g. "/downloads/Completed/Tv Shows") — needed to trigger import scans on held folders declare -A HOSTN_SONARR_PATH_MAP=( # ["/tv"]="/mnt/user/Tv_Shows" @@ -452,6 +454,8 @@ HOSTN_RADARR_GENERAL_ROOT="" # rootFolderPath literal for the general root (e.g. "/movies") — target for reverse-kids-leak moves; leave blank to disable HOSTN_RADARR_KIDS_ROOT="" # rootFolderPath literal, as reported by Radarr API — leave blank if no dedicated kids root HOSTN_RADARR_ANIME_ROOT="" # rootFolderPath literal, as reported by Radarr API — leave blank if no dedicated anime root + HOSTN_RADARR_DOWNLOAD_DIR="" # host path of the completed-downloads folder Radarr imports from (e.g. "/mnt/cache/Temp_Storage/SABnzbd/Completed/Movies") — blank disables the download orphan cleaner for Radarr + HOSTN_RADARR_DOWNLOAD_CONTAINER_DIR="" # same folder as Radarr's container sees it (e.g. "/downloads/Completed/Movies") — needed to trigger import scans on held folders declare -A HOSTN_RADARR_PATH_MAP=( # ["/movies"]="/mnt/user/Movies" diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 3da552d..9e31b3a 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -439,6 +439,7 @@ "unRAID_Essentials/clear_logs.sh" # purge aged logs — Sunday only, low priority "Arrs_Stack/arr_full_rescan.sh" # full disk↔DB reconciliation for Lidarr/Sonarr/Radarr — keeps tracked stats honest, runs before discovery so it works off fresh data "Arrs_Stack/arr_corruption_scan.sh --remediate" # ffprobe-based corruption sweep of Sonarr's tracked files — deletes+re-searches only after CORRUPTION_SCAN_STRIKE_LIMIT consecutive hits on the same file + "Arrs_Stack/arr_download_orphan_cleaner.sh" # sweep orphaned completed downloads out of the SAB Completed folders — deletes junk + already-imported leftovers, triggers import scans for genuinely-missing content "Arrs_Stack/playback_aware_lidarr_discovery.sh" # behavior-driven music discovery using weekly Emby playback history "Arrs_Stack/playback_aware_radarr_discovery.sh" # behavior-driven movie discovery using TMDB recommendations "Arrs_Stack/playback_aware_sonarr_discovery.sh" # behavior-driven TV discovery using TMDB recommendations @@ -1257,6 +1258,19 @@ RADARR_DROPPED_ADD_EXCLUSION=true # add removed movies to import exclusion list SONARR_DROPPED_ADD_EXCLUSION=true # add removed series to import exclusion list +# ━━━ Download Orphan Cleaner (arr_download_orphan_cleaner.sh) ━━━ +# Weekly sweep of the SABnzbd Completed folders Sonarr/Radarr import from — deletes junk and +# parse-verified already-in-library leftovers the arr queue no longer references, triggers +# import scans for anything the library is actually missing. Built 2026-07-26 after 755G of +# orphaned completed downloads (accumulating since 2022) filled HOST1's cache pool to 89%. +# Per-host dirs: HOST*_SONARR_DOWNLOAD_DIR / HOST*_RADARR_DOWNLOAD_DIR (+ _CONTAINER_DIR). + DOWNLOAD_ORPHAN_CLEANER_ENABLED=true + DOWNLOAD_ORPHAN_AGE=7 # days — entries younger than this may be mid-import, never touched + DOWNLOAD_ORPHAN_MIN_VIDEO_MB=50 # no video file above this = junk (par2 debris, samples, dead archives) + DOWNLOAD_ORPHAN_MAX_DELETE_GB=100 # abort delete pass over this — a partial queue fetch would classify + # live downloads as orphans, and a huge total is that failure's symptom; + # --i-know-what-im-doing overrides for known backlogs + # ━━━ Arr Content Classification (radarr/sonarr_classification_scan.sh) ━━━ # # Curated lists validated against real library data 2026-07-17 — every entry here was