From 0431e720def1a68eea1c63860649f534909cf7de Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Wed, 26 Aug 2026 17:55:21 -0400 Subject: [PATCH] A cap that aborts cannot drain a backlog bigger than itself, so make it a per-run budget and let the queue clear over consecutive nights --- Arrs_Stack/radarr_cleanup.sh | 45 ++++++++++++++++++++++++++++----- common.sh | 48 ++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 6 deletions(-) mode change 100755 => 100644 Arrs_Stack/radarr_cleanup.sh mode change 100755 => 100644 common.sh diff --git a/Arrs_Stack/radarr_cleanup.sh b/Arrs_Stack/radarr_cleanup.sh old mode 100755 new mode 100644 index e8c8494..f9659c2 --- a/Arrs_Stack/radarr_cleanup.sh +++ b/Arrs_Stack/radarr_cleanup.sh @@ -450,6 +450,8 @@ NOW=$(date +%s) # just delete them directly instead of re-walking and re-classifying every SCAN_ROOTS entry a # second time (2026-07-17) — the size-threshold check below needs to know the total before # deleting anything, not before knowing what to delete. +# Carries size and ctime alongside the path now, because the budget pass below has to order by +# age and stop at a byte ceiling — neither of which a bare path list can answer. TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt" > "$TO_DELETE_FILE" @@ -487,12 +489,12 @@ while read -r FILE_SIZE FILE_CTIME filepath; do warn "$ICON_TRASH ORPHAN: $filepath" (( ORPHAN_COUNT++ )) ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE )) - echo "$filepath" >> "$TO_DELETE_FILE" + printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE" else log "JUNK: $filepath" (( JUNK_COUNT++ )) JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE )) - echo "$filepath" >> "$TO_DELETE_FILE" + printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE" fi # -printf gets size + mtime directly from find's own stat() during the walk, instead of a @@ -510,7 +512,34 @@ TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) # ============================================================================================== # ━━━ Safety Layer 7 — Deletion Size Threshold ━━━ # ============================================================================================== -check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$RADARR_MAX_DELETE_GB" "Radarr Cleanup" +# The ceiling is a per-run budget, not a veto. It still means what it always meant — no single run +# removes more than RADARR_MAX_DELETE_GB — but a backlog larger than the ceiling now drains over +# consecutive nights instead of failing the orchestrator forever on a queue it cannot clear. +BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt" + +if [[ "$I_KNOW" == true ]]; then + warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied" + cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE" + _BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES + _BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK="" +else + apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$RADARR_MAX_DELETE_GB" + + if [[ -n "$_BUDGET_STUCK" ]]; then + # One file larger than the whole budget can never fit, so it would be re-found and + # re-deferred every night. Name it rather than loop on it silently. + error "Single file exceeds the ${RADARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run" + error " $_BUDGET_STUCK" + error "Raise RADARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing" + notify "Radarr cleanup stalled on $(hostname) — one file exceeds the ${RADARR_MAX_DELETE_GB}GB budget" \ + "Radarr Cleanup" "warning" + elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then + warn "Budget ${RADARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run" + warn "Oldest first — the deferred files are the newest and are re-evaluated tomorrow" + notify "Radarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname) — $_BUDGET_DEFERRED_COUNT file(s) deferred to the next run" \ + "Radarr Cleanup" "normal" + fi +fi # ── Execute Deletions ───────────────────────────────────────────────────────────────────────── # Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and @@ -519,7 +548,7 @@ if [[ "$DRY_RUN" == false ]]; then while IFS= read -r filepath; do [[ -z "$filepath" ]] && continue rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath" - done < "$TO_DELETE_FILE" + done < "$BUDGET_FILE" info "Cleaning up empty folders..." for host_path in "${SCAN_ROOTS[@]}"; do @@ -545,6 +574,8 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles, echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)" echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)" echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${RADARR_ORPHAN_AGE} days)" +[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \ + echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${RADARR_MAX_DELETE_GB}GB run budget" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" @@ -553,8 +584,10 @@ if [[ "$DRY_RUN" == true ]]; then elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then echo "$ICON_DONE Clean — nothing to remove" else - warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" - notify "Radarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \ + # What was actually removed, not what was classified. With a budget in force those differ, and + # reporting the classification as the outcome is the oldest bug shape in this codebase. + warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED classified files ($(format_bytes "$_BUDGET_KEPT_BYTES"))" + notify "Radarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED classified files ($(format_bytes "$_BUDGET_KEPT_BYTES"))$([[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && echo ", $_BUDGET_DEFERRED_COUNT deferred")" \ "Radarr Cleanup" "warning" # Notify Emby to clean missing files — removes ghost entries immediately notify_emby_scan diff --git a/common.sh b/common.sh old mode 100755 new mode 100644 index 98fb10d..1b4ffbd --- a/common.sh +++ b/common.sh @@ -2734,6 +2734,54 @@ check_delete_size_threshold() { fi } +# Applies a per-run delete budget instead of refusing the whole run. Reads "\t\t" +# from $1, writes the paths that fit inside $3 GB to $2 (oldest ctime first), and reports what was +# held back in _BUDGET_*. +# +# A cap that aborts cannot drain a backlog larger than itself. Every run re-finds the same lump, +# exceeds the same ceiling and stops, so the queue is stuck permanently and only a human with +# --i-know-what-im-doing can clear it. Observed 2026-08-24..26: a partnership merge restored ten +# months of superseded Radarr files in one night, 126GB against a 100GB cap, and the daily +# orchestrator then failed three nights running on a queue it could never drain. Budgeting keeps +# the ceiling meaning exactly what it meant — no single run removes more than max_gb — while the +# backlog clears over consecutive runs with nobody touching it. +# +# Strict oldest-first, stopping at the first file that does not fit rather than skipping it for a +# smaller one further down. Skipping would drain more per run but lets a large file be passed over +# indefinitely; stopping guarantees every deferred file is newer than everything already removed, +# so nothing can starve. +# +# One case is deliberately left for a human: a single file larger than the entire budget can never +# fit, so _BUDGET_STUCK is set and the caller says so rather than looping forever on it. +apply_delete_budget() { + local src="$1" dst="$2" max_gb="$3" + local max_bytes tab + tab=$(printf '\t') + max_bytes=$(awk "BEGIN {printf \"%d\", $max_gb * 1073741824}") + + _BUDGET_KEPT_BYTES=0; _BUDGET_KEPT_COUNT=0 + _BUDGET_DEFERRED_BYTES=0; _BUDGET_DEFERRED_COUNT=0 + _BUDGET_STUCK="" + + : > "$dst" + [[ -s "$src" ]] || return 0 + + local out + out=$(sort -t"$tab" -k2,2n "$src" | awk -F'\t' -v max="$max_bytes" -v dst="$dst" ' + { + if (stop) { defb += $1; defc++; next } + if (used + $1 > max) { stop = 1 + if (kept == 0) stuck = $3 + defb += $1; defc++; next } + used += $1; kept++ + print $3 > dst + } + END { printf "%d\t%d\t%d\t%d\t%s", used+0, kept+0, defb+0, defc+0, stuck } + ') + IFS=$'\t' read -r _BUDGET_KEPT_BYTES _BUDGET_KEPT_COUNT \ + _BUDGET_DEFERRED_BYTES _BUDGET_DEFERRED_COUNT _BUDGET_STUCK <<< "$out" +} + # True if filepath's extension (case-insensitive) matches one of the given extensions. # Usage: has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}" has_extension() {