diff --git a/Arrs_Stack/lidarr_cleanup.sh b/Arrs_Stack/lidarr_cleanup.sh old mode 100755 new mode 100644 index 5d43a32..4c81f87 --- a/Arrs_Stack/lidarr_cleanup.sh +++ b/Arrs_Stack/lidarr_cleanup.sh @@ -457,6 +457,30 @@ NOW=$(date +%s) TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt" > "$TO_DELETE_FILE" +# ── Orphan strikes ──────────────────────────────────────────────────────────────────────────── +# Same contract as radarr_cleanup.sh: a file must classify for deletion on +# LIDARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is removed. Covers the partial +# classification failure that is too small to trip the tracked-count floor above. The file is +# rebuilt from each run rather than edited, which is what prunes it. +LIDARR_ORPHAN_STRIKE_LIMIT="${LIDARR_ORPHAN_STRIKE_LIMIT:-2}" +STRIKES_FILE="${LIDARR_ORPHAN_STRIKES_FILE:-$DB_DIR/lidarr_orphan_strikes.tsv}" +mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true +touch "$STRIKES_FILE" 2>/dev/null || true +STRIKES_NEW="$TMP_DIR/strikes_new.tsv" +> "$STRIKES_NEW" +HELD_COUNT=0 +HELD_BYTES=0 + +orphan_strike_ok() { + local path="$1" prev strikes + prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}" + strikes=$(( ${prev:-0} + 1 )) + printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW" + (( strikes >= LIDARR_ORPHAN_STRIKE_LIMIT )) && return 0 + warn " strike $strikes/$LIDARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path" + return 1 +} + while read -r FILE_SIZE FILE_CTIME filepath; do [[ -z "$filepath" ]] && continue FILE_CTIME="${FILE_CTIME%%.*}" @@ -493,12 +517,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do warn "$ICON_TRASH ORPHAN: $filepath" (( ORPHAN_COUNT++ )) ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE )) - echo "$filepath" >> "$TO_DELETE_FILE" + if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi + printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE" else log "JUNK: $filepath" (( JUNK_COUNT++ )) JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE )) - echo "$filepath" >> "$TO_DELETE_FILE" + if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi + printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE" fi # -printf gets size + mtime directly from find's own stat() during the walk, instead of a @@ -506,13 +532,43 @@ while read -r FILE_SIZE FILE_CTIME filepath; do # 4.3ms), since find already has to stat() every entry anyway to know it's -type f. done < <(find "$LIDARR_MUSIC_ROOT" -type f -printf '%s %C@ %p\n' 2>/dev/null) -TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES )) -TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) +# Eligible, not classified: a file still serving its strikes is an orphan but is not queued this +# run, so it must not appear in the denominator the budget reports against. +TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES )) +TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT )) + +# Rebuilt, never edited. Skipped on a dry run: a preview that advanced real counters would make +# the next real run delete a run early. +if [[ "$DRY_RUN" == false ]]; then + mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE" +fi # ============================================================================================== # ━━━ Safety Layer 7 — Deletion Size Threshold ━━━ # ============================================================================================== -check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$LIDARR_MAX_DELETE_GB" "Lidarr Cleanup" +# A per-run budget, not a veto — see apply_delete_budget() in common.sh. The ceiling still caps +# any single run; it just no longer deadlocks on a backlog larger than itself. +BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt" + +if [[ "$I_KNOW" == true ]]; then + warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied" + cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE" + _BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES + _BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK="" +else + apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$LIDARR_MAX_DELETE_GB" + if [[ -n "$_BUDGET_STUCK" ]]; then + error "Single file exceeds the ${LIDARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run" + error " $_BUDGET_STUCK" + error "Raise LIDARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing" + notify "Lidarr cleanup stalled on $(hostname) — one file exceeds the ${LIDARR_MAX_DELETE_GB}GB budget" \ + "Lidarr Cleanup" "warning" + elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then + warn "Budget ${LIDARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run" + notify "Lidarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname) — $_BUDGET_DEFERRED_COUNT file(s) deferred" \ + "Lidarr Cleanup" "normal" + fi +fi # ── Execute Deletions ───────────────────────────────────────────────────────────────────────── # All safety layers passed — delete orphans and junk. Reuses TO_DELETE_FILE from the @@ -521,7 +577,7 @@ if [[ "$DRY_RUN" == false ]]; then while IFS= read -r filepath; do [[ -z "$filepath" ]] && continue rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath" - done < "$TO_DELETE_FILE" + done < "$BUDGET_FILE" info "Cleaning up empty folders..." find "$LIDARR_MUSIC_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null @@ -546,6 +602,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (cover art, metadata echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)" echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)" echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${LIDARR_ORPHAN_AGE} days)" +[[ "${HELD_COUNT:-0}" -gt 0 ]] && \ + echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${LIDARR_ORPHAN_STRIKE_LIMIT} consecutive runs" +[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \ + echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${LIDARR_MAX_DELETE_GB}GB run budget" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" @@ -554,8 +614,10 @@ if [[ "$DRY_RUN" == true ]]; then elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then echo "$ICON_DONE Clean — nothing to remove" else - warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" - notify "Lidarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "warning" + # What was actually removed, not what was classified. With strikes and a budget in force those + # differ, and reporting the classification as the outcome is the oldest bug shape here. + warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files ($(format_bytes "$_BUDGET_KEPT_BYTES"))" + notify "Lidarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "warning" # Notify Emby to clean missing files — removes ghost entries immediately notify_emby_scan fi diff --git a/Arrs_Stack/sonarr_cleanup.sh b/Arrs_Stack/sonarr_cleanup.sh old mode 100755 new mode 100644 index 1569d88..8becac2 --- a/Arrs_Stack/sonarr_cleanup.sh +++ b/Arrs_Stack/sonarr_cleanup.sh @@ -440,6 +440,30 @@ NOW=$(date +%s) TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt" > "$TO_DELETE_FILE" +# ── Orphan strikes ──────────────────────────────────────────────────────────────────────────── +# Same contract as radarr_cleanup.sh: a file must classify for deletion on +# SONARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is removed. Covers the partial +# classification failure that is too small to trip the tracked-count floor above. The file is +# rebuilt from each run rather than edited, which is what prunes it. +SONARR_ORPHAN_STRIKE_LIMIT="${SONARR_ORPHAN_STRIKE_LIMIT:-2}" +STRIKES_FILE="${SONARR_ORPHAN_STRIKES_FILE:-$DB_DIR/sonarr_orphan_strikes.tsv}" +mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true +touch "$STRIKES_FILE" 2>/dev/null || true +STRIKES_NEW="$TMP_DIR/strikes_new.tsv" +> "$STRIKES_NEW" +HELD_COUNT=0 +HELD_BYTES=0 + +orphan_strike_ok() { + local path="$1" prev strikes + prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}" + strikes=$(( ${prev:-0} + 1 )) + printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW" + (( strikes >= SONARR_ORPHAN_STRIKE_LIMIT )) && return 0 + warn " strike $strikes/$SONARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path" + return 1 +} + while read -r FILE_SIZE FILE_CTIME filepath; do [[ -z "$filepath" ]] && continue FILE_CTIME="${FILE_CTIME%%.*}" @@ -474,12 +498,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do warn "$ICON_TRASH ORPHAN: $filepath" (( ORPHAN_COUNT++ )) ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE )) - echo "$filepath" >> "$TO_DELETE_FILE" + if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi + printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE" else log "JUNK: $filepath" (( JUNK_COUNT++ )) JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE )) - echo "$filepath" >> "$TO_DELETE_FILE" + if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi + printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE" fi # -printf gets size + mtime directly from find's own stat() during the walk, instead of a @@ -491,13 +517,43 @@ done < <( done | sort -u ) -TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES )) -TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) +# Eligible, not classified: a file still serving its strikes is an orphan but is not queued this +# run, so it must not appear in the denominator the budget reports against. +TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES )) +TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT )) + +# Rebuilt, never edited. Skipped on a dry run: a preview that advanced real counters would make +# the next real run delete a run early. +if [[ "$DRY_RUN" == false ]]; then + mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE" +fi # ============================================================================================== # ━━━ Safety Layer 7 — Deletion Size Threshold ━━━ # ============================================================================================== -check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$SONARR_MAX_DELETE_GB" "Sonarr Cleanup" +# A per-run budget, not a veto — see apply_delete_budget() in common.sh. The ceiling still caps +# any single run; it just no longer deadlocks on a backlog larger than itself. +BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt" + +if [[ "$I_KNOW" == true ]]; then + warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied" + cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE" + _BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES + _BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK="" +else + apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$SONARR_MAX_DELETE_GB" + if [[ -n "$_BUDGET_STUCK" ]]; then + error "Single file exceeds the ${SONARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run" + error " $_BUDGET_STUCK" + error "Raise SONARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing" + notify "Sonarr cleanup stalled on $(hostname) — one file exceeds the ${SONARR_MAX_DELETE_GB}GB budget" \ + "Sonarr Cleanup" "warning" + elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then + warn "Budget ${SONARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run" + notify "Sonarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname) — $_BUDGET_DEFERRED_COUNT file(s) deferred" \ + "Sonarr Cleanup" "normal" + fi +fi # ── Execute Deletions ───────────────────────────────────────────────────────────────────────── # Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and @@ -506,7 +562,7 @@ if [[ "$DRY_RUN" == false ]]; then while IFS= read -r filepath; do [[ -z "$filepath" ]] && continue rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath" - done < "$TO_DELETE_FILE" + done < "$BUDGET_FILE" info "Cleaning up empty folders..." for host_path in "${SCAN_ROOTS[@]}"; do @@ -532,6 +588,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles, echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)" echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)" echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${SONARR_ORPHAN_AGE} days)" +[[ "${HELD_COUNT:-0}" -gt 0 ]] && \ + echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${SONARR_ORPHAN_STRIKE_LIMIT} consecutive runs" +[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \ + echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${SONARR_MAX_DELETE_GB}GB run budget" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" @@ -540,8 +600,10 @@ if [[ "$DRY_RUN" == true ]]; then elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then echo "$ICON_DONE Clean — nothing to remove" else - warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" - notify "Sonarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \ + # What was actually removed, not what was classified. With strikes and a budget in force those + # differ, and reporting the classification as the outcome is the oldest bug shape here. + warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files ($(format_bytes "$_BUDGET_KEPT_BYTES"))" + notify "Sonarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \ "Sonarr Cleanup" "warning" # Notify Emby to clean missing files — removes ghost entries immediately notify_emby_scan diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 8aef08a..6948a6d 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -1268,6 +1268,8 @@ LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run # protects against API returning partial data on a bad day LIDARR_TRACKED_COUNT_FILE="${DB_DIR}/lidarr_tracked.count" + LIDARR_ORPHAN_STRIKES_FILE="${DB_DIR}/lidarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path + LIDARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify before it is removed LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan # Lidarr tracked-data cache — shared by lidarr_cleanup.sh, lidarr_duplicate_artist_cleanup.sh, @@ -1343,6 +1345,8 @@ SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run # protects against API returning partial data on a bad day SONARR_TRACKED_COUNT_FILE="${DB_DIR}/sonarr_tracked.count" + SONARR_ORPHAN_STRIKES_FILE="${DB_DIR}/sonarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path + SONARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify before it is removed SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan SONARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveSeries command to # reach "completed" — generous because a large series can sit