Compare commits

...
12 Commits
Author SHA1 Message Date
Gmer4Lfe 5fa7750046 Sonarr and Lidarr had the same deadlock and the same missing confirmation as Radarr, so give them the same budget and strikes 2026-08-26 21:20:05 -04:00
Gmer4Lfe 1838eed855 Give AI_ASSIST_CLEANUP the consumer it has never had: it describes the shape of a classification and decides nothing, so switching it off changes no deletion 2026-08-26 18:03:17 -04:00
Gmer4Lfe 96d8a5e3f0 Make a file earn its deletion over consecutive runs, so a partial classification failure too small to trip the tracked-count gate cannot remove anything 2026-08-26 17:58:42 -04:00
Gmer4Lfe 0431e720de A cap that aborts cannot drain a backlog bigger than itself, so make it a per-run budget and let the queue clear over consecutive nights 2026-08-26 17:55:21 -04:00
Gmer4Lfe 7f4921de49 Give the wizard an assistant scoped to the step you are actually stuck on, so a first-run question can say "this" and mean something 2026-08-26 17:31:52 -04:00
Gmer4Lfe d4c19baa32 Folding the Scheduler dock into the shared chat left its scope behind, so the troubleshooter has been diagnosing without the log tail for whatever was open 2026-08-26 17:25:28 -04:00
Gmer4Lfe d444fd8036 A path with an apostrophe broke the remote shell quoting and a big file outran the connect timeout, so intact backups were reported corrupt and absent 2026-08-25 21:30:57 -04:00
Gmer4Lfe d42b1e2dda ffprobe exit 0 with a chapter-track warning is not corruption; guard the delete path against a detector that is wrong at scale 2026-08-25 18:29:43 -04:00
Gmer4Lfe 9492dc4c39 dig names the resolver it could not reach in its error text, so a DNS timeout was scraped as the answer and restarted DDNS over nothing 2026-08-25 18:29:43 -04:00
Gmer4Lfe 172beca3c5 A state key is a file path, not a regex — a release tag like [Bluray-1080p] holds an invalid range, so grep bailed and the tempfile swap wiped every other entry 2026-08-25 18:29:43 -04:00
Gmer4Lfe f0c1289519 Let the file own why the row cap is conditional; the README states that it is 2026-08-25 17:21:20 -04:00
Gmer4Lfe 128172d3a8 The row counts below four columns were the naive division, not what the board does 2026-08-25 17:21:20 -04:00
12 changed files with 675 additions and 59 deletions
+120 -8
View File
@@ -273,6 +273,9 @@ touch "$CORRUPTION_SCAN_STATE_FILE"
CORRUPTION_SCAN_STRIKES_FILE="${CORRUPTION_SCAN_STRIKES_FILE:-$DATA_DIR/corruption_scan_strikes.tsv}"
CORRUPTION_SCAN_STRIKE_LIMIT="${CORRUPTION_SCAN_STRIKE_LIMIT:-2}"
CORRUPTION_SCAN_MAX_CORRUPT_PCT="${CORRUPTION_SCAN_MAX_CORRUPT_PCT:-10}"
CORRUPTION_SCAN_MAX_CONSECUTIVE="${CORRUPTION_SCAN_MAX_CONSECUTIVE:-15}"
CORRUPTION_SCAN_GUARD_MIN_SCANNED="${CORRUPTION_SCAN_GUARD_MIN_SCANNED:-20}"
mkdir -p "$(dirname "$CORRUPTION_SCAN_STRIKES_FILE")"
touch "$CORRUPTION_SCAN_STRIKES_FILE"
@@ -301,6 +304,19 @@ reset_scan_strikes() {
[[ -n "$current" && "$current" != "0" ]] && set_scan_strikes "$1" 0
}
# Bails out of the whole run without committing anything. Safe to call at any point before
# the commit phase: strikes are queued in memory until then, so an abort leaves the strike
# file exactly as the previous run left it and deletes nothing.
abort_scan() {
local why="$1"
error "Corruption scan ABORTED — $why"
error "No strikes recorded and nothing remediated this run — the library was not trusted."
[[ -n "${FRESH_CLEAN_TMP:-}" ]] && rm -f "$FRESH_CLEAN_TMP"
notify "Corruption scan aborted on $(hostname) ($MY_ID) — $why. Nothing deleted." \
"Arr Corruption Scan" "warning"
exit 1
}
# Per-arr API shape differences — everything else in the scan/strike/remediate loop below is
# identical between Sonarr and Radarr.
declare -A ARR_FILE_ENDPOINT=( [sonarr]="episodefile" [radarr]="moviefile" )
@@ -322,6 +338,8 @@ if [[ "$SHOW_STATUS" == true ]]; then
echo "$ICON_GEAR State file: $CORRUPTION_SCAN_STATE_FILE"
echo "$ICON_GEAR Strike limit: $CORRUPTION_SCAN_STRIKE_LIMIT"
echo "$ICON_GEAR Remediate: $REMEDIATE"
echo "$ICON_GEAR Corrupt ceiling: ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% of scanned (min ${CORRUPTION_SCAN_GUARD_MIN_SCANNED} scanned)"
echo "$ICON_GEAR Consecutive trip: $CORRUPTION_SCAN_MAX_CONSECUTIVE"
echo "$ICON_GEAR Scan limit: ${SCAN_LIMIT:-unlimited} (per arr)"
echo "$ICON_GEAR Path filter: ${PATH_FILTER:-none}"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
@@ -357,13 +375,36 @@ ffprobe_translate_path() {
# Probes one file. Echoes "clean" or "corrupt:<reason>". Never trusts a truncated/garbled
# stderr as automatically corrupt — only a real non-empty ffprobe stderr counts.
probe_file() {
local host_path="$1" container_path output
local host_path="$1" container_path output rc
container_path=$(ffprobe_translate_path "$host_path") || { echo "unmapped"; return; }
output=$(docker exec "$FFPROBE_CONTAINER" "$FFPROBE_BIN" -v error "$container_path" 2>&1)
rc=$?
# docker exec writes its own failures to the same stream ffprobe uses, so a stopped
# container or an unreachable daemon is otherwise indistinguishable from a corrupt
# header. A stopped container exits 1 with a daemon message; a missing binary exits
# 127 — neither is evidence about the file, so both must be caught.
if (( rc >= 125 )) \
|| [[ "$output" == "Error response from daemon:"* \
|| "$output" == "Cannot connect to the Docker daemon"* \
|| "$output" == "error during connect:"* ]]; then
echo "probe_error:${output//$'\n'/ }"
return
fi
if [[ -z "$output" ]]; then
echo "clean"
else
elif (( rc != 0 )); then
# ffprobe could not parse the file — EBML header parsing failed, moov atom not found,
# contradictionary STSC and STCO. This is the only class that may be remediated.
echo "corrupt:${output//$'\n'/ }"
else
# Exit 0 with stderr output: a recoverable muxing complaint, most commonly
# "Referenced QT chapter track not found", which many recent .mp4 releases emit and
# which says nothing about playability. Equating any stderr with corruption is what
# produced 103 "corrupt" files on 2026-08-23 — 28 of 43 newly scanned Radarr items.
# Reported for visibility, never strike-tracked, never remediated.
echo "suspect:${output//$'\n'/ }"
fi
}
@@ -442,7 +483,7 @@ TOTAL_SCANNED=0
TOTAL_CORRUPT=0
TOTAL_REMEDIATED=0
TOTAL_REMEDIATE_FAILED=0
declare -A ARR_SCANNED ARR_SKIPPED_CACHED ARR_SKIPPED_UNMAPPED ARR_CORRUPT ARR_STRIKE_HELD ARR_REMEDIATED ARR_REMEDIATE_FAILED
declare -A ARR_SCANNED ARR_SKIPPED_CACHED ARR_SKIPPED_UNMAPPED ARR_CORRUPT ARR_SUSPECT ARR_PROBE_ERRORS ARR_STRIKE_HELD ARR_REMEDIATED ARR_REMEDIATE_FAILED
for arr in sonarr radarr; do
url_var="${arr^^}_URL"; key_var="${arr^^}_API_KEY"
@@ -532,6 +573,12 @@ for arr in sonarr radarr; do
STRIKE_HELD=0
REMEDIATED=0
REMEDIATE_FAILED=0
PROBE_ERRORS=0
SUSPECT_COUNT=0
CONSECUTIVE_BAD=0
QUEUE_PATH=()
QUEUE_STRIKES=()
QUEUE_ITEM=()
FRESH_CLEAN_TMP=$(mktemp)
@@ -566,23 +613,82 @@ for arr in sonarr radarr; do
continue
fi
# A docker-level failure is not evidence about the file. Count it, never queue it.
if [[ "$result" == probe_error:* ]]; then
(( PROBE_ERRORS++ ))
(( CONSECUTIVE_BAD++ ))
warn " ? $host_path — probe failed, NOT counted as corrupt: ${result#probe_error:}"
if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then
abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly"
fi
continue
fi
if [[ "$result" == "clean" ]]; then
CONSECUTIVE_BAD=0
reset_scan_strikes "$host_path"
echo -e "${host_path}\t${stamp}" >> "$FRESH_CLEAN_TMP"
[[ "$ENABLE_LOGGING" == true ]] && echo " $ICON_SUCCESS $host_path"
continue
fi
# corrupt:<reason>
# A successful probe that merely warned. Proves the container is alive, so it clears
# the consecutive-failure tripwire, but it never becomes a strike.
if [[ "$result" == suspect:* ]]; then
CONSECUTIVE_BAD=0
(( SUSPECT_COUNT++ ))
[[ "$ENABLE_LOGGING" == true ]] && warn " ~ $host_path — ffprobe warning (exit 0), NOT corrupt: ${result#suspect:}"
continue
fi
# corrupt:<reason> — queued, NOT committed. Nothing reaches the strike file and nothing
# is deleted until this arr has been fully probed and the guards below have passed. A
# container that dies mid-scan makes every remaining file read as corrupt, and a delete
# cannot be undone — so the destructive half has to wait until the corrupt rate for the
# whole run is known. 2026-08-23: one Jellyfin restart produced 103 false positives.
reason="${result#corrupt:}"
(( CORRUPT_COUNT++ ))
strikes=$(increment_scan_strikes "$host_path")
(( CONSECUTIVE_BAD++ ))
prev_strikes=$(get_scan_strikes "$host_path")
prev_strikes="${prev_strikes//[^0-9]/}"
strikes=$(( ${prev_strikes:-0} + 1 ))
QUEUE_PATH+=("$host_path")
QUEUE_STRIKES+=("$strikes")
QUEUE_ITEM+=("$item")
echo " $ICON_ERROR CORRUPT: $host_path (strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT)"
[[ "$ENABLE_LOGGING" == true ]] && echo " $reason"
if [[ "$REMEDIATE" != true ]]; then
continue
if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then
abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly"
fi
done < <(echo "$ITEMS" | jq -c '.[]')
# ━━━ False-positive guards — run before anything is committed ━━━
if (( CORRUPT_COUNT > 0 )); then
# The pre-flight check only proves the container was up when the scan started.
# Re-check now: a mid-scan death is exactly what this guard exists to catch.
check_container_health "$FFPROBE_CONTAINER" "${DOCKER_TIMEOUT:-30}" "Arr Corruption Scan"
if (( SCANNED >= CORRUPTION_SCAN_GUARD_MIN_SCANNED )); then
corrupt_pct=$(( CORRUPT_COUNT * 100 / SCANNED ))
if (( corrupt_pct >= CORRUPTION_SCAN_MAX_CORRUPT_PCT )); then
abort_scan "$CORRUPT_COUNT of $SCANNED probed files (${corrupt_pct}%) read as corrupt — at or above the ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% ceiling"
fi
fi
fi
# ━━━ Guards passed — commit strikes, then remediate whatever reached the limit ━━━
for _q in "${!QUEUE_PATH[@]}"; do
host_path="${QUEUE_PATH[$_q]}"
strikes="${QUEUE_STRIKES[$_q]}"
item="${QUEUE_ITEM[$_q]}"
set_scan_strikes "$host_path" "$strikes"
[[ "$REMEDIATE" != true ]] && continue
if (( strikes < CORRUPTION_SCAN_STRIKE_LIMIT )); then
warn " $host_path — strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT, not yet remediating (needs repeat confirmation)"
@@ -591,6 +697,8 @@ for arr in sonarr radarr; do
fi
reset_scan_strikes "$host_path"
file_id=$(echo "$item" | jq -r '.file_id')
parent_id=$(echo "$item" | jq -r '.parent_id')
title=$(echo "$item" | jq -r '.title')
http_code=$(curl -sf -o /dev/null -w "%{http_code}" -X DELETE \
@@ -625,7 +733,7 @@ for arr in sonarr radarr; do
warn " $title — deleted and verified, but search trigger returned HTTP $search_code"
(( REMEDIATE_FAILED++ ))
fi
done < <(echo "$ITEMS" | jq -c '.[]')
done
merge_clean_state "$FRESH_CLEAN_TMP"
rm -f "$FRESH_CLEAN_TMP"
@@ -634,6 +742,8 @@ for arr in sonarr radarr; do
ARR_SKIPPED_CACHED[$arr]=$SKIPPED_CACHED
ARR_SKIPPED_UNMAPPED[$arr]=$SKIPPED_UNMAPPED
ARR_CORRUPT[$arr]=$CORRUPT_COUNT
ARR_SUSPECT[$arr]=$SUSPECT_COUNT
ARR_PROBE_ERRORS[$arr]=$PROBE_ERRORS
ARR_STRIKE_HELD[$arr]=$STRIKE_HELD
ARR_REMEDIATED[$arr]=$REMEDIATED
ARR_REMEDIATE_FAILED[$arr]=$REMEDIATE_FAILED
@@ -658,6 +768,8 @@ for arr in sonarr radarr; do
echo " $ICON_SUCCESS Skipped (cached): ${ARR_SKIPPED_CACHED[$arr]}"
echo " $ICON_WARN Skipped (unmapped): ${ARR_SKIPPED_UNMAPPED[$arr]}"
echo " $ICON_ERROR Corrupt found: ${ARR_CORRUPT[$arr]}"
echo " $ICON_WARN Warnings (exit 0): ${ARR_SUSPECT[$arr]} (reported, never remediated)"
echo " $ICON_WARN Probe errors: ${ARR_PROBE_ERRORS[$arr]} (not counted as corrupt)"
if [[ "$REMEDIATE" == true ]]; then
echo " $ICON_WARN Held (strikes): ${ARR_STRIKE_HELD[$arr]}"
echo " $ICON_SUCCESS Remediated: ${ARR_REMEDIATED[$arr]}"
Executable → Regular
+70 -8
View File
@@ -457,6 +457,30 @@ NOW=$(date +%s)
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
> "$TO_DELETE_FILE"
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
# Same contract as radarr_cleanup.sh: a file must classify for deletion on
# LIDARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is removed. Covers the partial
# classification failure that is too small to trip the tracked-count floor above. The file is
# rebuilt from each run rather than edited, which is what prunes it.
LIDARR_ORPHAN_STRIKE_LIMIT="${LIDARR_ORPHAN_STRIKE_LIMIT:-2}"
STRIKES_FILE="${LIDARR_ORPHAN_STRIKES_FILE:-$DB_DIR/lidarr_orphan_strikes.tsv}"
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
touch "$STRIKES_FILE" 2>/dev/null || true
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
> "$STRIKES_NEW"
HELD_COUNT=0
HELD_BYTES=0
orphan_strike_ok() {
local path="$1" prev strikes
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
strikes=$(( ${prev:-0} + 1 ))
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
(( strikes >= LIDARR_ORPHAN_STRIKE_LIMIT )) && return 0
warn " strike $strikes/$LIDARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
return 1
}
while read -r FILE_SIZE FILE_CTIME filepath; do
[[ -z "$filepath" ]] && continue
FILE_CTIME="${FILE_CTIME%%.*}"
@@ -493,12 +517,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
warn "$ICON_TRASH ORPHAN: $filepath"
(( ORPHAN_COUNT++ ))
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
echo "$filepath" >> "$TO_DELETE_FILE"
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
else
log "JUNK: $filepath"
(( JUNK_COUNT++ ))
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
echo "$filepath" >> "$TO_DELETE_FILE"
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
fi
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
@@ -506,13 +532,43 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
# 4.3ms), since find already has to stat() every entry anyway to know it's -type f.
done < <(find "$LIDARR_MUSIC_ROOT" -type f -printf '%s %C@ %p\n' 2>/dev/null)
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
# Eligible, not classified: a file still serving its strikes is an orphan but is not queued this
# run, so it must not appear in the denominator the budget reports against.
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
# Rebuilt, never edited. Skipped on a dry run: a preview that advanced real counters would make
# the next real run delete a run early.
if [[ "$DRY_RUN" == false ]]; then
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
fi
# ==============================================================================================
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
# ==============================================================================================
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$LIDARR_MAX_DELETE_GB" "Lidarr Cleanup"
# A per-run budget, not a veto — see apply_delete_budget() in common.sh. The ceiling still caps
# any single run; it just no longer deadlocks on a backlog larger than itself.
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
if [[ "$I_KNOW" == true ]]; then
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
else
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$LIDARR_MAX_DELETE_GB"
if [[ -n "$_BUDGET_STUCK" ]]; then
error "Single file exceeds the ${LIDARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
error " $_BUDGET_STUCK"
error "Raise LIDARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
notify "Lidarr cleanup stalled on $(hostname) — one file exceeds the ${LIDARR_MAX_DELETE_GB}GB budget" \
"Lidarr Cleanup" "warning"
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
warn "Budget ${LIDARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
notify "Lidarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname)$_BUDGET_DEFERRED_COUNT file(s) deferred" \
"Lidarr Cleanup" "normal"
fi
fi
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
# All safety layers passed — delete orphans and junk. Reuses TO_DELETE_FILE from the
@@ -521,7 +577,7 @@ if [[ "$DRY_RUN" == false ]]; then
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
done < "$TO_DELETE_FILE"
done < "$BUDGET_FILE"
info "Cleaning up empty folders..."
find "$LIDARR_MUSIC_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null
@@ -546,6 +602,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (cover art, metadata
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${LIDARR_ORPHAN_AGE} days)"
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${LIDARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${LIDARR_MAX_DELETE_GB}GB run budget"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
@@ -554,8 +614,10 @@ if [[ "$DRY_RUN" == true ]]; then
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
echo "$ICON_DONE Clean — nothing to remove"
else
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
notify "Lidarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "warning"
# What was actually removed, not what was classified. With strikes and a budget in force those
# differ, and reporting the classification as the outcome is the oldest bug shape here.
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
notify "Lidarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "warning"
# Notify Emby to clean missing files — removes ghost entries immediately
notify_emby_scan
fi
Executable → Regular
+134 -8
View File
@@ -450,9 +450,46 @@ NOW=$(date +%s)
# just delete them directly instead of re-walking and re-classifying every SCAN_ROOTS entry a
# second time (2026-07-17) — the size-threshold check below needs to know the total before
# deleting anything, not before knowing what to delete.
# Carries size and ctime alongside the path now, because the budget pass below has to order by
# age and stop at a byte ceiling — neither of which a bare path list can answer.
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
> "$TO_DELETE_FILE"
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
# A file must classify for deletion on RADARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is
# actually removed. Gate 6 already refuses a run whose tracked count collapsed; this covers the
# partial failure underneath that threshold — one root folder failing to enumerate makes its
# movies look orphaned while the overall percentage still looks fine, and a transient fault will
# not reproduce on the next run.
#
# The file is REBUILT from this run's classifications rather than edited in place, which is what
# prunes it: anything that stopped being an orphan simply is not written again, so a file that
# Radarr re-adopts loses its strikes without needing a reset pass to find it.
#
# Keyed by host path, which is why this could not have worked before 2026-08-26 — wd_state_set
# built a regex from the key, and a release tag like [Bluray-1080p] holds the reversed range 1-0,
# so every write truncated the store to one line. See common.sh.
RADARR_ORPHAN_STRIKE_LIMIT="${RADARR_ORPHAN_STRIKE_LIMIT:-2}"
STRIKES_FILE="${RADARR_ORPHAN_STRIKES_FILE:-$DB_DIR/radarr_orphan_strikes.tsv}"
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
touch "$STRIKES_FILE" 2>/dev/null || true
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
> "$STRIKES_NEW"
HELD_COUNT=0
HELD_BYTES=0
# Records this run's strike for a file and says whether it has served enough of them.
# Returns 0 when the file may be deleted, 1 when it is still accruing.
orphan_strike_ok() {
local path="$1" prev strikes
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
strikes=$(( ${prev:-0} + 1 ))
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
(( strikes >= RADARR_ORPHAN_STRIKE_LIMIT )) && return 0
warn " strike $strikes/$RADARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
return 1
}
while read -r FILE_SIZE FILE_CTIME filepath; do
[[ -z "$filepath" ]] && continue
FILE_CTIME="${FILE_CTIME%%.*}"
@@ -487,12 +524,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
warn "$ICON_TRASH ORPHAN: $filepath"
(( ORPHAN_COUNT++ ))
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
echo "$filepath" >> "$TO_DELETE_FILE"
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
else
log "JUNK: $filepath"
(( JUNK_COUNT++ ))
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
echo "$filepath" >> "$TO_DELETE_FILE"
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
fi
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
@@ -504,13 +543,94 @@ done < <(
done | sort -u
)
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
# Eligible, not classified. A file still serving its strikes was counted as an orphan above — it
# is one — but it is not going to be deleted this run, so it must not appear in the denominator
# the budget reports against or the run claims to have skipped work it never queued.
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
# Rebuilt, never edited: a path absent from this run is absent from the file, so a file Radarr
# re-adopts drops its strikes with no reset pass needed. Skipped on a dry run — a preview that
# advanced real strike counters would make the next real run delete a run early.
if [[ "$DRY_RUN" == false ]]; then
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
fi
# ==============================================================================================
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
# ==============================================================================================
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$RADARR_MAX_DELETE_GB" "Radarr Cleanup"
# The ceiling is a per-run budget, not a veto. It still means what it always meant — no single run
# removes more than RADARR_MAX_DELETE_GB — but a backlog larger than the ceiling now drains over
# consecutive nights instead of failing the orchestrator forever on a queue it cannot clear.
# ── AI note (AI_ASSIST_CLEANUP) ───────────────────────────────────────────────────────────────
# Describes the shape of what was classified. It decides nothing: the eligible set, the budget and
# the strikes are all settled above and none of them read this. Switch AI_ASSIST_CLEANUP off and
# the run removes exactly the same files — the log just loses a paragraph.
#
# ctime clustering is the signal worth surfacing. A normal upgrade cycle dribbles in over weeks; a
# lump sharing one narrow ctime window with mtimes spread across months is a bulk write-back, which
# is what a partnership merge against a partner holding older copies produces. That distinction
# took a person an evening on 2026-08-26 and is the whole reason this note exists.
if [[ "$ORPHAN_COUNT" -gt 0 ]] && [[ -s "$TO_DELETE_FILE" ]]; then
_ai_ev=$(awk -F'\t' '
{ n++; bytes += $1
c = int($2)
if (cmin == 0 || c < cmin) cmin = c
if (c > cmax) cmax = c
bucket[int(c / 21600)]++ }
END {
for (b in bucket) if (bucket[b] > top) { top = bucket[b] }
printf "files=%d bytes_gb=%.1f ctime_span_hours=%.1f largest_6h_ctime_bucket=%d\n",
n, bytes/1073741824, (cmax-cmin)/3600, top
}' "$TO_DELETE_FILE")
_ai_mt=$(cut -d"$(printf '\t')" -f3 "$TO_DELETE_FILE" | head -8 \
| while IFS= read -r p; do [[ -f "$p" ]] && \
printf '%s %s\n' "$(stat -c %y "$p" 2>/dev/null | cut -c1-7)" "$(basename "$p")"; done)
_ai_note=$(ai_assist_note AI_ASSIST_CLEANUP "You are looking at files an automated media-library cleanup has classified for deletion on an Unraid server. They are files on disk that the Radarr database no longer references.
EVIDENCE
$_ai_ev
sample (modification month, then path):
$_ai_mt
A normal quality-upgrade cycle produces orphans whose ctimes are spread out over weeks, because each upgrade happens on its own day. A bulk event - a sync or restore writing files back onto this host - produces orphans sharing one narrow ctime window while their modification times stay spread across months, because the copy preserves modification time but resets ctime.
In no more than three sentences, say which of those two this looks like and name the numbers above that support it. Do not recommend an action. Do not speculate beyond the evidence given.") || _ai_note=""
if [[ -n "$_ai_note" ]]; then
echo ""
echo "━━━ $ICON_GEAR AI note on this classification ━━━"
printf '%s\n' "$_ai_note"
fi
unset _ai_ev _ai_mt
fi
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
if [[ "$I_KNOW" == true ]]; then
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
else
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$RADARR_MAX_DELETE_GB"
if [[ -n "$_BUDGET_STUCK" ]]; then
# One file larger than the whole budget can never fit, so it would be re-found and
# re-deferred every night. Name it rather than loop on it silently.
error "Single file exceeds the ${RADARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
error " $_BUDGET_STUCK"
error "Raise RADARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
notify "Radarr cleanup stalled on $(hostname) — one file exceeds the ${RADARR_MAX_DELETE_GB}GB budget" \
"Radarr Cleanup" "warning"
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
warn "Budget ${RADARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
warn "Oldest first — the deferred files are the newest and are re-evaluated tomorrow"
notify "Radarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname)$_BUDGET_DEFERRED_COUNT file(s) deferred to the next run" \
"Radarr Cleanup" "normal"
fi
fi
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
# Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and
@@ -519,7 +639,7 @@ if [[ "$DRY_RUN" == false ]]; then
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
done < "$TO_DELETE_FILE"
done < "$BUDGET_FILE"
info "Cleaning up empty folders..."
for host_path in "${SCAN_ROOTS[@]}"; do
@@ -545,6 +665,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles,
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${RADARR_ORPHAN_AGE} days)"
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${RADARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${RADARR_MAX_DELETE_GB}GB run budget"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
@@ -553,8 +677,10 @@ if [[ "$DRY_RUN" == true ]]; then
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
echo "$ICON_DONE Clean — nothing to remove"
else
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
notify "Radarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
# What was actually removed, not what was classified. With a budget in force those differ, and
# reporting the classification as the outcome is the oldest bug shape in this codebase.
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED classified files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
notify "Radarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED classified files ($(format_bytes "$_BUDGET_KEPT_BYTES"))$([[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && echo ", $_BUDGET_DEFERRED_COUNT deferred")" \
"Radarr Cleanup" "warning"
# Notify Emby to clean missing files — removes ghost entries immediately
notify_emby_scan
Executable → Regular
+70 -8
View File
@@ -440,6 +440,30 @@ NOW=$(date +%s)
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
> "$TO_DELETE_FILE"
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
# Same contract as radarr_cleanup.sh: a file must classify for deletion on
# SONARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is removed. Covers the partial
# classification failure that is too small to trip the tracked-count floor above. The file is
# rebuilt from each run rather than edited, which is what prunes it.
SONARR_ORPHAN_STRIKE_LIMIT="${SONARR_ORPHAN_STRIKE_LIMIT:-2}"
STRIKES_FILE="${SONARR_ORPHAN_STRIKES_FILE:-$DB_DIR/sonarr_orphan_strikes.tsv}"
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
touch "$STRIKES_FILE" 2>/dev/null || true
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
> "$STRIKES_NEW"
HELD_COUNT=0
HELD_BYTES=0
orphan_strike_ok() {
local path="$1" prev strikes
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
strikes=$(( ${prev:-0} + 1 ))
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
(( strikes >= SONARR_ORPHAN_STRIKE_LIMIT )) && return 0
warn " strike $strikes/$SONARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
return 1
}
while read -r FILE_SIZE FILE_CTIME filepath; do
[[ -z "$filepath" ]] && continue
FILE_CTIME="${FILE_CTIME%%.*}"
@@ -474,12 +498,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
warn "$ICON_TRASH ORPHAN: $filepath"
(( ORPHAN_COUNT++ ))
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
echo "$filepath" >> "$TO_DELETE_FILE"
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
else
log "JUNK: $filepath"
(( JUNK_COUNT++ ))
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
echo "$filepath" >> "$TO_DELETE_FILE"
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
fi
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
@@ -491,13 +517,43 @@ done < <(
done | sort -u
)
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
# Eligible, not classified: a file still serving its strikes is an orphan but is not queued this
# run, so it must not appear in the denominator the budget reports against.
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
# Rebuilt, never edited. Skipped on a dry run: a preview that advanced real counters would make
# the next real run delete a run early.
if [[ "$DRY_RUN" == false ]]; then
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
fi
# ==============================================================================================
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
# ==============================================================================================
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$SONARR_MAX_DELETE_GB" "Sonarr Cleanup"
# A per-run budget, not a veto — see apply_delete_budget() in common.sh. The ceiling still caps
# any single run; it just no longer deadlocks on a backlog larger than itself.
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
if [[ "$I_KNOW" == true ]]; then
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
else
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$SONARR_MAX_DELETE_GB"
if [[ -n "$_BUDGET_STUCK" ]]; then
error "Single file exceeds the ${SONARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
error " $_BUDGET_STUCK"
error "Raise SONARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
notify "Sonarr cleanup stalled on $(hostname) — one file exceeds the ${SONARR_MAX_DELETE_GB}GB budget" \
"Sonarr Cleanup" "warning"
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
warn "Budget ${SONARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
notify "Sonarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname)$_BUDGET_DEFERRED_COUNT file(s) deferred" \
"Sonarr Cleanup" "normal"
fi
fi
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
# Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and
@@ -506,7 +562,7 @@ if [[ "$DRY_RUN" == false ]]; then
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
done < "$TO_DELETE_FILE"
done < "$BUDGET_FILE"
info "Cleaning up empty folders..."
for host_path in "${SCAN_ROOTS[@]}"; do
@@ -532,6 +588,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles,
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${SONARR_ORPHAN_AGE} days)"
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${SONARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${SONARR_MAX_DELETE_GB}GB run budget"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
@@ -540,8 +600,10 @@ if [[ "$DRY_RUN" == true ]]; then
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
echo "$ICON_DONE Clean — nothing to remove"
else
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
notify "Sonarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
# What was actually removed, not what was classified. With strikes and a budget in force those
# differ, and reporting the classification as the outcome is the oldest bug shape here.
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
notify "Sonarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
"Sonarr Cleanup" "warning"
# Notify Emby to clean missing files — removes ghost entries immediately
notify_emby_scan
+27 -1
View File
@@ -1268,6 +1268,8 @@
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day
LIDARR_TRACKED_COUNT_FILE="${DB_DIR}/lidarr_tracked.count"
LIDARR_ORPHAN_STRIKES_FILE="${DB_DIR}/lidarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
LIDARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify before it is removed
LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
# Lidarr tracked-data cache — shared by lidarr_cleanup.sh, lidarr_duplicate_artist_cleanup.sh,
@@ -1343,6 +1345,8 @@
SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day
SONARR_TRACKED_COUNT_FILE="${DB_DIR}/sonarr_tracked.count"
SONARR_ORPHAN_STRIKES_FILE="${DB_DIR}/sonarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
SONARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify before it is removed
SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
SONARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveSeries command to
# reach "completed" — generous because a large series can sit
@@ -1355,6 +1359,15 @@
# against a one-off ffprobe hiccup (mid-write file, NFS blip)
# triggering an unnecessary delete. Resets to 0 the moment a
# file probes clean again.
CORRUPTION_SCAN_MAX_CORRUPT_PCT=10 # abort the run, committing nothing, if this share of
# newly-scanned files reads as corrupt. A healthy library sits
# near zero; a high rate means the detector is wrong, not the
# library. Only counts ffprobe exit != 0.
CORRUPTION_SCAN_MAX_CONSECUTIVE=15 # abort after this many files in a row fail to probe
# cleanly — catches the ffprobe container dying mid-scan,
# which the pre-flight health check cannot see.
CORRUPTION_SCAN_GUARD_MIN_SCANNED=20 # below this many newly-scanned files the percentage
# ceiling is not applied — too small a sample to judge.
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
SONARR_PROTECTED_PATTERNS=(
# Subtitles
@@ -1381,6 +1394,12 @@
RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day
RADARR_TRACKED_COUNT_FILE="${DB_DIR}/radarr_tracked.count"
RADARR_ORPHAN_STRIKES_FILE="${DB_DIR}/radarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
RADARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify for deletion before it is
# removed. Gate 6 already catches an API returning far too few
# tracked files; this catches the partial failure too small to trip
# that percentage — one root folder failing to enumerate makes its
# movies look orphaned, and a transient one will not repeat.
RADARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
RADARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveMovie command to
# reach "completed" — mirrors SONARR_MOVE_POLL_TIMEOUT
@@ -1647,6 +1666,10 @@
# HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES
BACKUP_VERIFY_SAMPLE=10 # random files to check per share
BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample
BACKUP_VERIFY_MD5_TIMEOUT_MAX=600 # ceiling for one remote checksum. The per-file budget
# scales with size (~50MB/s); this caps it. A fixed
# connect-sized timeout killed multi-GB checksums and
# the empty result was then reported as MISSING.
# ━━━ SMART Health ━━━
# Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*.
@@ -2025,7 +2048,10 @@
AI_ASSIST_REPORTS=false # tier 1 — digest / coffee report narration
AI_ASSIST_WATCHDOG=false # tier 2 — file a finding when a watchdog counter passes its limit (needs AI_REPAIR_ENABLED)
AI_ASSIST_DISCOVERY=false # tier 2 — discovery / classification judgement calls
AI_ASSIST_CLEANUP=false # tier 2 — orphan and stuck-import triage
AI_ASSIST_CLEANUP=false # tier 2 — orphan and stuck-import triage. Describes the shape of a
# classification in the log; decides nothing. Off = identical deletions.
AI_ASSIST_TIMEOUT=45 # seconds any single assist may take. An assist that can stall a
# nightly cleanup is not an assist — it is silently skipped past this.
AI_ASSIST_ONBOARD=false # tier 3 — onboarding / settings assistance
# ━━━ AI Repair ━━━
Executable → Regular
+43 -7
View File
@@ -122,6 +122,7 @@ source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
SSH_TIMEOUT=15
BACKUP_VERIFY_MD5_TIMEOUT_MAX="${BACKUP_VERIFY_MD5_TIMEOUT_MAX:-600}"
# ==============================================================================================
# ━━━ Setup ━━━
@@ -228,6 +229,7 @@ TOTAL_CHECKED=0
TOTAL_MATCH=0
TOTAL_MISMATCH=0
TOTAL_MISSING=0
TOTAL_UNVERIFIED=0
SHARES_WITH_ISSUES=()
for share in "${VERIFY_SHARES[@]}"; do
@@ -265,6 +267,7 @@ for share in "${VERIFY_SHARES[@]}"; do
SHARE_MATCH=0
SHARE_MISMATCH=0
SHARE_MISSING=0
SHARE_UNVERIFIED=0
for local_file in "${SAMPLE_FILES[@]}"; do
[[ -z "$local_file" ]] && continue
@@ -276,19 +279,49 @@ for share in "${VERIFY_SHARES[@]}"; do
continue
fi
# Remote checksum via SSH — timeout protected
remote_md5=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
# The path is interpolated into a remote shell command, so it must be escaped for
# reuse as one word. A bare '$local_file' inside single quotes breaks on the first
# apostrophe — "Frieren - Beyond Journey's End" ended the quote early, md5sum fell
# back to reading stdin, and the empty-input hash d41d8cd9... was reported as a
# MISMATCH against a file that is byte-identical on the remote.
printf -v remote_q '%q' "$local_file"
# Existence and content are separate questions. Asking them together means a slow
# checksum is indistinguishable from an absent file.
remote_exists=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" \
-o StrictHostKeyChecking=no \
root@"$REMOTE_SERVER" \
"md5sum '$local_file' 2>/dev/null | awk '{print \$1}'" 2>/dev/null)
"test -f $remote_q && echo yes" 2>/dev/null </dev/null)
(( TOTAL_CHECKED++ ))
if [[ -z "$remote_md5" ]]; then
if [[ "$remote_exists" != "yes" ]]; then
warn "$ICON_ERROR MISSING: $(basename "$local_file")"
(( SHARE_MISSING++ ))
(( TOTAL_MISSING++ ))
continue
fi
# md5sum of a multi-GB file cannot finish inside a connect-sized timeout. Budget by
# size — a 5.9GB file needs ~30s and was being killed at 15s, then counted MISSING
# even though it was present and correct.
local_size=$(stat -c%s "$local_file" 2>/dev/null || echo 0)
md5_timeout=$(( local_size / 52428800 + SSH_TIMEOUT ))
(( md5_timeout > BACKUP_VERIFY_MD5_TIMEOUT_MAX )) && md5_timeout=$BACKUP_VERIFY_MD5_TIMEOUT_MAX
remote_md5=$(timeout "$md5_timeout" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" \
-o StrictHostKeyChecking=no \
root@"$REMOTE_SERVER" \
"md5sum $remote_q 2>/dev/null | awk '{print \$1}'" 2>/dev/null </dev/null)
if [[ -z "$remote_md5" ]]; then
# Present but unreadable within budget. Reporting this as a mismatch or a miss
# would be a claim the run did not earn.
warn "$ICON_WARN UNVERIFIED (checksum timed out after ${md5_timeout}s): $(basename "$local_file")"
(( SHARE_UNVERIFIED++ ))
(( TOTAL_UNVERIFIED++ ))
elif [[ "$local_md5" == "$remote_md5" ]]; then
log "MATCH: $(basename "$local_file")"
(( SHARE_MATCH++ ))
@@ -303,8 +336,8 @@ for share in "${VERIFY_SHARES[@]}"; do
done
# Per-share result — only visible if issues found
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 ]]; then
warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH"
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 || "$SHARE_UNVERIFIED" -gt 0 ]]; then
warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH unverified: $SHARE_UNVERIFIED"
SHARES_WITH_ISSUES+=("$SHARE_NAME")
else
echo "$SHARE_NAME — all $SHARE_MATCH files match ✅"
@@ -325,10 +358,11 @@ echo "$ICON_VERIFY Checked: $TOTAL_CHECKED files"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 || "$TOTAL_UNVERIFIED" -gt 0 ]]; then
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
warn "Missing: $TOTAL_MISSING"
[[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
[[ "$TOTAL_UNVERIFIED" -gt 0 ]] && warn "Unverified: $TOTAL_UNVERIFIED (present, checksum timed out)"
fi
if [[ "$DRY_RUN" == true ]]; then
@@ -337,6 +371,8 @@ elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention: ${SHARES_WITH_ISSUES[*]}"
notify "Backup verify FAILED on $(hostname)$REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" \
"Backup Verify" "warning"
elif [[ "$TOTAL_UNVERIFIED" -gt 0 ]]; then
warn "Status: $TOTAL_MATCH verified, $TOTAL_UNVERIFIED could not be checksummed in time — NOT a clean run"
else
echo "$ICON_DONE Status: all $TOTAL_CHECKED files match across ${#VERIFY_SHARES[@]} shares ✅"
fi
+5 -6
View File
@@ -110,12 +110,11 @@ naming every card on the Monitor grid, its width and its order, and generates th
from it — the column ladder, the span clamps at each width, the row-height cap, and the
compensation when a conditional card is absent. Column counts are 8/4/2/1 and spans are 1/2/4/8,
which is what lets the board re-cut itself at any width with no holes and no hand-placed card.
Breakpoints are arithmetic over `VV_MON_CARD_FLOOR`, never chosen by eye. The row-height cap is
conditional on the same arithmetic: it applies only where a rung fits one screen, because a rung
with more rows than that scrolls the page however its rows are sized, and capping there would clip
every card to buy nothing. `pages/monitor.php` carries the card bodies and nothing about where
they go; moving a card is moving a line in that array. The page cross-checks the declaration
against the cards that actually rendered and says so in the browser if they disagree.
Breakpoints are arithmetic over `VV_MON_CARD_FLOOR`, never chosen by eye, and the row-height cap
is conditional on the same arithmetic it applies only where a rung fits one screen.
`pages/monitor.php` carries the card bodies and nothing about where they go; moving a card is
moving a line in that array. The page cross-checks the declaration against the cards that
actually rendered and says so in the browser if they disagree.
**Caching.** Several endpoints serve from `$VV_CACHE_DIR` (`/tmp/varaverk/api`, tmpfs) rather than hitting live
APIs on every page view, refreshed by `Tools/api_cache_writer.sh`. `?live=1` bypasses the
+9
View File
@@ -1363,6 +1363,15 @@ vv_ai_profiles_script();
profile: profile,
question: q,
history: JSON.stringify(sendable()),
// Where the caller is standing, so a question can say "this setting" and mean it. The
// backend whitelists it and passes it to the worker, which turns it into a location
// line; a page that sets no scope sends '' and the worker omits that line.
//
// Keep this here. The scope-aware dock in 185abdb sent it from the page's own ask call;
// 8eeb4c3 folded that dock into this component and the line did not come with it, so
// for the whole of that window Scheduler computed a scope, re-read it at send time, and
// the model never saw it. Nothing errored — the answer just arrived ungrounded.
scope: (typeof o.scope === 'function' ? o.scope() : (o.scope || '')),
kind: (PROFILES[profile].kind && kindEl) ? kindEl.value : '',
// A checkbox where the page offers one, otherwise whatever the page decides from the
// profile, otherwise on. The Scheduler reasons only when diagnosing: working out what a
+8 -3
View File
@@ -13,9 +13,14 @@
// Powers of two, all the way down.
// Column counts are 8, 4, 2, 1 and every span is 1, 2, 4 or 8. That pairing is what makes
// the board tile with no holes at every width without a single hand-placed card: 32 span
// units divide into 8 columns as 4 rows, into 4 as 8 rows, into 2 as 16, into 1 as 32.
// A rung of 6 or 10 columns does not divide a span of 4, which is why the previous ladder
// needed a per-breakpoint override for every wide card and still left holes.
// units divide into 8 columns as 4 rows and into 4 as 8. A rung of 6 or 10 columns does
// not divide a span of 4, which is why the previous ladder needed a per-breakpoint
// override for every wide card and still left holes.
//
// Below four columns the arithmetic stops being a division. Spans clamp to the column
// count, so a span-4 card contributes 2 at two columns and 1 at one, and the board is 13
// rows and then 18 rather than the 16 and 32 a naive division predicts. It still tiles
// without holes, for a different reason: a clamped card fills its row outright.
//
// Breakpoints are arithmetic, not taste.
// A column count is viable exactly when the cards still fit:
+63
View File
@@ -37,6 +37,12 @@
// Step 2: auto-populate + guide + checklist.
// master.conf pull (for partner servers) lives in the checklist, not here.
// The assistant is a bonus on this page, never a dependency: vv_ai_ui_on() is false on a node
// with AI off or no model reachable, which is the normal state of the fresh install this wizard
// exists to serve. The card simply is not rendered and setup proceeds exactly as before.
require_once dirname(__DIR__) . '/include/ai_chat.php';
if (vv_ai_ui_on()) vv_ai_chat_assets();
$detectedHostname = vv_get_hostname();
// ── Identity already on disk? ────────────────────────────────────────────────────────────────
@@ -313,6 +319,21 @@ hr.vv-hr { border: none; border-top: 1px solid #1e1e1e; margin: 22px 0; }
<div id="vv-onboard-panel"></div>
<div id="vv-done-banner"></div>
<?php if (vv_ai_ui_on()): ?>
<hr class="vv-hr">
<div id="vv-su-ai-card">
<?php vv_ai_chat_markup('vv-su-ai', [
'profile' => 'varaverk',
'compact' => true,
'title' => 'Setup assistant',
'height' => '260px',
'tall' => '460px',
'empty' => 'Stuck on a step? Ask what it wants and why — "what is the API key for", '
. '"why does SSH need the other host", "what does populate actually do".',
]); ?>
</div>
<?php endif; ?>
<div style="margin-top:18px;text-align:right;">
<a href="#" id="vv-exit-link" onclick="vvGoNext(event)"
style="font-size:11px;color:#3a3a3a;text-decoration:none;">Skip →</a>
@@ -443,6 +464,7 @@ function vvShowStep2(redirect, apiKey) {
_vvRedirect = redirect || '?tab=scheduler';
document.getElementById('vv-step1').style.display = 'none';
document.getElementById('vv-step2').style.display = 'block';
vvSetupAiInit(); // card is visible now, so it can measure itself correctly
if (apiKey && apiKey.ok) {
const btn = document.getElementById('vv-key-btn');
const status = document.getElementById('vv-key-status');
@@ -676,6 +698,47 @@ function vvStartPhase2Watch(sinceMs) {
_vvPartnerPoll = setInterval(check, 6000);
}
// ── Setup assistant ───────────────────────────────────────────────────────────
// Scoped to the step the operator is actually stuck on, so "what does this want" resolves without
// them naming it. The worker turns a non-empty scope into a line saying what is open in the
// WebGUI; the varaverk profile holds no scoped_log capability, so a checklist id never triggers a
// log lookup and cannot produce a "log missing" note for something that was never a script.
//
// Re-read at send time, not captured: the checklist re-polls while the page is open and the step
// they are on can change between opening the composer and pressing Ask.
//
// First not-ok, not-deferred item — deferred means "I have decided to skip this", which is not
// where they are stuck. Falls back to the bare page when the list is clean or has not loaded.
function vvSetupScope() {
const d = _vvLastChecklist;
if (!d || !Array.isArray(d.items)) return 'Setup';
const stuck = d.items.find(i => !i.ok && !i.deferred);
return stuck && stuck.id ? 'Setup/' + stuck.id : 'Setup';
}
// Mounted when Step 2 is revealed, not at parse time. This is the only page where the card starts
// inside a display:none block, and a chat that measured itself while hidden would come up wrong
// with nothing to correct it — the component carries no resize observer. Idempotent because
// vvShowStep2 is reachable more than once.
let _vvSetupAiUp = false;
function vvSetupAiInit() {
if (_vvSetupAiUp) return;
if (typeof VvAiChat !== 'function') return;
if (!document.getElementById('vv-su-ai-chat')) return; // AI off — card was never rendered
_vvSetupAiUp = true;
VvAiChat({
prefix: 'vv-su-ai',
profile: 'varaverk',
// Pinned, for the same reason the Partnership card pins it: resuming whatever thread was last
// touched anywhere in the UI could land a first-run operator mid-way through someone else's
// Scheduler conversation, on a card with no picker to get back from.
resumeProfile: 'varaverk',
scope: vvSetupScope,
scopeLabel: 'Setup',
empty: 'Stuck on a step? Ask what it wants and why.',
});
}
function vvLoadChecklist() {
fetch('/plugins/varaverk/api/checklist.php?_=' + Date.now())
.then(r => r.json()).then(d => {
+11 -1
View File
@@ -270,7 +270,17 @@ log "$ICON_SUCCESS Internet reachable"
# ==============================================================================================
if [[ -n "$DDNS_DOMAIN" ]] && [[ -n "$DDNS_CONTAINER" ]]; then
PUBLIC_IP=$(curl -sf --max-time 5 https://ifconfig.me 2>/dev/null | tr -d '[:space:]')
DNS_IP=$(dig +short "$DDNS_DOMAIN" @1.1.1.1 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1)
# dig's failure text names the resolver it could not reach (";; communications error to
# 1.1.1.1#53: timed out"), so scraping its output for an address yields the server, not the
# answer — a guaranteed mismatch that restarts DDNS over what is only a DNS timeout. Trust
# the exit status, and anchor the match so only a bare answer line counts. NXDOMAIN exits 0
# with no output and correctly falls through to the "could not resolve" branch below.
if DNS_ANSWER=$(dig +short "$DDNS_DOMAIN" @1.1.1.1 2>/dev/null); then
DNS_IP=$(printf '%s\n' "$DNS_ANSWER" \
| grep -Eox '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1)
else
DNS_IP=""
fi
if [[ -z "$PUBLIC_IP" ]]; then
warn "Could not determine public IP — skipping DDNS check"
Executable → Regular
+112 -6
View File
@@ -2273,8 +2273,10 @@ acquire_rsync_lock() {
# ==============================================================================================
# Generic get/set for flat "key<sep>value" state files (one entry per line) — the pattern
# every watchdog's strike-tracking and state-file logic was independently reimplementing.
# grep -v + tempfile-swap on write, not sed -i in place — avoids sed treating a key containing
# regex metacharacters (container names, etc.) as part of the substitution pattern.
# Keys are matched as literal prefixes via awk substr(), never as regexes. A key is often a
# media file path, and a release tag like [Bluray-1080p] is a valid-looking bracket expression
# holding the reversed range 1-0 — grep -E rejects it, exits 2, and the tempfile-swap then
# commits an empty file, silently wiping every other entry.
#
# Usage: wd_state_get "$key" "$file" [sep=:]
# wd_state_set "$key" "$value" "$file" [sep=:]
@@ -2284,14 +2286,24 @@ acquire_rsync_lock() {
# keep their own thin same-named wrapper around these rather than changing call sites.
wd_state_get() {
local key="$1" file="$2" sep="${3:-:}"
grep -E "^${key}${sep}" "$file" 2>/dev/null | cut -d"$sep" -f2-
[[ -f "$file" ]] || return 0
awk -v pfx="${key}${sep}" \
'substr($0, 1, length(pfx)) == pfx { print substr($0, length(pfx) + 1); exit }' \
"$file" 2>/dev/null
}
# Returns 1 without touching the state file if the rewrite fails, so a read error costs the
# caller one update rather than the whole file.
wd_state_set() {
local key="$1" value="$2" file="$3" sep="${4:-:}"
grep -vE "^${key}${sep}" "$file" 2>/dev/null > "${file}.tmp"
echo "${key}${sep}${value}" >> "${file}.tmp"
mv "${file}.tmp" "$file"
local tmp="${file}.tmp"
: > "$tmp" || return 1
if [[ -f "$file" ]]; then
awk -v pfx="${key}${sep}" \
'substr($0, 1, length(pfx)) != pfx' "$file" > "$tmp" || { rm -f "$tmp"; return 1; }
fi
echo "${key}${sep}${value}" >> "$tmp"
mv "$tmp" "$file"
}
# ==============================================================================================
@@ -2722,6 +2734,100 @@ check_delete_size_threshold() {
fi
}
# Applies a per-run delete budget instead of refusing the whole run. Reads "<size>\t<ctime>\t<path>"
# from $1, writes the paths that fit inside $3 GB to $2 (oldest ctime first), and reports what was
# held back in _BUDGET_*.
#
# A cap that aborts cannot drain a backlog larger than itself. Every run re-finds the same lump,
# exceeds the same ceiling and stops, so the queue is stuck permanently and only a human with
# --i-know-what-im-doing can clear it. Observed 2026-08-24..26: a partnership merge restored ten
# months of superseded Radarr files in one night, 126GB against a 100GB cap, and the daily
# orchestrator then failed three nights running on a queue it could never drain. Budgeting keeps
# the ceiling meaning exactly what it meant — no single run removes more than max_gb — while the
# backlog clears over consecutive runs with nobody touching it.
#
# Strict oldest-first, stopping at the first file that does not fit rather than skipping it for a
# smaller one further down. Skipping would drain more per run but lets a large file be passed over
# indefinitely; stopping guarantees every deferred file is newer than everything already removed,
# so nothing can starve.
#
# One case is deliberately left for a human: a single file larger than the entire budget can never
# fit, so _BUDGET_STUCK is set and the caller says so rather than looping forever on it.
apply_delete_budget() {
local src="$1" dst="$2" max_gb="$3"
local max_bytes tab
tab=$(printf '\t')
max_bytes=$(awk "BEGIN {printf \"%d\", $max_gb * 1073741824}")
_BUDGET_KEPT_BYTES=0; _BUDGET_KEPT_COUNT=0
_BUDGET_DEFERRED_BYTES=0; _BUDGET_DEFERRED_COUNT=0
_BUDGET_STUCK=""
: > "$dst"
[[ -s "$src" ]] || return 0
local out
out=$(sort -t"$tab" -k2,2n "$src" | awk -F'\t' -v max="$max_bytes" -v dst="$dst" '
{
if (stop) { defb += $1; defc++; next }
if (used + $1 > max) { stop = 1
if (kept == 0) stuck = $3
defb += $1; defc++; next }
used += $1; kept++
print $3 > dst
}
END { printf "%d\t%d\t%d\t%d\t%s", used+0, kept+0, defb+0, defc+0, stuck }
')
IFS=$'\t' read -r _BUDGET_KEPT_BYTES _BUDGET_KEPT_COUNT \
_BUDGET_DEFERRED_BYTES _BUDGET_DEFERRED_COUNT _BUDGET_STUCK <<< "$out"
}
# Asks the local generation model to characterise a block of evidence, for the one line a human
# would otherwise have to derive by hand. Prints the note on stdout and returns 0; returns 1 and
# prints nothing whenever anything at all is missing, off, slow or malformed.
#
# The contract is that failure is indistinguishable from the feature being switched off, because
# every caller must behave identically either way. An assist that can block a nightly cleanup is
# not an assist — so this never retries, never blocks longer than its timeout, and never returns
# a partial answer for a caller to interpret.
#
# Usage: note=$(ai_assist_note AI_ASSIST_CLEANUP "$prompt") && echo "$note"
# $1 name of the AI_ASSIST_* flag governing this caller — passed by name, checked here, so a
# caller cannot accidentally run an assist its own toggle says is off
# $2 the prompt, evidence included
# $3 timeout in seconds (default AI_ASSIST_TIMEOUT, else 45)
ai_assist_note() {
local flag="$1" prompt="$2" ai_timeout="${3:-${AI_ASSIST_TIMEOUT:-45}}"
[[ "${AI_ENABLED:-false}" == "true" ]] || return 1
[[ "${!flag:-false}" == "true" ]] || return 1
[[ -n "${MY_ID:-}" ]] || return 1
command -v jq >/dev/null 2>&1 || return 1
local u_var="${MY_ID}_OLLAMA_URL" m_var="${MY_ID}_OLLAMA_MODEL"
local url="${!u_var:-}" model="${!m_var:-}"
[[ -n "$url" && -n "$model" ]] || return 1
local body out
# temperature 0: this is a description of evidence, and the same evidence should not produce a
# different characterisation on a rerun.
body=$(jq -nc --arg m "$model" --arg p "$prompt" \
'{model:$m, prompt:$p, stream:false, options:{temperature:0}}' 2>/dev/null) || return 1
out=$(timeout "$ai_timeout" curl -sf --max-time "$ai_timeout" \
-H 'Content-Type: application/json' -d "$body" \
"${url%/}/api/generate" 2>/dev/null | jq -r '.response // empty' 2>/dev/null)
# Reasoning models emit a <think> block before the answer. It is not the note — a log line a
# human is meant to skim cannot open with several hundred words of the model talking itself
# through the arithmetic. Strip to the last close tag; a response with no block is unchanged.
[[ "$out" == *"</think>"* ]] && out="${out##*</think>}"
out="${out#"${out%%[![:space:]]*}"}"
[[ -n "$out" ]] || return 1
printf '%s\n' "$out"
}
# True if filepath's extension (case-insensitive) matches one of the given extensions.
# Usage: has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"
has_extension() {