Compare commits
10
Commits
f0c1289519
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5fa7750046 | ||
|
|
1838eed855 | ||
|
|
96d8a5e3f0 | ||
|
|
0431e720de | ||
|
|
7f4921de49 | ||
|
|
d4c19baa32 | ||
|
|
d444fd8036 | ||
|
|
d42b1e2dda | ||
|
|
9492dc4c39 | ||
|
|
172beca3c5 |
Executable → Regular
+120
-8
@@ -273,6 +273,9 @@ touch "$CORRUPTION_SCAN_STATE_FILE"
|
||||
|
||||
CORRUPTION_SCAN_STRIKES_FILE="${CORRUPTION_SCAN_STRIKES_FILE:-$DATA_DIR/corruption_scan_strikes.tsv}"
|
||||
CORRUPTION_SCAN_STRIKE_LIMIT="${CORRUPTION_SCAN_STRIKE_LIMIT:-2}"
|
||||
CORRUPTION_SCAN_MAX_CORRUPT_PCT="${CORRUPTION_SCAN_MAX_CORRUPT_PCT:-10}"
|
||||
CORRUPTION_SCAN_MAX_CONSECUTIVE="${CORRUPTION_SCAN_MAX_CONSECUTIVE:-15}"
|
||||
CORRUPTION_SCAN_GUARD_MIN_SCANNED="${CORRUPTION_SCAN_GUARD_MIN_SCANNED:-20}"
|
||||
mkdir -p "$(dirname "$CORRUPTION_SCAN_STRIKES_FILE")"
|
||||
touch "$CORRUPTION_SCAN_STRIKES_FILE"
|
||||
|
||||
@@ -301,6 +304,19 @@ reset_scan_strikes() {
|
||||
[[ -n "$current" && "$current" != "0" ]] && set_scan_strikes "$1" 0
|
||||
}
|
||||
|
||||
# Bails out of the whole run without committing anything. Safe to call at any point before
|
||||
# the commit phase: strikes are queued in memory until then, so an abort leaves the strike
|
||||
# file exactly as the previous run left it and deletes nothing.
|
||||
abort_scan() {
|
||||
local why="$1"
|
||||
error "Corruption scan ABORTED — $why"
|
||||
error "No strikes recorded and nothing remediated this run — the library was not trusted."
|
||||
[[ -n "${FRESH_CLEAN_TMP:-}" ]] && rm -f "$FRESH_CLEAN_TMP"
|
||||
notify "Corruption scan aborted on $(hostname) ($MY_ID) — $why. Nothing deleted." \
|
||||
"Arr Corruption Scan" "warning"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Per-arr API shape differences — everything else in the scan/strike/remediate loop below is
|
||||
# identical between Sonarr and Radarr.
|
||||
declare -A ARR_FILE_ENDPOINT=( [sonarr]="episodefile" [radarr]="moviefile" )
|
||||
@@ -322,6 +338,8 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo "$ICON_GEAR State file: $CORRUPTION_SCAN_STATE_FILE"
|
||||
echo "$ICON_GEAR Strike limit: $CORRUPTION_SCAN_STRIKE_LIMIT"
|
||||
echo "$ICON_GEAR Remediate: $REMEDIATE"
|
||||
echo "$ICON_GEAR Corrupt ceiling: ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% of scanned (min ${CORRUPTION_SCAN_GUARD_MIN_SCANNED} scanned)"
|
||||
echo "$ICON_GEAR Consecutive trip: $CORRUPTION_SCAN_MAX_CONSECUTIVE"
|
||||
echo "$ICON_GEAR Scan limit: ${SCAN_LIMIT:-unlimited} (per arr)"
|
||||
echo "$ICON_GEAR Path filter: ${PATH_FILTER:-none}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
@@ -357,13 +375,36 @@ ffprobe_translate_path() {
|
||||
# Probes one file. Echoes "clean" or "corrupt:<reason>". Never trusts a truncated/garbled
|
||||
# stderr as automatically corrupt — only a real non-empty ffprobe stderr counts.
|
||||
probe_file() {
|
||||
local host_path="$1" container_path output
|
||||
local host_path="$1" container_path output rc
|
||||
container_path=$(ffprobe_translate_path "$host_path") || { echo "unmapped"; return; }
|
||||
output=$(docker exec "$FFPROBE_CONTAINER" "$FFPROBE_BIN" -v error "$container_path" 2>&1)
|
||||
rc=$?
|
||||
|
||||
# docker exec writes its own failures to the same stream ffprobe uses, so a stopped
|
||||
# container or an unreachable daemon is otherwise indistinguishable from a corrupt
|
||||
# header. A stopped container exits 1 with a daemon message; a missing binary exits
|
||||
# 127 — neither is evidence about the file, so both must be caught.
|
||||
if (( rc >= 125 )) \
|
||||
|| [[ "$output" == "Error response from daemon:"* \
|
||||
|| "$output" == "Cannot connect to the Docker daemon"* \
|
||||
|| "$output" == "error during connect:"* ]]; then
|
||||
echo "probe_error:${output//$'\n'/ }"
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ -z "$output" ]]; then
|
||||
echo "clean"
|
||||
else
|
||||
elif (( rc != 0 )); then
|
||||
# ffprobe could not parse the file — EBML header parsing failed, moov atom not found,
|
||||
# contradictionary STSC and STCO. This is the only class that may be remediated.
|
||||
echo "corrupt:${output//$'\n'/ }"
|
||||
else
|
||||
# Exit 0 with stderr output: a recoverable muxing complaint, most commonly
|
||||
# "Referenced QT chapter track not found", which many recent .mp4 releases emit and
|
||||
# which says nothing about playability. Equating any stderr with corruption is what
|
||||
# produced 103 "corrupt" files on 2026-08-23 — 28 of 43 newly scanned Radarr items.
|
||||
# Reported for visibility, never strike-tracked, never remediated.
|
||||
echo "suspect:${output//$'\n'/ }"
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -442,7 +483,7 @@ TOTAL_SCANNED=0
|
||||
TOTAL_CORRUPT=0
|
||||
TOTAL_REMEDIATED=0
|
||||
TOTAL_REMEDIATE_FAILED=0
|
||||
declare -A ARR_SCANNED ARR_SKIPPED_CACHED ARR_SKIPPED_UNMAPPED ARR_CORRUPT ARR_STRIKE_HELD ARR_REMEDIATED ARR_REMEDIATE_FAILED
|
||||
declare -A ARR_SCANNED ARR_SKIPPED_CACHED ARR_SKIPPED_UNMAPPED ARR_CORRUPT ARR_SUSPECT ARR_PROBE_ERRORS ARR_STRIKE_HELD ARR_REMEDIATED ARR_REMEDIATE_FAILED
|
||||
|
||||
for arr in sonarr radarr; do
|
||||
url_var="${arr^^}_URL"; key_var="${arr^^}_API_KEY"
|
||||
@@ -532,6 +573,12 @@ for arr in sonarr radarr; do
|
||||
STRIKE_HELD=0
|
||||
REMEDIATED=0
|
||||
REMEDIATE_FAILED=0
|
||||
PROBE_ERRORS=0
|
||||
SUSPECT_COUNT=0
|
||||
CONSECUTIVE_BAD=0
|
||||
QUEUE_PATH=()
|
||||
QUEUE_STRIKES=()
|
||||
QUEUE_ITEM=()
|
||||
|
||||
FRESH_CLEAN_TMP=$(mktemp)
|
||||
|
||||
@@ -566,23 +613,82 @@ for arr in sonarr radarr; do
|
||||
continue
|
||||
fi
|
||||
|
||||
# A docker-level failure is not evidence about the file. Count it, never queue it.
|
||||
if [[ "$result" == probe_error:* ]]; then
|
||||
(( PROBE_ERRORS++ ))
|
||||
(( CONSECUTIVE_BAD++ ))
|
||||
warn " ? $host_path — probe failed, NOT counted as corrupt: ${result#probe_error:}"
|
||||
if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then
|
||||
abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$result" == "clean" ]]; then
|
||||
CONSECUTIVE_BAD=0
|
||||
reset_scan_strikes "$host_path"
|
||||
echo -e "${host_path}\t${stamp}" >> "$FRESH_CLEAN_TMP"
|
||||
[[ "$ENABLE_LOGGING" == true ]] && echo " $ICON_SUCCESS $host_path"
|
||||
continue
|
||||
fi
|
||||
|
||||
# corrupt:<reason>
|
||||
# A successful probe that merely warned. Proves the container is alive, so it clears
|
||||
# the consecutive-failure tripwire, but it never becomes a strike.
|
||||
if [[ "$result" == suspect:* ]]; then
|
||||
CONSECUTIVE_BAD=0
|
||||
(( SUSPECT_COUNT++ ))
|
||||
[[ "$ENABLE_LOGGING" == true ]] && warn " ~ $host_path — ffprobe warning (exit 0), NOT corrupt: ${result#suspect:}"
|
||||
continue
|
||||
fi
|
||||
|
||||
# corrupt:<reason> — queued, NOT committed. Nothing reaches the strike file and nothing
|
||||
# is deleted until this arr has been fully probed and the guards below have passed. A
|
||||
# container that dies mid-scan makes every remaining file read as corrupt, and a delete
|
||||
# cannot be undone — so the destructive half has to wait until the corrupt rate for the
|
||||
# whole run is known. 2026-08-23: one Jellyfin restart produced 103 false positives.
|
||||
reason="${result#corrupt:}"
|
||||
(( CORRUPT_COUNT++ ))
|
||||
strikes=$(increment_scan_strikes "$host_path")
|
||||
(( CONSECUTIVE_BAD++ ))
|
||||
|
||||
prev_strikes=$(get_scan_strikes "$host_path")
|
||||
prev_strikes="${prev_strikes//[^0-9]/}"
|
||||
strikes=$(( ${prev_strikes:-0} + 1 ))
|
||||
|
||||
QUEUE_PATH+=("$host_path")
|
||||
QUEUE_STRIKES+=("$strikes")
|
||||
QUEUE_ITEM+=("$item")
|
||||
|
||||
echo " $ICON_ERROR CORRUPT: $host_path (strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT)"
|
||||
[[ "$ENABLE_LOGGING" == true ]] && echo " $reason"
|
||||
|
||||
if [[ "$REMEDIATE" != true ]]; then
|
||||
continue
|
||||
if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then
|
||||
abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly"
|
||||
fi
|
||||
done < <(echo "$ITEMS" | jq -c '.[]')
|
||||
|
||||
# ━━━ False-positive guards — run before anything is committed ━━━
|
||||
if (( CORRUPT_COUNT > 0 )); then
|
||||
# The pre-flight check only proves the container was up when the scan started.
|
||||
# Re-check now: a mid-scan death is exactly what this guard exists to catch.
|
||||
check_container_health "$FFPROBE_CONTAINER" "${DOCKER_TIMEOUT:-30}" "Arr Corruption Scan"
|
||||
|
||||
if (( SCANNED >= CORRUPTION_SCAN_GUARD_MIN_SCANNED )); then
|
||||
corrupt_pct=$(( CORRUPT_COUNT * 100 / SCANNED ))
|
||||
if (( corrupt_pct >= CORRUPTION_SCAN_MAX_CORRUPT_PCT )); then
|
||||
abort_scan "$CORRUPT_COUNT of $SCANNED probed files (${corrupt_pct}%) read as corrupt — at or above the ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% ceiling"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ━━━ Guards passed — commit strikes, then remediate whatever reached the limit ━━━
|
||||
for _q in "${!QUEUE_PATH[@]}"; do
|
||||
host_path="${QUEUE_PATH[$_q]}"
|
||||
strikes="${QUEUE_STRIKES[$_q]}"
|
||||
item="${QUEUE_ITEM[$_q]}"
|
||||
|
||||
set_scan_strikes "$host_path" "$strikes"
|
||||
|
||||
[[ "$REMEDIATE" != true ]] && continue
|
||||
|
||||
if (( strikes < CORRUPTION_SCAN_STRIKE_LIMIT )); then
|
||||
warn " $host_path — strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT, not yet remediating (needs repeat confirmation)"
|
||||
@@ -591,6 +697,8 @@ for arr in sonarr radarr; do
|
||||
fi
|
||||
reset_scan_strikes "$host_path"
|
||||
|
||||
file_id=$(echo "$item" | jq -r '.file_id')
|
||||
parent_id=$(echo "$item" | jq -r '.parent_id')
|
||||
title=$(echo "$item" | jq -r '.title')
|
||||
|
||||
http_code=$(curl -sf -o /dev/null -w "%{http_code}" -X DELETE \
|
||||
@@ -625,7 +733,7 @@ for arr in sonarr radarr; do
|
||||
warn " $title — deleted and verified, but search trigger returned HTTP $search_code"
|
||||
(( REMEDIATE_FAILED++ ))
|
||||
fi
|
||||
done < <(echo "$ITEMS" | jq -c '.[]')
|
||||
done
|
||||
|
||||
merge_clean_state "$FRESH_CLEAN_TMP"
|
||||
rm -f "$FRESH_CLEAN_TMP"
|
||||
@@ -634,6 +742,8 @@ for arr in sonarr radarr; do
|
||||
ARR_SKIPPED_CACHED[$arr]=$SKIPPED_CACHED
|
||||
ARR_SKIPPED_UNMAPPED[$arr]=$SKIPPED_UNMAPPED
|
||||
ARR_CORRUPT[$arr]=$CORRUPT_COUNT
|
||||
ARR_SUSPECT[$arr]=$SUSPECT_COUNT
|
||||
ARR_PROBE_ERRORS[$arr]=$PROBE_ERRORS
|
||||
ARR_STRIKE_HELD[$arr]=$STRIKE_HELD
|
||||
ARR_REMEDIATED[$arr]=$REMEDIATED
|
||||
ARR_REMEDIATE_FAILED[$arr]=$REMEDIATE_FAILED
|
||||
@@ -658,6 +768,8 @@ for arr in sonarr radarr; do
|
||||
echo " $ICON_SUCCESS Skipped (cached): ${ARR_SKIPPED_CACHED[$arr]}"
|
||||
echo " $ICON_WARN Skipped (unmapped): ${ARR_SKIPPED_UNMAPPED[$arr]}"
|
||||
echo " $ICON_ERROR Corrupt found: ${ARR_CORRUPT[$arr]}"
|
||||
echo " $ICON_WARN Warnings (exit 0): ${ARR_SUSPECT[$arr]} (reported, never remediated)"
|
||||
echo " $ICON_WARN Probe errors: ${ARR_PROBE_ERRORS[$arr]} (not counted as corrupt)"
|
||||
if [[ "$REMEDIATE" == true ]]; then
|
||||
echo " $ICON_WARN Held (strikes): ${ARR_STRIKE_HELD[$arr]}"
|
||||
echo " $ICON_SUCCESS Remediated: ${ARR_REMEDIATED[$arr]}"
|
||||
|
||||
Executable → Regular
+70
-8
@@ -457,6 +457,30 @@ NOW=$(date +%s)
|
||||
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
||||
> "$TO_DELETE_FILE"
|
||||
|
||||
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
|
||||
# Same contract as radarr_cleanup.sh: a file must classify for deletion on
|
||||
# LIDARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is removed. Covers the partial
|
||||
# classification failure that is too small to trip the tracked-count floor above. The file is
|
||||
# rebuilt from each run rather than edited, which is what prunes it.
|
||||
LIDARR_ORPHAN_STRIKE_LIMIT="${LIDARR_ORPHAN_STRIKE_LIMIT:-2}"
|
||||
STRIKES_FILE="${LIDARR_ORPHAN_STRIKES_FILE:-$DB_DIR/lidarr_orphan_strikes.tsv}"
|
||||
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
|
||||
touch "$STRIKES_FILE" 2>/dev/null || true
|
||||
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
|
||||
> "$STRIKES_NEW"
|
||||
HELD_COUNT=0
|
||||
HELD_BYTES=0
|
||||
|
||||
orphan_strike_ok() {
|
||||
local path="$1" prev strikes
|
||||
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
|
||||
strikes=$(( ${prev:-0} + 1 ))
|
||||
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
|
||||
(( strikes >= LIDARR_ORPHAN_STRIKE_LIMIT )) && return 0
|
||||
warn " strike $strikes/$LIDARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
|
||||
return 1
|
||||
}
|
||||
|
||||
while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
FILE_CTIME="${FILE_CTIME%%.*}"
|
||||
@@ -493,12 +517,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||
(( ORPHAN_COUNT++ ))
|
||||
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
else
|
||||
log "JUNK: $filepath"
|
||||
(( JUNK_COUNT++ ))
|
||||
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
fi
|
||||
|
||||
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
||||
@@ -506,13 +532,43 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
# 4.3ms), since find already has to stat() every entry anyway to know it's -type f.
|
||||
done < <(find "$LIDARR_MUSIC_ROOT" -type f -printf '%s %C@ %p\n' 2>/dev/null)
|
||||
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||
# Eligible, not classified: a file still serving its strikes is an orphan but is not queued this
|
||||
# run, so it must not appear in the denominator the budget reports against.
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
|
||||
|
||||
# Rebuilt, never edited. Skipped on a dry run: a preview that advanced real counters would make
|
||||
# the next real run delete a run early.
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
||||
# ==============================================================================================
|
||||
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$LIDARR_MAX_DELETE_GB" "Lidarr Cleanup"
|
||||
# A per-run budget, not a veto — see apply_delete_budget() in common.sh. The ceiling still caps
|
||||
# any single run; it just no longer deadlocks on a backlog larger than itself.
|
||||
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
|
||||
|
||||
if [[ "$I_KNOW" == true ]]; then
|
||||
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
|
||||
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
|
||||
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
|
||||
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
|
||||
else
|
||||
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$LIDARR_MAX_DELETE_GB"
|
||||
if [[ -n "$_BUDGET_STUCK" ]]; then
|
||||
error "Single file exceeds the ${LIDARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
|
||||
error " $_BUDGET_STUCK"
|
||||
error "Raise LIDARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
|
||||
notify "Lidarr cleanup stalled on $(hostname) — one file exceeds the ${LIDARR_MAX_DELETE_GB}GB budget" \
|
||||
"Lidarr Cleanup" "warning"
|
||||
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
|
||||
warn "Budget ${LIDARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
|
||||
notify "Lidarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname) — $_BUDGET_DEFERRED_COUNT file(s) deferred" \
|
||||
"Lidarr Cleanup" "normal"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||
# All safety layers passed — delete orphans and junk. Reuses TO_DELETE_FILE from the
|
||||
@@ -521,7 +577,7 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||
done < "$TO_DELETE_FILE"
|
||||
done < "$BUDGET_FILE"
|
||||
|
||||
info "Cleaning up empty folders..."
|
||||
find "$LIDARR_MUSIC_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null
|
||||
@@ -546,6 +602,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (cover art, metadata
|
||||
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${LIDARR_ORPHAN_AGE} days)"
|
||||
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${LIDARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
|
||||
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${LIDARR_MAX_DELETE_GB}GB run budget"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
@@ -554,8 +614,10 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Clean — nothing to remove"
|
||||
else
|
||||
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
||||
notify "Lidarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "warning"
|
||||
# What was actually removed, not what was classified. With strikes and a budget in force those
|
||||
# differ, and reporting the classification as the outcome is the oldest bug shape here.
|
||||
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
|
||||
notify "Lidarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "warning"
|
||||
# Notify Emby to clean missing files — removes ghost entries immediately
|
||||
notify_emby_scan
|
||||
fi
|
||||
|
||||
Executable → Regular
+134
-8
@@ -450,9 +450,46 @@ NOW=$(date +%s)
|
||||
# just delete them directly instead of re-walking and re-classifying every SCAN_ROOTS entry a
|
||||
# second time (2026-07-17) — the size-threshold check below needs to know the total before
|
||||
# deleting anything, not before knowing what to delete.
|
||||
# Carries size and ctime alongside the path now, because the budget pass below has to order by
|
||||
# age and stop at a byte ceiling — neither of which a bare path list can answer.
|
||||
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
||||
> "$TO_DELETE_FILE"
|
||||
|
||||
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
|
||||
# A file must classify for deletion on RADARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is
|
||||
# actually removed. Gate 6 already refuses a run whose tracked count collapsed; this covers the
|
||||
# partial failure underneath that threshold — one root folder failing to enumerate makes its
|
||||
# movies look orphaned while the overall percentage still looks fine, and a transient fault will
|
||||
# not reproduce on the next run.
|
||||
#
|
||||
# The file is REBUILT from this run's classifications rather than edited in place, which is what
|
||||
# prunes it: anything that stopped being an orphan simply is not written again, so a file that
|
||||
# Radarr re-adopts loses its strikes without needing a reset pass to find it.
|
||||
#
|
||||
# Keyed by host path, which is why this could not have worked before 2026-08-26 — wd_state_set
|
||||
# built a regex from the key, and a release tag like [Bluray-1080p] holds the reversed range 1-0,
|
||||
# so every write truncated the store to one line. See common.sh.
|
||||
RADARR_ORPHAN_STRIKE_LIMIT="${RADARR_ORPHAN_STRIKE_LIMIT:-2}"
|
||||
STRIKES_FILE="${RADARR_ORPHAN_STRIKES_FILE:-$DB_DIR/radarr_orphan_strikes.tsv}"
|
||||
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
|
||||
touch "$STRIKES_FILE" 2>/dev/null || true
|
||||
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
|
||||
> "$STRIKES_NEW"
|
||||
HELD_COUNT=0
|
||||
HELD_BYTES=0
|
||||
|
||||
# Records this run's strike for a file and says whether it has served enough of them.
|
||||
# Returns 0 when the file may be deleted, 1 when it is still accruing.
|
||||
orphan_strike_ok() {
|
||||
local path="$1" prev strikes
|
||||
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
|
||||
strikes=$(( ${prev:-0} + 1 ))
|
||||
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
|
||||
(( strikes >= RADARR_ORPHAN_STRIKE_LIMIT )) && return 0
|
||||
warn " strike $strikes/$RADARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
|
||||
return 1
|
||||
}
|
||||
|
||||
while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
FILE_CTIME="${FILE_CTIME%%.*}"
|
||||
@@ -487,12 +524,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||
(( ORPHAN_COUNT++ ))
|
||||
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
else
|
||||
log "JUNK: $filepath"
|
||||
(( JUNK_COUNT++ ))
|
||||
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
fi
|
||||
|
||||
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
||||
@@ -504,13 +543,94 @@ done < <(
|
||||
done | sort -u
|
||||
)
|
||||
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||
# Eligible, not classified. A file still serving its strikes was counted as an orphan above — it
|
||||
# is one — but it is not going to be deleted this run, so it must not appear in the denominator
|
||||
# the budget reports against or the run claims to have skipped work it never queued.
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
|
||||
|
||||
# Rebuilt, never edited: a path absent from this run is absent from the file, so a file Radarr
|
||||
# re-adopts drops its strikes with no reset pass needed. Skipped on a dry run — a preview that
|
||||
# advanced real strike counters would make the next real run delete a run early.
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
||||
# ==============================================================================================
|
||||
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$RADARR_MAX_DELETE_GB" "Radarr Cleanup"
|
||||
# The ceiling is a per-run budget, not a veto. It still means what it always meant — no single run
|
||||
# removes more than RADARR_MAX_DELETE_GB — but a backlog larger than the ceiling now drains over
|
||||
# consecutive nights instead of failing the orchestrator forever on a queue it cannot clear.
|
||||
# ── AI note (AI_ASSIST_CLEANUP) ───────────────────────────────────────────────────────────────
|
||||
# Describes the shape of what was classified. It decides nothing: the eligible set, the budget and
|
||||
# the strikes are all settled above and none of them read this. Switch AI_ASSIST_CLEANUP off and
|
||||
# the run removes exactly the same files — the log just loses a paragraph.
|
||||
#
|
||||
# ctime clustering is the signal worth surfacing. A normal upgrade cycle dribbles in over weeks; a
|
||||
# lump sharing one narrow ctime window with mtimes spread across months is a bulk write-back, which
|
||||
# is what a partnership merge against a partner holding older copies produces. That distinction
|
||||
# took a person an evening on 2026-08-26 and is the whole reason this note exists.
|
||||
if [[ "$ORPHAN_COUNT" -gt 0 ]] && [[ -s "$TO_DELETE_FILE" ]]; then
|
||||
_ai_ev=$(awk -F'\t' '
|
||||
{ n++; bytes += $1
|
||||
c = int($2)
|
||||
if (cmin == 0 || c < cmin) cmin = c
|
||||
if (c > cmax) cmax = c
|
||||
bucket[int(c / 21600)]++ }
|
||||
END {
|
||||
for (b in bucket) if (bucket[b] > top) { top = bucket[b] }
|
||||
printf "files=%d bytes_gb=%.1f ctime_span_hours=%.1f largest_6h_ctime_bucket=%d\n",
|
||||
n, bytes/1073741824, (cmax-cmin)/3600, top
|
||||
}' "$TO_DELETE_FILE")
|
||||
_ai_mt=$(cut -d"$(printf '\t')" -f3 "$TO_DELETE_FILE" | head -8 \
|
||||
| while IFS= read -r p; do [[ -f "$p" ]] && \
|
||||
printf '%s %s\n' "$(stat -c %y "$p" 2>/dev/null | cut -c1-7)" "$(basename "$p")"; done)
|
||||
|
||||
_ai_note=$(ai_assist_note AI_ASSIST_CLEANUP "You are looking at files an automated media-library cleanup has classified for deletion on an Unraid server. They are files on disk that the Radarr database no longer references.
|
||||
|
||||
EVIDENCE
|
||||
$_ai_ev
|
||||
sample (modification month, then path):
|
||||
$_ai_mt
|
||||
|
||||
A normal quality-upgrade cycle produces orphans whose ctimes are spread out over weeks, because each upgrade happens on its own day. A bulk event - a sync or restore writing files back onto this host - produces orphans sharing one narrow ctime window while their modification times stay spread across months, because the copy preserves modification time but resets ctime.
|
||||
|
||||
In no more than three sentences, say which of those two this looks like and name the numbers above that support it. Do not recommend an action. Do not speculate beyond the evidence given.") || _ai_note=""
|
||||
|
||||
if [[ -n "$_ai_note" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR AI note on this classification ━━━"
|
||||
printf '%s\n' "$_ai_note"
|
||||
fi
|
||||
unset _ai_ev _ai_mt
|
||||
fi
|
||||
|
||||
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
|
||||
|
||||
if [[ "$I_KNOW" == true ]]; then
|
||||
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
|
||||
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
|
||||
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
|
||||
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
|
||||
else
|
||||
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$RADARR_MAX_DELETE_GB"
|
||||
|
||||
if [[ -n "$_BUDGET_STUCK" ]]; then
|
||||
# One file larger than the whole budget can never fit, so it would be re-found and
|
||||
# re-deferred every night. Name it rather than loop on it silently.
|
||||
error "Single file exceeds the ${RADARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
|
||||
error " $_BUDGET_STUCK"
|
||||
error "Raise RADARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
|
||||
notify "Radarr cleanup stalled on $(hostname) — one file exceeds the ${RADARR_MAX_DELETE_GB}GB budget" \
|
||||
"Radarr Cleanup" "warning"
|
||||
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
|
||||
warn "Budget ${RADARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
|
||||
warn "Oldest first — the deferred files are the newest and are re-evaluated tomorrow"
|
||||
notify "Radarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname) — $_BUDGET_DEFERRED_COUNT file(s) deferred to the next run" \
|
||||
"Radarr Cleanup" "normal"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||
# Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and
|
||||
@@ -519,7 +639,7 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||
done < "$TO_DELETE_FILE"
|
||||
done < "$BUDGET_FILE"
|
||||
|
||||
info "Cleaning up empty folders..."
|
||||
for host_path in "${SCAN_ROOTS[@]}"; do
|
||||
@@ -545,6 +665,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles,
|
||||
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${RADARR_ORPHAN_AGE} days)"
|
||||
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${RADARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
|
||||
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${RADARR_MAX_DELETE_GB}GB run budget"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
@@ -553,8 +677,10 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Clean — nothing to remove"
|
||||
else
|
||||
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
||||
notify "Radarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
|
||||
# What was actually removed, not what was classified. With a budget in force those differ, and
|
||||
# reporting the classification as the outcome is the oldest bug shape in this codebase.
|
||||
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED classified files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
|
||||
notify "Radarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED classified files ($(format_bytes "$_BUDGET_KEPT_BYTES"))$([[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && echo ", $_BUDGET_DEFERRED_COUNT deferred")" \
|
||||
"Radarr Cleanup" "warning"
|
||||
# Notify Emby to clean missing files — removes ghost entries immediately
|
||||
notify_emby_scan
|
||||
|
||||
Executable → Regular
+70
-8
@@ -440,6 +440,30 @@ NOW=$(date +%s)
|
||||
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
||||
> "$TO_DELETE_FILE"
|
||||
|
||||
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
|
||||
# Same contract as radarr_cleanup.sh: a file must classify for deletion on
|
||||
# SONARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is removed. Covers the partial
|
||||
# classification failure that is too small to trip the tracked-count floor above. The file is
|
||||
# rebuilt from each run rather than edited, which is what prunes it.
|
||||
SONARR_ORPHAN_STRIKE_LIMIT="${SONARR_ORPHAN_STRIKE_LIMIT:-2}"
|
||||
STRIKES_FILE="${SONARR_ORPHAN_STRIKES_FILE:-$DB_DIR/sonarr_orphan_strikes.tsv}"
|
||||
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
|
||||
touch "$STRIKES_FILE" 2>/dev/null || true
|
||||
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
|
||||
> "$STRIKES_NEW"
|
||||
HELD_COUNT=0
|
||||
HELD_BYTES=0
|
||||
|
||||
orphan_strike_ok() {
|
||||
local path="$1" prev strikes
|
||||
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
|
||||
strikes=$(( ${prev:-0} + 1 ))
|
||||
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
|
||||
(( strikes >= SONARR_ORPHAN_STRIKE_LIMIT )) && return 0
|
||||
warn " strike $strikes/$SONARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
|
||||
return 1
|
||||
}
|
||||
|
||||
while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
FILE_CTIME="${FILE_CTIME%%.*}"
|
||||
@@ -474,12 +498,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||
(( ORPHAN_COUNT++ ))
|
||||
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
else
|
||||
log "JUNK: $filepath"
|
||||
(( JUNK_COUNT++ ))
|
||||
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
fi
|
||||
|
||||
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
||||
@@ -491,13 +517,43 @@ done < <(
|
||||
done | sort -u
|
||||
)
|
||||
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||
# Eligible, not classified: a file still serving its strikes is an orphan but is not queued this
|
||||
# run, so it must not appear in the denominator the budget reports against.
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
|
||||
|
||||
# Rebuilt, never edited. Skipped on a dry run: a preview that advanced real counters would make
|
||||
# the next real run delete a run early.
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
||||
# ==============================================================================================
|
||||
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$SONARR_MAX_DELETE_GB" "Sonarr Cleanup"
|
||||
# A per-run budget, not a veto — see apply_delete_budget() in common.sh. The ceiling still caps
|
||||
# any single run; it just no longer deadlocks on a backlog larger than itself.
|
||||
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
|
||||
|
||||
if [[ "$I_KNOW" == true ]]; then
|
||||
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
|
||||
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
|
||||
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
|
||||
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
|
||||
else
|
||||
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$SONARR_MAX_DELETE_GB"
|
||||
if [[ -n "$_BUDGET_STUCK" ]]; then
|
||||
error "Single file exceeds the ${SONARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
|
||||
error " $_BUDGET_STUCK"
|
||||
error "Raise SONARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
|
||||
notify "Sonarr cleanup stalled on $(hostname) — one file exceeds the ${SONARR_MAX_DELETE_GB}GB budget" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
|
||||
warn "Budget ${SONARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
|
||||
notify "Sonarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname) — $_BUDGET_DEFERRED_COUNT file(s) deferred" \
|
||||
"Sonarr Cleanup" "normal"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||
# Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and
|
||||
@@ -506,7 +562,7 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||
done < "$TO_DELETE_FILE"
|
||||
done < "$BUDGET_FILE"
|
||||
|
||||
info "Cleaning up empty folders..."
|
||||
for host_path in "${SCAN_ROOTS[@]}"; do
|
||||
@@ -532,6 +588,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles,
|
||||
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${SONARR_ORPHAN_AGE} days)"
|
||||
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${SONARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
|
||||
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${SONARR_MAX_DELETE_GB}GB run budget"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
@@ -540,8 +600,10 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Clean — nothing to remove"
|
||||
else
|
||||
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
||||
notify "Sonarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
|
||||
# What was actually removed, not what was classified. With strikes and a budget in force those
|
||||
# differ, and reporting the classification as the outcome is the oldest bug shape here.
|
||||
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
|
||||
notify "Sonarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
# Notify Emby to clean missing files — removes ghost entries immediately
|
||||
notify_emby_scan
|
||||
|
||||
@@ -1268,6 +1268,8 @@
|
||||
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||
# protects against API returning partial data on a bad day
|
||||
LIDARR_TRACKED_COUNT_FILE="${DB_DIR}/lidarr_tracked.count"
|
||||
LIDARR_ORPHAN_STRIKES_FILE="${DB_DIR}/lidarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
|
||||
LIDARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify before it is removed
|
||||
LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||
|
||||
# Lidarr tracked-data cache — shared by lidarr_cleanup.sh, lidarr_duplicate_artist_cleanup.sh,
|
||||
@@ -1343,6 +1345,8 @@
|
||||
SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||
# protects against API returning partial data on a bad day
|
||||
SONARR_TRACKED_COUNT_FILE="${DB_DIR}/sonarr_tracked.count"
|
||||
SONARR_ORPHAN_STRIKES_FILE="${DB_DIR}/sonarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
|
||||
SONARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify before it is removed
|
||||
SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||
SONARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveSeries command to
|
||||
# reach "completed" — generous because a large series can sit
|
||||
@@ -1355,6 +1359,15 @@
|
||||
# against a one-off ffprobe hiccup (mid-write file, NFS blip)
|
||||
# triggering an unnecessary delete. Resets to 0 the moment a
|
||||
# file probes clean again.
|
||||
CORRUPTION_SCAN_MAX_CORRUPT_PCT=10 # abort the run, committing nothing, if this share of
|
||||
# newly-scanned files reads as corrupt. A healthy library sits
|
||||
# near zero; a high rate means the detector is wrong, not the
|
||||
# library. Only counts ffprobe exit != 0.
|
||||
CORRUPTION_SCAN_MAX_CONSECUTIVE=15 # abort after this many files in a row fail to probe
|
||||
# cleanly — catches the ffprobe container dying mid-scan,
|
||||
# which the pre-flight health check cannot see.
|
||||
CORRUPTION_SCAN_GUARD_MIN_SCANNED=20 # below this many newly-scanned files the percentage
|
||||
# ceiling is not applied — too small a sample to judge.
|
||||
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
|
||||
SONARR_PROTECTED_PATTERNS=(
|
||||
# Subtitles
|
||||
@@ -1381,6 +1394,12 @@
|
||||
RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||
# protects against API returning partial data on a bad day
|
||||
RADARR_TRACKED_COUNT_FILE="${DB_DIR}/radarr_tracked.count"
|
||||
RADARR_ORPHAN_STRIKES_FILE="${DB_DIR}/radarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
|
||||
RADARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify for deletion before it is
|
||||
# removed. Gate 6 already catches an API returning far too few
|
||||
# tracked files; this catches the partial failure too small to trip
|
||||
# that percentage — one root folder failing to enumerate makes its
|
||||
# movies look orphaned, and a transient one will not repeat.
|
||||
RADARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||
RADARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveMovie command to
|
||||
# reach "completed" — mirrors SONARR_MOVE_POLL_TIMEOUT
|
||||
@@ -1647,6 +1666,10 @@
|
||||
# HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES
|
||||
BACKUP_VERIFY_SAMPLE=10 # random files to check per share
|
||||
BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample
|
||||
BACKUP_VERIFY_MD5_TIMEOUT_MAX=600 # ceiling for one remote checksum. The per-file budget
|
||||
# scales with size (~50MB/s); this caps it. A fixed
|
||||
# connect-sized timeout killed multi-GB checksums and
|
||||
# the empty result was then reported as MISSING.
|
||||
|
||||
# ━━━ SMART Health ━━━
|
||||
# Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*.
|
||||
@@ -2025,7 +2048,10 @@
|
||||
AI_ASSIST_REPORTS=false # tier 1 — digest / coffee report narration
|
||||
AI_ASSIST_WATCHDOG=false # tier 2 — file a finding when a watchdog counter passes its limit (needs AI_REPAIR_ENABLED)
|
||||
AI_ASSIST_DISCOVERY=false # tier 2 — discovery / classification judgement calls
|
||||
AI_ASSIST_CLEANUP=false # tier 2 — orphan and stuck-import triage
|
||||
AI_ASSIST_CLEANUP=false # tier 2 — orphan and stuck-import triage. Describes the shape of a
|
||||
# classification in the log; decides nothing. Off = identical deletions.
|
||||
AI_ASSIST_TIMEOUT=45 # seconds any single assist may take. An assist that can stall a
|
||||
# nightly cleanup is not an assist — it is silently skipped past this.
|
||||
AI_ASSIST_ONBOARD=false # tier 3 — onboarding / settings assistance
|
||||
|
||||
# ━━━ AI Repair ━━━
|
||||
|
||||
Executable → Regular
+46
-10
@@ -122,6 +122,7 @@ source "$SCRIPT_DIR/../load_config.sh"
|
||||
parse_args "$@"
|
||||
|
||||
SSH_TIMEOUT=15
|
||||
BACKUP_VERIFY_MD5_TIMEOUT_MAX="${BACKUP_VERIFY_MD5_TIMEOUT_MAX:-600}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
@@ -228,6 +229,7 @@ TOTAL_CHECKED=0
|
||||
TOTAL_MATCH=0
|
||||
TOTAL_MISMATCH=0
|
||||
TOTAL_MISSING=0
|
||||
TOTAL_UNVERIFIED=0
|
||||
SHARES_WITH_ISSUES=()
|
||||
|
||||
for share in "${VERIFY_SHARES[@]}"; do
|
||||
@@ -265,6 +267,7 @@ for share in "${VERIFY_SHARES[@]}"; do
|
||||
SHARE_MATCH=0
|
||||
SHARE_MISMATCH=0
|
||||
SHARE_MISSING=0
|
||||
SHARE_UNVERIFIED=0
|
||||
|
||||
for local_file in "${SAMPLE_FILES[@]}"; do
|
||||
[[ -z "$local_file" ]] && continue
|
||||
@@ -276,19 +279,49 @@ for share in "${VERIFY_SHARES[@]}"; do
|
||||
continue
|
||||
fi
|
||||
|
||||
# Remote checksum via SSH — timeout protected
|
||||
remote_md5=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
# The path is interpolated into a remote shell command, so it must be escaped for
|
||||
# reuse as one word. A bare '$local_file' inside single quotes breaks on the first
|
||||
# apostrophe — "Frieren - Beyond Journey's End" ended the quote early, md5sum fell
|
||||
# back to reading stdin, and the empty-input hash d41d8cd9... was reported as a
|
||||
# MISMATCH against a file that is byte-identical on the remote.
|
||||
printf -v remote_q '%q' "$local_file"
|
||||
|
||||
# Existence and content are separate questions. Asking them together means a slow
|
||||
# checksum is indistinguishable from an absent file.
|
||||
remote_exists=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o StrictHostKeyChecking=no \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"md5sum '$local_file' 2>/dev/null | awk '{print \$1}'" 2>/dev/null)
|
||||
"test -f $remote_q && echo yes" 2>/dev/null </dev/null)
|
||||
|
||||
(( TOTAL_CHECKED++ ))
|
||||
|
||||
if [[ -z "$remote_md5" ]]; then
|
||||
if [[ "$remote_exists" != "yes" ]]; then
|
||||
warn "$ICON_ERROR MISSING: $(basename "$local_file")"
|
||||
(( SHARE_MISSING++ ))
|
||||
(( TOTAL_MISSING++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# md5sum of a multi-GB file cannot finish inside a connect-sized timeout. Budget by
|
||||
# size — a 5.9GB file needs ~30s and was being killed at 15s, then counted MISSING
|
||||
# even though it was present and correct.
|
||||
local_size=$(stat -c%s "$local_file" 2>/dev/null || echo 0)
|
||||
md5_timeout=$(( local_size / 52428800 + SSH_TIMEOUT ))
|
||||
(( md5_timeout > BACKUP_VERIFY_MD5_TIMEOUT_MAX )) && md5_timeout=$BACKUP_VERIFY_MD5_TIMEOUT_MAX
|
||||
|
||||
remote_md5=$(timeout "$md5_timeout" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o StrictHostKeyChecking=no \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"md5sum $remote_q 2>/dev/null | awk '{print \$1}'" 2>/dev/null </dev/null)
|
||||
|
||||
if [[ -z "$remote_md5" ]]; then
|
||||
# Present but unreadable within budget. Reporting this as a mismatch or a miss
|
||||
# would be a claim the run did not earn.
|
||||
warn "$ICON_WARN UNVERIFIED (checksum timed out after ${md5_timeout}s): $(basename "$local_file")"
|
||||
(( SHARE_UNVERIFIED++ ))
|
||||
(( TOTAL_UNVERIFIED++ ))
|
||||
elif [[ "$local_md5" == "$remote_md5" ]]; then
|
||||
log "MATCH: $(basename "$local_file")"
|
||||
(( SHARE_MATCH++ ))
|
||||
@@ -303,8 +336,8 @@ for share in "${VERIFY_SHARES[@]}"; do
|
||||
done
|
||||
|
||||
# Per-share result — only visible if issues found
|
||||
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 ]]; then
|
||||
warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH"
|
||||
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 || "$SHARE_UNVERIFIED" -gt 0 ]]; then
|
||||
warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH unverified: $SHARE_UNVERIFIED"
|
||||
SHARES_WITH_ISSUES+=("$SHARE_NAME")
|
||||
else
|
||||
echo "$SHARE_NAME — all $SHARE_MATCH files match ✅"
|
||||
@@ -325,10 +358,11 @@ echo "$ICON_VERIFY Checked: $TOTAL_CHECKED files"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
||||
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
|
||||
warn "Missing: $TOTAL_MISSING"
|
||||
[[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
|
||||
if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 || "$TOTAL_UNVERIFIED" -gt 0 ]]; then
|
||||
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
|
||||
warn "Missing: $TOTAL_MISSING"
|
||||
[[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
|
||||
[[ "$TOTAL_UNVERIFIED" -gt 0 ]] && warn "Unverified: $TOTAL_UNVERIFIED (present, checksum timed out)"
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
@@ -337,6 +371,8 @@ elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention: ${SHARES_WITH_ISSUES[*]}"
|
||||
notify "Backup verify FAILED on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" \
|
||||
"Backup Verify" "warning"
|
||||
elif [[ "$TOTAL_UNVERIFIED" -gt 0 ]]; then
|
||||
warn "Status: $TOTAL_MATCH verified, $TOTAL_UNVERIFIED could not be checksummed in time — NOT a clean run"
|
||||
else
|
||||
echo "$ICON_DONE Status: all $TOTAL_CHECKED files match across ${#VERIFY_SHARES[@]} shares ✅"
|
||||
fi
|
||||
|
||||
@@ -1363,6 +1363,15 @@ vv_ai_profiles_script();
|
||||
profile: profile,
|
||||
question: q,
|
||||
history: JSON.stringify(sendable()),
|
||||
// Where the caller is standing, so a question can say "this setting" and mean it. The
|
||||
// backend whitelists it and passes it to the worker, which turns it into a location
|
||||
// line; a page that sets no scope sends '' and the worker omits that line.
|
||||
//
|
||||
// Keep this here. The scope-aware dock in 185abdb sent it from the page's own ask call;
|
||||
// 8eeb4c3 folded that dock into this component and the line did not come with it, so
|
||||
// for the whole of that window Scheduler computed a scope, re-read it at send time, and
|
||||
// the model never saw it. Nothing errored — the answer just arrived ungrounded.
|
||||
scope: (typeof o.scope === 'function' ? o.scope() : (o.scope || '')),
|
||||
kind: (PROFILES[profile].kind && kindEl) ? kindEl.value : '',
|
||||
// A checkbox where the page offers one, otherwise whatever the page decides from the
|
||||
// profile, otherwise on. The Scheduler reasons only when diagnosing: working out what a
|
||||
|
||||
@@ -37,6 +37,12 @@
|
||||
// Step 2: auto-populate + guide + checklist.
|
||||
// master.conf pull (for partner servers) lives in the checklist, not here.
|
||||
|
||||
// The assistant is a bonus on this page, never a dependency: vv_ai_ui_on() is false on a node
|
||||
// with AI off or no model reachable, which is the normal state of the fresh install this wizard
|
||||
// exists to serve. The card simply is not rendered and setup proceeds exactly as before.
|
||||
require_once dirname(__DIR__) . '/include/ai_chat.php';
|
||||
if (vv_ai_ui_on()) vv_ai_chat_assets();
|
||||
|
||||
$detectedHostname = vv_get_hostname();
|
||||
|
||||
// ── Identity already on disk? ────────────────────────────────────────────────────────────────
|
||||
@@ -313,6 +319,21 @@ hr.vv-hr { border: none; border-top: 1px solid #1e1e1e; margin: 22px 0; }
|
||||
<div id="vv-onboard-panel"></div>
|
||||
<div id="vv-done-banner"></div>
|
||||
|
||||
<?php if (vv_ai_ui_on()): ?>
|
||||
<hr class="vv-hr">
|
||||
<div id="vv-su-ai-card">
|
||||
<?php vv_ai_chat_markup('vv-su-ai', [
|
||||
'profile' => 'varaverk',
|
||||
'compact' => true,
|
||||
'title' => 'Setup assistant',
|
||||
'height' => '260px',
|
||||
'tall' => '460px',
|
||||
'empty' => 'Stuck on a step? Ask what it wants and why — "what is the API key for", '
|
||||
. '"why does SSH need the other host", "what does populate actually do".',
|
||||
]); ?>
|
||||
</div>
|
||||
<?php endif; ?>
|
||||
|
||||
<div style="margin-top:18px;text-align:right;">
|
||||
<a href="#" id="vv-exit-link" onclick="vvGoNext(event)"
|
||||
style="font-size:11px;color:#3a3a3a;text-decoration:none;">Skip →</a>
|
||||
@@ -443,6 +464,7 @@ function vvShowStep2(redirect, apiKey) {
|
||||
_vvRedirect = redirect || '?tab=scheduler';
|
||||
document.getElementById('vv-step1').style.display = 'none';
|
||||
document.getElementById('vv-step2').style.display = 'block';
|
||||
vvSetupAiInit(); // card is visible now, so it can measure itself correctly
|
||||
if (apiKey && apiKey.ok) {
|
||||
const btn = document.getElementById('vv-key-btn');
|
||||
const status = document.getElementById('vv-key-status');
|
||||
@@ -676,6 +698,47 @@ function vvStartPhase2Watch(sinceMs) {
|
||||
_vvPartnerPoll = setInterval(check, 6000);
|
||||
}
|
||||
|
||||
// ── Setup assistant ───────────────────────────────────────────────────────────
|
||||
// Scoped to the step the operator is actually stuck on, so "what does this want" resolves without
|
||||
// them naming it. The worker turns a non-empty scope into a line saying what is open in the
|
||||
// WebGUI; the varaverk profile holds no scoped_log capability, so a checklist id never triggers a
|
||||
// log lookup and cannot produce a "log missing" note for something that was never a script.
|
||||
//
|
||||
// Re-read at send time, not captured: the checklist re-polls while the page is open and the step
|
||||
// they are on can change between opening the composer and pressing Ask.
|
||||
//
|
||||
// First not-ok, not-deferred item — deferred means "I have decided to skip this", which is not
|
||||
// where they are stuck. Falls back to the bare page when the list is clean or has not loaded.
|
||||
function vvSetupScope() {
|
||||
const d = _vvLastChecklist;
|
||||
if (!d || !Array.isArray(d.items)) return 'Setup';
|
||||
const stuck = d.items.find(i => !i.ok && !i.deferred);
|
||||
return stuck && stuck.id ? 'Setup/' + stuck.id : 'Setup';
|
||||
}
|
||||
|
||||
// Mounted when Step 2 is revealed, not at parse time. This is the only page where the card starts
|
||||
// inside a display:none block, and a chat that measured itself while hidden would come up wrong
|
||||
// with nothing to correct it — the component carries no resize observer. Idempotent because
|
||||
// vvShowStep2 is reachable more than once.
|
||||
let _vvSetupAiUp = false;
|
||||
function vvSetupAiInit() {
|
||||
if (_vvSetupAiUp) return;
|
||||
if (typeof VvAiChat !== 'function') return;
|
||||
if (!document.getElementById('vv-su-ai-chat')) return; // AI off — card was never rendered
|
||||
_vvSetupAiUp = true;
|
||||
VvAiChat({
|
||||
prefix: 'vv-su-ai',
|
||||
profile: 'varaverk',
|
||||
// Pinned, for the same reason the Partnership card pins it: resuming whatever thread was last
|
||||
// touched anywhere in the UI could land a first-run operator mid-way through someone else's
|
||||
// Scheduler conversation, on a card with no picker to get back from.
|
||||
resumeProfile: 'varaverk',
|
||||
scope: vvSetupScope,
|
||||
scopeLabel: 'Setup',
|
||||
empty: 'Stuck on a step? Ask what it wants and why.',
|
||||
});
|
||||
}
|
||||
|
||||
function vvLoadChecklist() {
|
||||
fetch('/plugins/varaverk/api/checklist.php?_=' + Date.now())
|
||||
.then(r => r.json()).then(d => {
|
||||
|
||||
@@ -270,7 +270,17 @@ log "$ICON_SUCCESS Internet reachable"
|
||||
# ==============================================================================================
|
||||
if [[ -n "$DDNS_DOMAIN" ]] && [[ -n "$DDNS_CONTAINER" ]]; then
|
||||
PUBLIC_IP=$(curl -sf --max-time 5 https://ifconfig.me 2>/dev/null | tr -d '[:space:]')
|
||||
DNS_IP=$(dig +short "$DDNS_DOMAIN" @1.1.1.1 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1)
|
||||
# dig's failure text names the resolver it could not reach (";; communications error to
|
||||
# 1.1.1.1#53: timed out"), so scraping its output for an address yields the server, not the
|
||||
# answer — a guaranteed mismatch that restarts DDNS over what is only a DNS timeout. Trust
|
||||
# the exit status, and anchor the match so only a bare answer line counts. NXDOMAIN exits 0
|
||||
# with no output and correctly falls through to the "could not resolve" branch below.
|
||||
if DNS_ANSWER=$(dig +short "$DDNS_DOMAIN" @1.1.1.1 2>/dev/null); then
|
||||
DNS_IP=$(printf '%s\n' "$DNS_ANSWER" \
|
||||
| grep -Eox '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1)
|
||||
else
|
||||
DNS_IP=""
|
||||
fi
|
||||
|
||||
if [[ -z "$PUBLIC_IP" ]]; then
|
||||
warn "Could not determine public IP — skipping DDNS check"
|
||||
|
||||
@@ -2273,8 +2273,10 @@ acquire_rsync_lock() {
|
||||
# ==============================================================================================
|
||||
# Generic get/set for flat "key<sep>value" state files (one entry per line) — the pattern
|
||||
# every watchdog's strike-tracking and state-file logic was independently reimplementing.
|
||||
# grep -v + tempfile-swap on write, not sed -i in place — avoids sed treating a key containing
|
||||
# regex metacharacters (container names, etc.) as part of the substitution pattern.
|
||||
# Keys are matched as literal prefixes via awk substr(), never as regexes. A key is often a
|
||||
# media file path, and a release tag like [Bluray-1080p] is a valid-looking bracket expression
|
||||
# holding the reversed range 1-0 — grep -E rejects it, exits 2, and the tempfile-swap then
|
||||
# commits an empty file, silently wiping every other entry.
|
||||
#
|
||||
# Usage: wd_state_get "$key" "$file" [sep=:]
|
||||
# wd_state_set "$key" "$value" "$file" [sep=:]
|
||||
@@ -2284,14 +2286,24 @@ acquire_rsync_lock() {
|
||||
# keep their own thin same-named wrapper around these rather than changing call sites.
|
||||
wd_state_get() {
|
||||
local key="$1" file="$2" sep="${3:-:}"
|
||||
grep -E "^${key}${sep}" "$file" 2>/dev/null | cut -d"$sep" -f2-
|
||||
[[ -f "$file" ]] || return 0
|
||||
awk -v pfx="${key}${sep}" \
|
||||
'substr($0, 1, length(pfx)) == pfx { print substr($0, length(pfx) + 1); exit }' \
|
||||
"$file" 2>/dev/null
|
||||
}
|
||||
|
||||
# Returns 1 without touching the state file if the rewrite fails, so a read error costs the
|
||||
# caller one update rather than the whole file.
|
||||
wd_state_set() {
|
||||
local key="$1" value="$2" file="$3" sep="${4:-:}"
|
||||
grep -vE "^${key}${sep}" "$file" 2>/dev/null > "${file}.tmp"
|
||||
echo "${key}${sep}${value}" >> "${file}.tmp"
|
||||
mv "${file}.tmp" "$file"
|
||||
local tmp="${file}.tmp"
|
||||
: > "$tmp" || return 1
|
||||
if [[ -f "$file" ]]; then
|
||||
awk -v pfx="${key}${sep}" \
|
||||
'substr($0, 1, length(pfx)) != pfx' "$file" > "$tmp" || { rm -f "$tmp"; return 1; }
|
||||
fi
|
||||
echo "${key}${sep}${value}" >> "$tmp"
|
||||
mv "$tmp" "$file"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
@@ -2722,6 +2734,100 @@ check_delete_size_threshold() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Applies a per-run delete budget instead of refusing the whole run. Reads "<size>\t<ctime>\t<path>"
|
||||
# from $1, writes the paths that fit inside $3 GB to $2 (oldest ctime first), and reports what was
|
||||
# held back in _BUDGET_*.
|
||||
#
|
||||
# A cap that aborts cannot drain a backlog larger than itself. Every run re-finds the same lump,
|
||||
# exceeds the same ceiling and stops, so the queue is stuck permanently and only a human with
|
||||
# --i-know-what-im-doing can clear it. Observed 2026-08-24..26: a partnership merge restored ten
|
||||
# months of superseded Radarr files in one night, 126GB against a 100GB cap, and the daily
|
||||
# orchestrator then failed three nights running on a queue it could never drain. Budgeting keeps
|
||||
# the ceiling meaning exactly what it meant — no single run removes more than max_gb — while the
|
||||
# backlog clears over consecutive runs with nobody touching it.
|
||||
#
|
||||
# Strict oldest-first, stopping at the first file that does not fit rather than skipping it for a
|
||||
# smaller one further down. Skipping would drain more per run but lets a large file be passed over
|
||||
# indefinitely; stopping guarantees every deferred file is newer than everything already removed,
|
||||
# so nothing can starve.
|
||||
#
|
||||
# One case is deliberately left for a human: a single file larger than the entire budget can never
|
||||
# fit, so _BUDGET_STUCK is set and the caller says so rather than looping forever on it.
|
||||
apply_delete_budget() {
|
||||
local src="$1" dst="$2" max_gb="$3"
|
||||
local max_bytes tab
|
||||
tab=$(printf '\t')
|
||||
max_bytes=$(awk "BEGIN {printf \"%d\", $max_gb * 1073741824}")
|
||||
|
||||
_BUDGET_KEPT_BYTES=0; _BUDGET_KEPT_COUNT=0
|
||||
_BUDGET_DEFERRED_BYTES=0; _BUDGET_DEFERRED_COUNT=0
|
||||
_BUDGET_STUCK=""
|
||||
|
||||
: > "$dst"
|
||||
[[ -s "$src" ]] || return 0
|
||||
|
||||
local out
|
||||
out=$(sort -t"$tab" -k2,2n "$src" | awk -F'\t' -v max="$max_bytes" -v dst="$dst" '
|
||||
{
|
||||
if (stop) { defb += $1; defc++; next }
|
||||
if (used + $1 > max) { stop = 1
|
||||
if (kept == 0) stuck = $3
|
||||
defb += $1; defc++; next }
|
||||
used += $1; kept++
|
||||
print $3 > dst
|
||||
}
|
||||
END { printf "%d\t%d\t%d\t%d\t%s", used+0, kept+0, defb+0, defc+0, stuck }
|
||||
')
|
||||
IFS=$'\t' read -r _BUDGET_KEPT_BYTES _BUDGET_KEPT_COUNT \
|
||||
_BUDGET_DEFERRED_BYTES _BUDGET_DEFERRED_COUNT _BUDGET_STUCK <<< "$out"
|
||||
}
|
||||
|
||||
# Asks the local generation model to characterise a block of evidence, for the one line a human
|
||||
# would otherwise have to derive by hand. Prints the note on stdout and returns 0; returns 1 and
|
||||
# prints nothing whenever anything at all is missing, off, slow or malformed.
|
||||
#
|
||||
# The contract is that failure is indistinguishable from the feature being switched off, because
|
||||
# every caller must behave identically either way. An assist that can block a nightly cleanup is
|
||||
# not an assist — so this never retries, never blocks longer than its timeout, and never returns
|
||||
# a partial answer for a caller to interpret.
|
||||
#
|
||||
# Usage: note=$(ai_assist_note AI_ASSIST_CLEANUP "$prompt") && echo "$note"
|
||||
# $1 name of the AI_ASSIST_* flag governing this caller — passed by name, checked here, so a
|
||||
# caller cannot accidentally run an assist its own toggle says is off
|
||||
# $2 the prompt, evidence included
|
||||
# $3 timeout in seconds (default AI_ASSIST_TIMEOUT, else 45)
|
||||
ai_assist_note() {
|
||||
local flag="$1" prompt="$2" ai_timeout="${3:-${AI_ASSIST_TIMEOUT:-45}}"
|
||||
|
||||
[[ "${AI_ENABLED:-false}" == "true" ]] || return 1
|
||||
[[ "${!flag:-false}" == "true" ]] || return 1
|
||||
[[ -n "${MY_ID:-}" ]] || return 1
|
||||
command -v jq >/dev/null 2>&1 || return 1
|
||||
|
||||
local u_var="${MY_ID}_OLLAMA_URL" m_var="${MY_ID}_OLLAMA_MODEL"
|
||||
local url="${!u_var:-}" model="${!m_var:-}"
|
||||
[[ -n "$url" && -n "$model" ]] || return 1
|
||||
|
||||
local body out
|
||||
# temperature 0: this is a description of evidence, and the same evidence should not produce a
|
||||
# different characterisation on a rerun.
|
||||
body=$(jq -nc --arg m "$model" --arg p "$prompt" \
|
||||
'{model:$m, prompt:$p, stream:false, options:{temperature:0}}' 2>/dev/null) || return 1
|
||||
|
||||
out=$(timeout "$ai_timeout" curl -sf --max-time "$ai_timeout" \
|
||||
-H 'Content-Type: application/json' -d "$body" \
|
||||
"${url%/}/api/generate" 2>/dev/null | jq -r '.response // empty' 2>/dev/null)
|
||||
|
||||
# Reasoning models emit a <think> block before the answer. It is not the note — a log line a
|
||||
# human is meant to skim cannot open with several hundred words of the model talking itself
|
||||
# through the arithmetic. Strip to the last close tag; a response with no block is unchanged.
|
||||
[[ "$out" == *"</think>"* ]] && out="${out##*</think>}"
|
||||
out="${out#"${out%%[![:space:]]*}"}"
|
||||
|
||||
[[ -n "$out" ]] || return 1
|
||||
printf '%s\n' "$out"
|
||||
}
|
||||
|
||||
# True if filepath's extension (case-insensitive) matches one of the given extensions.
|
||||
# Usage: has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"
|
||||
has_extension() {
|
||||
|
||||
Reference in New Issue
Block a user