Compare commits
7
Commits
d42b1e2dda
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5fa7750046 | ||
|
|
1838eed855 | ||
|
|
96d8a5e3f0 | ||
|
|
0431e720de | ||
|
|
7f4921de49 | ||
|
|
d4c19baa32 | ||
|
|
d444fd8036 |
Executable → Regular
+70
-8
@@ -457,6 +457,30 @@ NOW=$(date +%s)
|
||||
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
||||
> "$TO_DELETE_FILE"
|
||||
|
||||
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
|
||||
# Same contract as radarr_cleanup.sh: a file must classify for deletion on
|
||||
# LIDARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is removed. Covers the partial
|
||||
# classification failure that is too small to trip the tracked-count floor above. The file is
|
||||
# rebuilt from each run rather than edited, which is what prunes it.
|
||||
LIDARR_ORPHAN_STRIKE_LIMIT="${LIDARR_ORPHAN_STRIKE_LIMIT:-2}"
|
||||
STRIKES_FILE="${LIDARR_ORPHAN_STRIKES_FILE:-$DB_DIR/lidarr_orphan_strikes.tsv}"
|
||||
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
|
||||
touch "$STRIKES_FILE" 2>/dev/null || true
|
||||
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
|
||||
> "$STRIKES_NEW"
|
||||
HELD_COUNT=0
|
||||
HELD_BYTES=0
|
||||
|
||||
orphan_strike_ok() {
|
||||
local path="$1" prev strikes
|
||||
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
|
||||
strikes=$(( ${prev:-0} + 1 ))
|
||||
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
|
||||
(( strikes >= LIDARR_ORPHAN_STRIKE_LIMIT )) && return 0
|
||||
warn " strike $strikes/$LIDARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
|
||||
return 1
|
||||
}
|
||||
|
||||
while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
FILE_CTIME="${FILE_CTIME%%.*}"
|
||||
@@ -493,12 +517,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||
(( ORPHAN_COUNT++ ))
|
||||
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
else
|
||||
log "JUNK: $filepath"
|
||||
(( JUNK_COUNT++ ))
|
||||
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
fi
|
||||
|
||||
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
||||
@@ -506,13 +532,43 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
# 4.3ms), since find already has to stat() every entry anyway to know it's -type f.
|
||||
done < <(find "$LIDARR_MUSIC_ROOT" -type f -printf '%s %C@ %p\n' 2>/dev/null)
|
||||
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||
# Eligible, not classified: a file still serving its strikes is an orphan but is not queued this
|
||||
# run, so it must not appear in the denominator the budget reports against.
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
|
||||
|
||||
# Rebuilt, never edited. Skipped on a dry run: a preview that advanced real counters would make
|
||||
# the next real run delete a run early.
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
||||
# ==============================================================================================
|
||||
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$LIDARR_MAX_DELETE_GB" "Lidarr Cleanup"
|
||||
# A per-run budget, not a veto — see apply_delete_budget() in common.sh. The ceiling still caps
|
||||
# any single run; it just no longer deadlocks on a backlog larger than itself.
|
||||
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
|
||||
|
||||
if [[ "$I_KNOW" == true ]]; then
|
||||
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
|
||||
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
|
||||
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
|
||||
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
|
||||
else
|
||||
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$LIDARR_MAX_DELETE_GB"
|
||||
if [[ -n "$_BUDGET_STUCK" ]]; then
|
||||
error "Single file exceeds the ${LIDARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
|
||||
error " $_BUDGET_STUCK"
|
||||
error "Raise LIDARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
|
||||
notify "Lidarr cleanup stalled on $(hostname) — one file exceeds the ${LIDARR_MAX_DELETE_GB}GB budget" \
|
||||
"Lidarr Cleanup" "warning"
|
||||
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
|
||||
warn "Budget ${LIDARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
|
||||
notify "Lidarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname) — $_BUDGET_DEFERRED_COUNT file(s) deferred" \
|
||||
"Lidarr Cleanup" "normal"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||
# All safety layers passed — delete orphans and junk. Reuses TO_DELETE_FILE from the
|
||||
@@ -521,7 +577,7 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||
done < "$TO_DELETE_FILE"
|
||||
done < "$BUDGET_FILE"
|
||||
|
||||
info "Cleaning up empty folders..."
|
||||
find "$LIDARR_MUSIC_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null
|
||||
@@ -546,6 +602,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (cover art, metadata
|
||||
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${LIDARR_ORPHAN_AGE} days)"
|
||||
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${LIDARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
|
||||
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${LIDARR_MAX_DELETE_GB}GB run budget"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
@@ -554,8 +614,10 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Clean — nothing to remove"
|
||||
else
|
||||
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
||||
notify "Lidarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "warning"
|
||||
# What was actually removed, not what was classified. With strikes and a budget in force those
|
||||
# differ, and reporting the classification as the outcome is the oldest bug shape here.
|
||||
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
|
||||
notify "Lidarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "warning"
|
||||
# Notify Emby to clean missing files — removes ghost entries immediately
|
||||
notify_emby_scan
|
||||
fi
|
||||
|
||||
Executable → Regular
+134
-8
@@ -450,9 +450,46 @@ NOW=$(date +%s)
|
||||
# just delete them directly instead of re-walking and re-classifying every SCAN_ROOTS entry a
|
||||
# second time (2026-07-17) — the size-threshold check below needs to know the total before
|
||||
# deleting anything, not before knowing what to delete.
|
||||
# Carries size and ctime alongside the path now, because the budget pass below has to order by
|
||||
# age and stop at a byte ceiling — neither of which a bare path list can answer.
|
||||
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
||||
> "$TO_DELETE_FILE"
|
||||
|
||||
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
|
||||
# A file must classify for deletion on RADARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is
|
||||
# actually removed. Gate 6 already refuses a run whose tracked count collapsed; this covers the
|
||||
# partial failure underneath that threshold — one root folder failing to enumerate makes its
|
||||
# movies look orphaned while the overall percentage still looks fine, and a transient fault will
|
||||
# not reproduce on the next run.
|
||||
#
|
||||
# The file is REBUILT from this run's classifications rather than edited in place, which is what
|
||||
# prunes it: anything that stopped being an orphan simply is not written again, so a file that
|
||||
# Radarr re-adopts loses its strikes without needing a reset pass to find it.
|
||||
#
|
||||
# Keyed by host path, which is why this could not have worked before 2026-08-26 — wd_state_set
|
||||
# built a regex from the key, and a release tag like [Bluray-1080p] holds the reversed range 1-0,
|
||||
# so every write truncated the store to one line. See common.sh.
|
||||
RADARR_ORPHAN_STRIKE_LIMIT="${RADARR_ORPHAN_STRIKE_LIMIT:-2}"
|
||||
STRIKES_FILE="${RADARR_ORPHAN_STRIKES_FILE:-$DB_DIR/radarr_orphan_strikes.tsv}"
|
||||
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
|
||||
touch "$STRIKES_FILE" 2>/dev/null || true
|
||||
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
|
||||
> "$STRIKES_NEW"
|
||||
HELD_COUNT=0
|
||||
HELD_BYTES=0
|
||||
|
||||
# Records this run's strike for a file and says whether it has served enough of them.
|
||||
# Returns 0 when the file may be deleted, 1 when it is still accruing.
|
||||
orphan_strike_ok() {
|
||||
local path="$1" prev strikes
|
||||
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
|
||||
strikes=$(( ${prev:-0} + 1 ))
|
||||
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
|
||||
(( strikes >= RADARR_ORPHAN_STRIKE_LIMIT )) && return 0
|
||||
warn " strike $strikes/$RADARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
|
||||
return 1
|
||||
}
|
||||
|
||||
while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
FILE_CTIME="${FILE_CTIME%%.*}"
|
||||
@@ -487,12 +524,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||
(( ORPHAN_COUNT++ ))
|
||||
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
else
|
||||
log "JUNK: $filepath"
|
||||
(( JUNK_COUNT++ ))
|
||||
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
fi
|
||||
|
||||
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
||||
@@ -504,13 +543,94 @@ done < <(
|
||||
done | sort -u
|
||||
)
|
||||
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||
# Eligible, not classified. A file still serving its strikes was counted as an orphan above — it
|
||||
# is one — but it is not going to be deleted this run, so it must not appear in the denominator
|
||||
# the budget reports against or the run claims to have skipped work it never queued.
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
|
||||
|
||||
# Rebuilt, never edited: a path absent from this run is absent from the file, so a file Radarr
|
||||
# re-adopts drops its strikes with no reset pass needed. Skipped on a dry run — a preview that
|
||||
# advanced real strike counters would make the next real run delete a run early.
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
||||
# ==============================================================================================
|
||||
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$RADARR_MAX_DELETE_GB" "Radarr Cleanup"
|
||||
# The ceiling is a per-run budget, not a veto. It still means what it always meant — no single run
|
||||
# removes more than RADARR_MAX_DELETE_GB — but a backlog larger than the ceiling now drains over
|
||||
# consecutive nights instead of failing the orchestrator forever on a queue it cannot clear.
|
||||
# ── AI note (AI_ASSIST_CLEANUP) ───────────────────────────────────────────────────────────────
|
||||
# Describes the shape of what was classified. It decides nothing: the eligible set, the budget and
|
||||
# the strikes are all settled above and none of them read this. Switch AI_ASSIST_CLEANUP off and
|
||||
# the run removes exactly the same files — the log just loses a paragraph.
|
||||
#
|
||||
# ctime clustering is the signal worth surfacing. A normal upgrade cycle dribbles in over weeks; a
|
||||
# lump sharing one narrow ctime window with mtimes spread across months is a bulk write-back, which
|
||||
# is what a partnership merge against a partner holding older copies produces. That distinction
|
||||
# took a person an evening on 2026-08-26 and is the whole reason this note exists.
|
||||
if [[ "$ORPHAN_COUNT" -gt 0 ]] && [[ -s "$TO_DELETE_FILE" ]]; then
|
||||
_ai_ev=$(awk -F'\t' '
|
||||
{ n++; bytes += $1
|
||||
c = int($2)
|
||||
if (cmin == 0 || c < cmin) cmin = c
|
||||
if (c > cmax) cmax = c
|
||||
bucket[int(c / 21600)]++ }
|
||||
END {
|
||||
for (b in bucket) if (bucket[b] > top) { top = bucket[b] }
|
||||
printf "files=%d bytes_gb=%.1f ctime_span_hours=%.1f largest_6h_ctime_bucket=%d\n",
|
||||
n, bytes/1073741824, (cmax-cmin)/3600, top
|
||||
}' "$TO_DELETE_FILE")
|
||||
_ai_mt=$(cut -d"$(printf '\t')" -f3 "$TO_DELETE_FILE" | head -8 \
|
||||
| while IFS= read -r p; do [[ -f "$p" ]] && \
|
||||
printf '%s %s\n' "$(stat -c %y "$p" 2>/dev/null | cut -c1-7)" "$(basename "$p")"; done)
|
||||
|
||||
_ai_note=$(ai_assist_note AI_ASSIST_CLEANUP "You are looking at files an automated media-library cleanup has classified for deletion on an Unraid server. They are files on disk that the Radarr database no longer references.
|
||||
|
||||
EVIDENCE
|
||||
$_ai_ev
|
||||
sample (modification month, then path):
|
||||
$_ai_mt
|
||||
|
||||
A normal quality-upgrade cycle produces orphans whose ctimes are spread out over weeks, because each upgrade happens on its own day. A bulk event - a sync or restore writing files back onto this host - produces orphans sharing one narrow ctime window while their modification times stay spread across months, because the copy preserves modification time but resets ctime.
|
||||
|
||||
In no more than three sentences, say which of those two this looks like and name the numbers above that support it. Do not recommend an action. Do not speculate beyond the evidence given.") || _ai_note=""
|
||||
|
||||
if [[ -n "$_ai_note" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR AI note on this classification ━━━"
|
||||
printf '%s\n' "$_ai_note"
|
||||
fi
|
||||
unset _ai_ev _ai_mt
|
||||
fi
|
||||
|
||||
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
|
||||
|
||||
if [[ "$I_KNOW" == true ]]; then
|
||||
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
|
||||
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
|
||||
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
|
||||
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
|
||||
else
|
||||
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$RADARR_MAX_DELETE_GB"
|
||||
|
||||
if [[ -n "$_BUDGET_STUCK" ]]; then
|
||||
# One file larger than the whole budget can never fit, so it would be re-found and
|
||||
# re-deferred every night. Name it rather than loop on it silently.
|
||||
error "Single file exceeds the ${RADARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
|
||||
error " $_BUDGET_STUCK"
|
||||
error "Raise RADARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
|
||||
notify "Radarr cleanup stalled on $(hostname) — one file exceeds the ${RADARR_MAX_DELETE_GB}GB budget" \
|
||||
"Radarr Cleanup" "warning"
|
||||
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
|
||||
warn "Budget ${RADARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
|
||||
warn "Oldest first — the deferred files are the newest and are re-evaluated tomorrow"
|
||||
notify "Radarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname) — $_BUDGET_DEFERRED_COUNT file(s) deferred to the next run" \
|
||||
"Radarr Cleanup" "normal"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||
# Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and
|
||||
@@ -519,7 +639,7 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||
done < "$TO_DELETE_FILE"
|
||||
done < "$BUDGET_FILE"
|
||||
|
||||
info "Cleaning up empty folders..."
|
||||
for host_path in "${SCAN_ROOTS[@]}"; do
|
||||
@@ -545,6 +665,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles,
|
||||
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${RADARR_ORPHAN_AGE} days)"
|
||||
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${RADARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
|
||||
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${RADARR_MAX_DELETE_GB}GB run budget"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
@@ -553,8 +677,10 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Clean — nothing to remove"
|
||||
else
|
||||
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
||||
notify "Radarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
|
||||
# What was actually removed, not what was classified. With a budget in force those differ, and
|
||||
# reporting the classification as the outcome is the oldest bug shape in this codebase.
|
||||
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED classified files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
|
||||
notify "Radarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED classified files ($(format_bytes "$_BUDGET_KEPT_BYTES"))$([[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && echo ", $_BUDGET_DEFERRED_COUNT deferred")" \
|
||||
"Radarr Cleanup" "warning"
|
||||
# Notify Emby to clean missing files — removes ghost entries immediately
|
||||
notify_emby_scan
|
||||
|
||||
Executable → Regular
+70
-8
@@ -440,6 +440,30 @@ NOW=$(date +%s)
|
||||
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
||||
> "$TO_DELETE_FILE"
|
||||
|
||||
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
|
||||
# Same contract as radarr_cleanup.sh: a file must classify for deletion on
|
||||
# SONARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is removed. Covers the partial
|
||||
# classification failure that is too small to trip the tracked-count floor above. The file is
|
||||
# rebuilt from each run rather than edited, which is what prunes it.
|
||||
SONARR_ORPHAN_STRIKE_LIMIT="${SONARR_ORPHAN_STRIKE_LIMIT:-2}"
|
||||
STRIKES_FILE="${SONARR_ORPHAN_STRIKES_FILE:-$DB_DIR/sonarr_orphan_strikes.tsv}"
|
||||
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
|
||||
touch "$STRIKES_FILE" 2>/dev/null || true
|
||||
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
|
||||
> "$STRIKES_NEW"
|
||||
HELD_COUNT=0
|
||||
HELD_BYTES=0
|
||||
|
||||
orphan_strike_ok() {
|
||||
local path="$1" prev strikes
|
||||
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
|
||||
strikes=$(( ${prev:-0} + 1 ))
|
||||
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
|
||||
(( strikes >= SONARR_ORPHAN_STRIKE_LIMIT )) && return 0
|
||||
warn " strike $strikes/$SONARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
|
||||
return 1
|
||||
}
|
||||
|
||||
while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
FILE_CTIME="${FILE_CTIME%%.*}"
|
||||
@@ -474,12 +498,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||
(( ORPHAN_COUNT++ ))
|
||||
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
else
|
||||
log "JUNK: $filepath"
|
||||
(( JUNK_COUNT++ ))
|
||||
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
fi
|
||||
|
||||
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
||||
@@ -491,13 +517,43 @@ done < <(
|
||||
done | sort -u
|
||||
)
|
||||
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||
# Eligible, not classified: a file still serving its strikes is an orphan but is not queued this
|
||||
# run, so it must not appear in the denominator the budget reports against.
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
|
||||
|
||||
# Rebuilt, never edited. Skipped on a dry run: a preview that advanced real counters would make
|
||||
# the next real run delete a run early.
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
||||
# ==============================================================================================
|
||||
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$SONARR_MAX_DELETE_GB" "Sonarr Cleanup"
|
||||
# A per-run budget, not a veto — see apply_delete_budget() in common.sh. The ceiling still caps
|
||||
# any single run; it just no longer deadlocks on a backlog larger than itself.
|
||||
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
|
||||
|
||||
if [[ "$I_KNOW" == true ]]; then
|
||||
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
|
||||
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
|
||||
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
|
||||
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
|
||||
else
|
||||
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$SONARR_MAX_DELETE_GB"
|
||||
if [[ -n "$_BUDGET_STUCK" ]]; then
|
||||
error "Single file exceeds the ${SONARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
|
||||
error " $_BUDGET_STUCK"
|
||||
error "Raise SONARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
|
||||
notify "Sonarr cleanup stalled on $(hostname) — one file exceeds the ${SONARR_MAX_DELETE_GB}GB budget" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
|
||||
warn "Budget ${SONARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
|
||||
notify "Sonarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname) — $_BUDGET_DEFERRED_COUNT file(s) deferred" \
|
||||
"Sonarr Cleanup" "normal"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||
# Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and
|
||||
@@ -506,7 +562,7 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||
done < "$TO_DELETE_FILE"
|
||||
done < "$BUDGET_FILE"
|
||||
|
||||
info "Cleaning up empty folders..."
|
||||
for host_path in "${SCAN_ROOTS[@]}"; do
|
||||
@@ -532,6 +588,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles,
|
||||
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${SONARR_ORPHAN_AGE} days)"
|
||||
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${SONARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
|
||||
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${SONARR_MAX_DELETE_GB}GB run budget"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
@@ -540,8 +600,10 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Clean — nothing to remove"
|
||||
else
|
||||
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
||||
notify "Sonarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
|
||||
# What was actually removed, not what was classified. With strikes and a budget in force those
|
||||
# differ, and reporting the classification as the outcome is the oldest bug shape here.
|
||||
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
|
||||
notify "Sonarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
# Notify Emby to clean missing files — removes ghost entries immediately
|
||||
notify_emby_scan
|
||||
|
||||
@@ -1268,6 +1268,8 @@
|
||||
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||
# protects against API returning partial data on a bad day
|
||||
LIDARR_TRACKED_COUNT_FILE="${DB_DIR}/lidarr_tracked.count"
|
||||
LIDARR_ORPHAN_STRIKES_FILE="${DB_DIR}/lidarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
|
||||
LIDARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify before it is removed
|
||||
LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||
|
||||
# Lidarr tracked-data cache — shared by lidarr_cleanup.sh, lidarr_duplicate_artist_cleanup.sh,
|
||||
@@ -1343,6 +1345,8 @@
|
||||
SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||
# protects against API returning partial data on a bad day
|
||||
SONARR_TRACKED_COUNT_FILE="${DB_DIR}/sonarr_tracked.count"
|
||||
SONARR_ORPHAN_STRIKES_FILE="${DB_DIR}/sonarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
|
||||
SONARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify before it is removed
|
||||
SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||
SONARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveSeries command to
|
||||
# reach "completed" — generous because a large series can sit
|
||||
@@ -1390,6 +1394,12 @@
|
||||
RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||
# protects against API returning partial data on a bad day
|
||||
RADARR_TRACKED_COUNT_FILE="${DB_DIR}/radarr_tracked.count"
|
||||
RADARR_ORPHAN_STRIKES_FILE="${DB_DIR}/radarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
|
||||
RADARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify for deletion before it is
|
||||
# removed. Gate 6 already catches an API returning far too few
|
||||
# tracked files; this catches the partial failure too small to trip
|
||||
# that percentage — one root folder failing to enumerate makes its
|
||||
# movies look orphaned, and a transient one will not repeat.
|
||||
RADARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||
RADARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveMovie command to
|
||||
# reach "completed" — mirrors SONARR_MOVE_POLL_TIMEOUT
|
||||
@@ -1656,6 +1666,10 @@
|
||||
# HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES
|
||||
BACKUP_VERIFY_SAMPLE=10 # random files to check per share
|
||||
BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample
|
||||
BACKUP_VERIFY_MD5_TIMEOUT_MAX=600 # ceiling for one remote checksum. The per-file budget
|
||||
# scales with size (~50MB/s); this caps it. A fixed
|
||||
# connect-sized timeout killed multi-GB checksums and
|
||||
# the empty result was then reported as MISSING.
|
||||
|
||||
# ━━━ SMART Health ━━━
|
||||
# Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*.
|
||||
@@ -2034,7 +2048,10 @@
|
||||
AI_ASSIST_REPORTS=false # tier 1 — digest / coffee report narration
|
||||
AI_ASSIST_WATCHDOG=false # tier 2 — file a finding when a watchdog counter passes its limit (needs AI_REPAIR_ENABLED)
|
||||
AI_ASSIST_DISCOVERY=false # tier 2 — discovery / classification judgement calls
|
||||
AI_ASSIST_CLEANUP=false # tier 2 — orphan and stuck-import triage
|
||||
AI_ASSIST_CLEANUP=false # tier 2 — orphan and stuck-import triage. Describes the shape of a
|
||||
# classification in the log; decides nothing. Off = identical deletions.
|
||||
AI_ASSIST_TIMEOUT=45 # seconds any single assist may take. An assist that can stall a
|
||||
# nightly cleanup is not an assist — it is silently skipped past this.
|
||||
AI_ASSIST_ONBOARD=false # tier 3 — onboarding / settings assistance
|
||||
|
||||
# ━━━ AI Repair ━━━
|
||||
|
||||
Executable → Regular
+46
-10
@@ -122,6 +122,7 @@ source "$SCRIPT_DIR/../load_config.sh"
|
||||
parse_args "$@"
|
||||
|
||||
SSH_TIMEOUT=15
|
||||
BACKUP_VERIFY_MD5_TIMEOUT_MAX="${BACKUP_VERIFY_MD5_TIMEOUT_MAX:-600}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
@@ -228,6 +229,7 @@ TOTAL_CHECKED=0
|
||||
TOTAL_MATCH=0
|
||||
TOTAL_MISMATCH=0
|
||||
TOTAL_MISSING=0
|
||||
TOTAL_UNVERIFIED=0
|
||||
SHARES_WITH_ISSUES=()
|
||||
|
||||
for share in "${VERIFY_SHARES[@]}"; do
|
||||
@@ -265,6 +267,7 @@ for share in "${VERIFY_SHARES[@]}"; do
|
||||
SHARE_MATCH=0
|
||||
SHARE_MISMATCH=0
|
||||
SHARE_MISSING=0
|
||||
SHARE_UNVERIFIED=0
|
||||
|
||||
for local_file in "${SAMPLE_FILES[@]}"; do
|
||||
[[ -z "$local_file" ]] && continue
|
||||
@@ -276,19 +279,49 @@ for share in "${VERIFY_SHARES[@]}"; do
|
||||
continue
|
||||
fi
|
||||
|
||||
# Remote checksum via SSH — timeout protected
|
||||
remote_md5=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
# The path is interpolated into a remote shell command, so it must be escaped for
|
||||
# reuse as one word. A bare '$local_file' inside single quotes breaks on the first
|
||||
# apostrophe — "Frieren - Beyond Journey's End" ended the quote early, md5sum fell
|
||||
# back to reading stdin, and the empty-input hash d41d8cd9... was reported as a
|
||||
# MISMATCH against a file that is byte-identical on the remote.
|
||||
printf -v remote_q '%q' "$local_file"
|
||||
|
||||
# Existence and content are separate questions. Asking them together means a slow
|
||||
# checksum is indistinguishable from an absent file.
|
||||
remote_exists=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o StrictHostKeyChecking=no \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"md5sum '$local_file' 2>/dev/null | awk '{print \$1}'" 2>/dev/null)
|
||||
"test -f $remote_q && echo yes" 2>/dev/null </dev/null)
|
||||
|
||||
(( TOTAL_CHECKED++ ))
|
||||
|
||||
if [[ -z "$remote_md5" ]]; then
|
||||
if [[ "$remote_exists" != "yes" ]]; then
|
||||
warn "$ICON_ERROR MISSING: $(basename "$local_file")"
|
||||
(( SHARE_MISSING++ ))
|
||||
(( TOTAL_MISSING++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# md5sum of a multi-GB file cannot finish inside a connect-sized timeout. Budget by
|
||||
# size — a 5.9GB file needs ~30s and was being killed at 15s, then counted MISSING
|
||||
# even though it was present and correct.
|
||||
local_size=$(stat -c%s "$local_file" 2>/dev/null || echo 0)
|
||||
md5_timeout=$(( local_size / 52428800 + SSH_TIMEOUT ))
|
||||
(( md5_timeout > BACKUP_VERIFY_MD5_TIMEOUT_MAX )) && md5_timeout=$BACKUP_VERIFY_MD5_TIMEOUT_MAX
|
||||
|
||||
remote_md5=$(timeout "$md5_timeout" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o StrictHostKeyChecking=no \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"md5sum $remote_q 2>/dev/null | awk '{print \$1}'" 2>/dev/null </dev/null)
|
||||
|
||||
if [[ -z "$remote_md5" ]]; then
|
||||
# Present but unreadable within budget. Reporting this as a mismatch or a miss
|
||||
# would be a claim the run did not earn.
|
||||
warn "$ICON_WARN UNVERIFIED (checksum timed out after ${md5_timeout}s): $(basename "$local_file")"
|
||||
(( SHARE_UNVERIFIED++ ))
|
||||
(( TOTAL_UNVERIFIED++ ))
|
||||
elif [[ "$local_md5" == "$remote_md5" ]]; then
|
||||
log "MATCH: $(basename "$local_file")"
|
||||
(( SHARE_MATCH++ ))
|
||||
@@ -303,8 +336,8 @@ for share in "${VERIFY_SHARES[@]}"; do
|
||||
done
|
||||
|
||||
# Per-share result — only visible if issues found
|
||||
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 ]]; then
|
||||
warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH"
|
||||
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 || "$SHARE_UNVERIFIED" -gt 0 ]]; then
|
||||
warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH unverified: $SHARE_UNVERIFIED"
|
||||
SHARES_WITH_ISSUES+=("$SHARE_NAME")
|
||||
else
|
||||
echo "$SHARE_NAME — all $SHARE_MATCH files match ✅"
|
||||
@@ -325,10 +358,11 @@ echo "$ICON_VERIFY Checked: $TOTAL_CHECKED files"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
||||
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
|
||||
warn "Missing: $TOTAL_MISSING"
|
||||
[[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
|
||||
if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 || "$TOTAL_UNVERIFIED" -gt 0 ]]; then
|
||||
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
|
||||
warn "Missing: $TOTAL_MISSING"
|
||||
[[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
|
||||
[[ "$TOTAL_UNVERIFIED" -gt 0 ]] && warn "Unverified: $TOTAL_UNVERIFIED (present, checksum timed out)"
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
@@ -337,6 +371,8 @@ elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention: ${SHARES_WITH_ISSUES[*]}"
|
||||
notify "Backup verify FAILED on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" \
|
||||
"Backup Verify" "warning"
|
||||
elif [[ "$TOTAL_UNVERIFIED" -gt 0 ]]; then
|
||||
warn "Status: $TOTAL_MATCH verified, $TOTAL_UNVERIFIED could not be checksummed in time — NOT a clean run"
|
||||
else
|
||||
echo "$ICON_DONE Status: all $TOTAL_CHECKED files match across ${#VERIFY_SHARES[@]} shares ✅"
|
||||
fi
|
||||
|
||||
@@ -1363,6 +1363,15 @@ vv_ai_profiles_script();
|
||||
profile: profile,
|
||||
question: q,
|
||||
history: JSON.stringify(sendable()),
|
||||
// Where the caller is standing, so a question can say "this setting" and mean it. The
|
||||
// backend whitelists it and passes it to the worker, which turns it into a location
|
||||
// line; a page that sets no scope sends '' and the worker omits that line.
|
||||
//
|
||||
// Keep this here. The scope-aware dock in 185abdb sent it from the page's own ask call;
|
||||
// 8eeb4c3 folded that dock into this component and the line did not come with it, so
|
||||
// for the whole of that window Scheduler computed a scope, re-read it at send time, and
|
||||
// the model never saw it. Nothing errored — the answer just arrived ungrounded.
|
||||
scope: (typeof o.scope === 'function' ? o.scope() : (o.scope || '')),
|
||||
kind: (PROFILES[profile].kind && kindEl) ? kindEl.value : '',
|
||||
// A checkbox where the page offers one, otherwise whatever the page decides from the
|
||||
// profile, otherwise on. The Scheduler reasons only when diagnosing: working out what a
|
||||
|
||||
@@ -37,6 +37,12 @@
|
||||
// Step 2: auto-populate + guide + checklist.
|
||||
// master.conf pull (for partner servers) lives in the checklist, not here.
|
||||
|
||||
// The assistant is a bonus on this page, never a dependency: vv_ai_ui_on() is false on a node
|
||||
// with AI off or no model reachable, which is the normal state of the fresh install this wizard
|
||||
// exists to serve. The card simply is not rendered and setup proceeds exactly as before.
|
||||
require_once dirname(__DIR__) . '/include/ai_chat.php';
|
||||
if (vv_ai_ui_on()) vv_ai_chat_assets();
|
||||
|
||||
$detectedHostname = vv_get_hostname();
|
||||
|
||||
// ── Identity already on disk? ────────────────────────────────────────────────────────────────
|
||||
@@ -313,6 +319,21 @@ hr.vv-hr { border: none; border-top: 1px solid #1e1e1e; margin: 22px 0; }
|
||||
<div id="vv-onboard-panel"></div>
|
||||
<div id="vv-done-banner"></div>
|
||||
|
||||
<?php if (vv_ai_ui_on()): ?>
|
||||
<hr class="vv-hr">
|
||||
<div id="vv-su-ai-card">
|
||||
<?php vv_ai_chat_markup('vv-su-ai', [
|
||||
'profile' => 'varaverk',
|
||||
'compact' => true,
|
||||
'title' => 'Setup assistant',
|
||||
'height' => '260px',
|
||||
'tall' => '460px',
|
||||
'empty' => 'Stuck on a step? Ask what it wants and why — "what is the API key for", '
|
||||
. '"why does SSH need the other host", "what does populate actually do".',
|
||||
]); ?>
|
||||
</div>
|
||||
<?php endif; ?>
|
||||
|
||||
<div style="margin-top:18px;text-align:right;">
|
||||
<a href="#" id="vv-exit-link" onclick="vvGoNext(event)"
|
||||
style="font-size:11px;color:#3a3a3a;text-decoration:none;">Skip →</a>
|
||||
@@ -443,6 +464,7 @@ function vvShowStep2(redirect, apiKey) {
|
||||
_vvRedirect = redirect || '?tab=scheduler';
|
||||
document.getElementById('vv-step1').style.display = 'none';
|
||||
document.getElementById('vv-step2').style.display = 'block';
|
||||
vvSetupAiInit(); // card is visible now, so it can measure itself correctly
|
||||
if (apiKey && apiKey.ok) {
|
||||
const btn = document.getElementById('vv-key-btn');
|
||||
const status = document.getElementById('vv-key-status');
|
||||
@@ -676,6 +698,47 @@ function vvStartPhase2Watch(sinceMs) {
|
||||
_vvPartnerPoll = setInterval(check, 6000);
|
||||
}
|
||||
|
||||
// ── Setup assistant ───────────────────────────────────────────────────────────
|
||||
// Scoped to the step the operator is actually stuck on, so "what does this want" resolves without
|
||||
// them naming it. The worker turns a non-empty scope into a line saying what is open in the
|
||||
// WebGUI; the varaverk profile holds no scoped_log capability, so a checklist id never triggers a
|
||||
// log lookup and cannot produce a "log missing" note for something that was never a script.
|
||||
//
|
||||
// Re-read at send time, not captured: the checklist re-polls while the page is open and the step
|
||||
// they are on can change between opening the composer and pressing Ask.
|
||||
//
|
||||
// First not-ok, not-deferred item — deferred means "I have decided to skip this", which is not
|
||||
// where they are stuck. Falls back to the bare page when the list is clean or has not loaded.
|
||||
function vvSetupScope() {
|
||||
const d = _vvLastChecklist;
|
||||
if (!d || !Array.isArray(d.items)) return 'Setup';
|
||||
const stuck = d.items.find(i => !i.ok && !i.deferred);
|
||||
return stuck && stuck.id ? 'Setup/' + stuck.id : 'Setup';
|
||||
}
|
||||
|
||||
// Mounted when Step 2 is revealed, not at parse time. This is the only page where the card starts
|
||||
// inside a display:none block, and a chat that measured itself while hidden would come up wrong
|
||||
// with nothing to correct it — the component carries no resize observer. Idempotent because
|
||||
// vvShowStep2 is reachable more than once.
|
||||
let _vvSetupAiUp = false;
|
||||
function vvSetupAiInit() {
|
||||
if (_vvSetupAiUp) return;
|
||||
if (typeof VvAiChat !== 'function') return;
|
||||
if (!document.getElementById('vv-su-ai-chat')) return; // AI off — card was never rendered
|
||||
_vvSetupAiUp = true;
|
||||
VvAiChat({
|
||||
prefix: 'vv-su-ai',
|
||||
profile: 'varaverk',
|
||||
// Pinned, for the same reason the Partnership card pins it: resuming whatever thread was last
|
||||
// touched anywhere in the UI could land a first-run operator mid-way through someone else's
|
||||
// Scheduler conversation, on a card with no picker to get back from.
|
||||
resumeProfile: 'varaverk',
|
||||
scope: vvSetupScope,
|
||||
scopeLabel: 'Setup',
|
||||
empty: 'Stuck on a step? Ask what it wants and why.',
|
||||
});
|
||||
}
|
||||
|
||||
function vvLoadChecklist() {
|
||||
fetch('/plugins/varaverk/api/checklist.php?_=' + Date.now())
|
||||
.then(r => r.json()).then(d => {
|
||||
|
||||
@@ -2734,6 +2734,100 @@ check_delete_size_threshold() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Applies a per-run delete budget instead of refusing the whole run. Reads "<size>\t<ctime>\t<path>"
|
||||
# from $1, writes the paths that fit inside $3 GB to $2 (oldest ctime first), and reports what was
|
||||
# held back in _BUDGET_*.
|
||||
#
|
||||
# A cap that aborts cannot drain a backlog larger than itself. Every run re-finds the same lump,
|
||||
# exceeds the same ceiling and stops, so the queue is stuck permanently and only a human with
|
||||
# --i-know-what-im-doing can clear it. Observed 2026-08-24..26: a partnership merge restored ten
|
||||
# months of superseded Radarr files in one night, 126GB against a 100GB cap, and the daily
|
||||
# orchestrator then failed three nights running on a queue it could never drain. Budgeting keeps
|
||||
# the ceiling meaning exactly what it meant — no single run removes more than max_gb — while the
|
||||
# backlog clears over consecutive runs with nobody touching it.
|
||||
#
|
||||
# Strict oldest-first, stopping at the first file that does not fit rather than skipping it for a
|
||||
# smaller one further down. Skipping would drain more per run but lets a large file be passed over
|
||||
# indefinitely; stopping guarantees every deferred file is newer than everything already removed,
|
||||
# so nothing can starve.
|
||||
#
|
||||
# One case is deliberately left for a human: a single file larger than the entire budget can never
|
||||
# fit, so _BUDGET_STUCK is set and the caller says so rather than looping forever on it.
|
||||
apply_delete_budget() {
|
||||
local src="$1" dst="$2" max_gb="$3"
|
||||
local max_bytes tab
|
||||
tab=$(printf '\t')
|
||||
max_bytes=$(awk "BEGIN {printf \"%d\", $max_gb * 1073741824}")
|
||||
|
||||
_BUDGET_KEPT_BYTES=0; _BUDGET_KEPT_COUNT=0
|
||||
_BUDGET_DEFERRED_BYTES=0; _BUDGET_DEFERRED_COUNT=0
|
||||
_BUDGET_STUCK=""
|
||||
|
||||
: > "$dst"
|
||||
[[ -s "$src" ]] || return 0
|
||||
|
||||
local out
|
||||
out=$(sort -t"$tab" -k2,2n "$src" | awk -F'\t' -v max="$max_bytes" -v dst="$dst" '
|
||||
{
|
||||
if (stop) { defb += $1; defc++; next }
|
||||
if (used + $1 > max) { stop = 1
|
||||
if (kept == 0) stuck = $3
|
||||
defb += $1; defc++; next }
|
||||
used += $1; kept++
|
||||
print $3 > dst
|
||||
}
|
||||
END { printf "%d\t%d\t%d\t%d\t%s", used+0, kept+0, defb+0, defc+0, stuck }
|
||||
')
|
||||
IFS=$'\t' read -r _BUDGET_KEPT_BYTES _BUDGET_KEPT_COUNT \
|
||||
_BUDGET_DEFERRED_BYTES _BUDGET_DEFERRED_COUNT _BUDGET_STUCK <<< "$out"
|
||||
}
|
||||
|
||||
# Asks the local generation model to characterise a block of evidence, for the one line a human
|
||||
# would otherwise have to derive by hand. Prints the note on stdout and returns 0; returns 1 and
|
||||
# prints nothing whenever anything at all is missing, off, slow or malformed.
|
||||
#
|
||||
# The contract is that failure is indistinguishable from the feature being switched off, because
|
||||
# every caller must behave identically either way. An assist that can block a nightly cleanup is
|
||||
# not an assist — so this never retries, never blocks longer than its timeout, and never returns
|
||||
# a partial answer for a caller to interpret.
|
||||
#
|
||||
# Usage: note=$(ai_assist_note AI_ASSIST_CLEANUP "$prompt") && echo "$note"
|
||||
# $1 name of the AI_ASSIST_* flag governing this caller — passed by name, checked here, so a
|
||||
# caller cannot accidentally run an assist its own toggle says is off
|
||||
# $2 the prompt, evidence included
|
||||
# $3 timeout in seconds (default AI_ASSIST_TIMEOUT, else 45)
|
||||
ai_assist_note() {
|
||||
local flag="$1" prompt="$2" ai_timeout="${3:-${AI_ASSIST_TIMEOUT:-45}}"
|
||||
|
||||
[[ "${AI_ENABLED:-false}" == "true" ]] || return 1
|
||||
[[ "${!flag:-false}" == "true" ]] || return 1
|
||||
[[ -n "${MY_ID:-}" ]] || return 1
|
||||
command -v jq >/dev/null 2>&1 || return 1
|
||||
|
||||
local u_var="${MY_ID}_OLLAMA_URL" m_var="${MY_ID}_OLLAMA_MODEL"
|
||||
local url="${!u_var:-}" model="${!m_var:-}"
|
||||
[[ -n "$url" && -n "$model" ]] || return 1
|
||||
|
||||
local body out
|
||||
# temperature 0: this is a description of evidence, and the same evidence should not produce a
|
||||
# different characterisation on a rerun.
|
||||
body=$(jq -nc --arg m "$model" --arg p "$prompt" \
|
||||
'{model:$m, prompt:$p, stream:false, options:{temperature:0}}' 2>/dev/null) || return 1
|
||||
|
||||
out=$(timeout "$ai_timeout" curl -sf --max-time "$ai_timeout" \
|
||||
-H 'Content-Type: application/json' -d "$body" \
|
||||
"${url%/}/api/generate" 2>/dev/null | jq -r '.response // empty' 2>/dev/null)
|
||||
|
||||
# Reasoning models emit a <think> block before the answer. It is not the note — a log line a
|
||||
# human is meant to skim cannot open with several hundred words of the model talking itself
|
||||
# through the arithmetic. Strip to the last close tag; a response with no block is unchanged.
|
||||
[[ "$out" == *"</think>"* ]] && out="${out##*</think>}"
|
||||
out="${out#"${out%%[![:space:]]*}"}"
|
||||
|
||||
[[ -n "$out" ]] || return 1
|
||||
printf '%s\n' "$out"
|
||||
}
|
||||
|
||||
# True if filepath's extension (case-insensitive) matches one of the given extensions.
|
||||
# Usage: has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"
|
||||
has_extension() {
|
||||
|
||||
Reference in New Issue
Block a user