Compare commits

...
19 Commits
Author SHA1 Message Date
Gmer4Lfe 5fa7750046 Sonarr and Lidarr had the same deadlock and the same missing confirmation as Radarr, so give them the same budget and strikes 2026-08-26 21:20:05 -04:00
Gmer4Lfe 1838eed855 Give AI_ASSIST_CLEANUP the consumer it has never had: it describes the shape of a classification and decides nothing, so switching it off changes no deletion 2026-08-26 18:03:17 -04:00
Gmer4Lfe 96d8a5e3f0 Make a file earn its deletion over consecutive runs, so a partial classification failure too small to trip the tracked-count gate cannot remove anything 2026-08-26 17:58:42 -04:00
Gmer4Lfe 0431e720de A cap that aborts cannot drain a backlog bigger than itself, so make it a per-run budget and let the queue clear over consecutive nights 2026-08-26 17:55:21 -04:00
Gmer4Lfe 7f4921de49 Give the wizard an assistant scoped to the step you are actually stuck on, so a first-run question can say "this" and mean something 2026-08-26 17:31:52 -04:00
Gmer4Lfe d4c19baa32 Folding the Scheduler dock into the shared chat left its scope behind, so the troubleshooter has been diagnosing without the log tail for whatever was open 2026-08-26 17:25:28 -04:00
Gmer4Lfe d444fd8036 A path with an apostrophe broke the remote shell quoting and a big file outran the connect timeout, so intact backups were reported corrupt and absent 2026-08-25 21:30:57 -04:00
Gmer4Lfe d42b1e2dda ffprobe exit 0 with a chapter-track warning is not corruption; guard the delete path against a detector that is wrong at scale 2026-08-25 18:29:43 -04:00
Gmer4Lfe 9492dc4c39 dig names the resolver it could not reach in its error text, so a DNS timeout was scraped as the answer and restarted DDNS over nothing 2026-08-25 18:29:43 -04:00
Gmer4Lfe 172beca3c5 A state key is a file path, not a regex — a release tag like [Bluray-1080p] holds an invalid range, so grep bailed and the tempfile swap wiped every other entry 2026-08-25 18:29:43 -04:00
Gmer4Lfe f0c1289519 Let the file own why the row cap is conditional; the README states that it is 2026-08-25 17:21:20 -04:00
Gmer4Lfe 128172d3a8 The row counts below four columns were the naive division, not what the board does 2026-08-25 17:21:20 -04:00
Gmer4Lfe 820e8325a6 Docs: the row-height cap is conditional now, and the board's narrow-screen behaviour was nowhere in the page help 2026-08-25 17:15:20 -04:00
Gmer4Lfe 74c6a0f5eb Monitor: hold the System card to the Network card's height below eight columns, where the two stop sharing a row 2026-08-25 17:07:18 -04:00
Gmer4Lfe d289a9101c Monitor: cap the System card's Varaverk block so the host figures set the card's height, not the sum of both 2026-08-25 17:01:52 -04:00
Gmer4Lfe 9553ecb16a The two functions the row-overflow hatch added were not in EXPORTS 2026-08-25 17:01:10 -04:00
Gmer4Lfe fb104bf05a Monitor: a rung taller than one screen sizes rows to content, since the page scrolls there anyway and the cap only clipped cards 2026-08-25 16:54:19 -04:00
Gmer4Lfe f4adc31215 Close the last four headers, and say what the destructive tools refuse to do 2026-08-25 16:51:14 -04:00
Gmer4Lfe c6254f2342 The dev checkers documented themselves under headings nothing else uses 2026-08-25 16:45:39 -04:00
23 changed files with 1044 additions and 103 deletions
+120 -8
View File
@@ -273,6 +273,9 @@ touch "$CORRUPTION_SCAN_STATE_FILE"
CORRUPTION_SCAN_STRIKES_FILE="${CORRUPTION_SCAN_STRIKES_FILE:-$DATA_DIR/corruption_scan_strikes.tsv}" CORRUPTION_SCAN_STRIKES_FILE="${CORRUPTION_SCAN_STRIKES_FILE:-$DATA_DIR/corruption_scan_strikes.tsv}"
CORRUPTION_SCAN_STRIKE_LIMIT="${CORRUPTION_SCAN_STRIKE_LIMIT:-2}" CORRUPTION_SCAN_STRIKE_LIMIT="${CORRUPTION_SCAN_STRIKE_LIMIT:-2}"
CORRUPTION_SCAN_MAX_CORRUPT_PCT="${CORRUPTION_SCAN_MAX_CORRUPT_PCT:-10}"
CORRUPTION_SCAN_MAX_CONSECUTIVE="${CORRUPTION_SCAN_MAX_CONSECUTIVE:-15}"
CORRUPTION_SCAN_GUARD_MIN_SCANNED="${CORRUPTION_SCAN_GUARD_MIN_SCANNED:-20}"
mkdir -p "$(dirname "$CORRUPTION_SCAN_STRIKES_FILE")" mkdir -p "$(dirname "$CORRUPTION_SCAN_STRIKES_FILE")"
touch "$CORRUPTION_SCAN_STRIKES_FILE" touch "$CORRUPTION_SCAN_STRIKES_FILE"
@@ -301,6 +304,19 @@ reset_scan_strikes() {
[[ -n "$current" && "$current" != "0" ]] && set_scan_strikes "$1" 0 [[ -n "$current" && "$current" != "0" ]] && set_scan_strikes "$1" 0
} }
# Bails out of the whole run without committing anything. Safe to call at any point before
# the commit phase: strikes are queued in memory until then, so an abort leaves the strike
# file exactly as the previous run left it and deletes nothing.
abort_scan() {
local why="$1"
error "Corruption scan ABORTED — $why"
error "No strikes recorded and nothing remediated this run — the library was not trusted."
[[ -n "${FRESH_CLEAN_TMP:-}" ]] && rm -f "$FRESH_CLEAN_TMP"
notify "Corruption scan aborted on $(hostname) ($MY_ID) — $why. Nothing deleted." \
"Arr Corruption Scan" "warning"
exit 1
}
# Per-arr API shape differences — everything else in the scan/strike/remediate loop below is # Per-arr API shape differences — everything else in the scan/strike/remediate loop below is
# identical between Sonarr and Radarr. # identical between Sonarr and Radarr.
declare -A ARR_FILE_ENDPOINT=( [sonarr]="episodefile" [radarr]="moviefile" ) declare -A ARR_FILE_ENDPOINT=( [sonarr]="episodefile" [radarr]="moviefile" )
@@ -322,6 +338,8 @@ if [[ "$SHOW_STATUS" == true ]]; then
echo "$ICON_GEAR State file: $CORRUPTION_SCAN_STATE_FILE" echo "$ICON_GEAR State file: $CORRUPTION_SCAN_STATE_FILE"
echo "$ICON_GEAR Strike limit: $CORRUPTION_SCAN_STRIKE_LIMIT" echo "$ICON_GEAR Strike limit: $CORRUPTION_SCAN_STRIKE_LIMIT"
echo "$ICON_GEAR Remediate: $REMEDIATE" echo "$ICON_GEAR Remediate: $REMEDIATE"
echo "$ICON_GEAR Corrupt ceiling: ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% of scanned (min ${CORRUPTION_SCAN_GUARD_MIN_SCANNED} scanned)"
echo "$ICON_GEAR Consecutive trip: $CORRUPTION_SCAN_MAX_CONSECUTIVE"
echo "$ICON_GEAR Scan limit: ${SCAN_LIMIT:-unlimited} (per arr)" echo "$ICON_GEAR Scan limit: ${SCAN_LIMIT:-unlimited} (per arr)"
echo "$ICON_GEAR Path filter: ${PATH_FILTER:-none}" echo "$ICON_GEAR Path filter: ${PATH_FILTER:-none}"
echo "━━━━━━━━━━━━━━━━━━━━━━━" echo "━━━━━━━━━━━━━━━━━━━━━━━"
@@ -357,13 +375,36 @@ ffprobe_translate_path() {
# Probes one file. Echoes "clean" or "corrupt:<reason>". Never trusts a truncated/garbled # Probes one file. Echoes "clean" or "corrupt:<reason>". Never trusts a truncated/garbled
# stderr as automatically corrupt — only a real non-empty ffprobe stderr counts. # stderr as automatically corrupt — only a real non-empty ffprobe stderr counts.
probe_file() { probe_file() {
local host_path="$1" container_path output local host_path="$1" container_path output rc
container_path=$(ffprobe_translate_path "$host_path") || { echo "unmapped"; return; } container_path=$(ffprobe_translate_path "$host_path") || { echo "unmapped"; return; }
output=$(docker exec "$FFPROBE_CONTAINER" "$FFPROBE_BIN" -v error "$container_path" 2>&1) output=$(docker exec "$FFPROBE_CONTAINER" "$FFPROBE_BIN" -v error "$container_path" 2>&1)
rc=$?
# docker exec writes its own failures to the same stream ffprobe uses, so a stopped
# container or an unreachable daemon is otherwise indistinguishable from a corrupt
# header. A stopped container exits 1 with a daemon message; a missing binary exits
# 127 — neither is evidence about the file, so both must be caught.
if (( rc >= 125 )) \
|| [[ "$output" == "Error response from daemon:"* \
|| "$output" == "Cannot connect to the Docker daemon"* \
|| "$output" == "error during connect:"* ]]; then
echo "probe_error:${output//$'\n'/ }"
return
fi
if [[ -z "$output" ]]; then if [[ -z "$output" ]]; then
echo "clean" echo "clean"
else elif (( rc != 0 )); then
# ffprobe could not parse the file — EBML header parsing failed, moov atom not found,
# contradictionary STSC and STCO. This is the only class that may be remediated.
echo "corrupt:${output//$'\n'/ }" echo "corrupt:${output//$'\n'/ }"
else
# Exit 0 with stderr output: a recoverable muxing complaint, most commonly
# "Referenced QT chapter track not found", which many recent .mp4 releases emit and
# which says nothing about playability. Equating any stderr with corruption is what
# produced 103 "corrupt" files on 2026-08-23 — 28 of 43 newly scanned Radarr items.
# Reported for visibility, never strike-tracked, never remediated.
echo "suspect:${output//$'\n'/ }"
fi fi
} }
@@ -442,7 +483,7 @@ TOTAL_SCANNED=0
TOTAL_CORRUPT=0 TOTAL_CORRUPT=0
TOTAL_REMEDIATED=0 TOTAL_REMEDIATED=0
TOTAL_REMEDIATE_FAILED=0 TOTAL_REMEDIATE_FAILED=0
declare -A ARR_SCANNED ARR_SKIPPED_CACHED ARR_SKIPPED_UNMAPPED ARR_CORRUPT ARR_STRIKE_HELD ARR_REMEDIATED ARR_REMEDIATE_FAILED declare -A ARR_SCANNED ARR_SKIPPED_CACHED ARR_SKIPPED_UNMAPPED ARR_CORRUPT ARR_SUSPECT ARR_PROBE_ERRORS ARR_STRIKE_HELD ARR_REMEDIATED ARR_REMEDIATE_FAILED
for arr in sonarr radarr; do for arr in sonarr radarr; do
url_var="${arr^^}_URL"; key_var="${arr^^}_API_KEY" url_var="${arr^^}_URL"; key_var="${arr^^}_API_KEY"
@@ -532,6 +573,12 @@ for arr in sonarr radarr; do
STRIKE_HELD=0 STRIKE_HELD=0
REMEDIATED=0 REMEDIATED=0
REMEDIATE_FAILED=0 REMEDIATE_FAILED=0
PROBE_ERRORS=0
SUSPECT_COUNT=0
CONSECUTIVE_BAD=0
QUEUE_PATH=()
QUEUE_STRIKES=()
QUEUE_ITEM=()
FRESH_CLEAN_TMP=$(mktemp) FRESH_CLEAN_TMP=$(mktemp)
@@ -566,23 +613,82 @@ for arr in sonarr radarr; do
continue continue
fi fi
# A docker-level failure is not evidence about the file. Count it, never queue it.
if [[ "$result" == probe_error:* ]]; then
(( PROBE_ERRORS++ ))
(( CONSECUTIVE_BAD++ ))
warn " ? $host_path — probe failed, NOT counted as corrupt: ${result#probe_error:}"
if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then
abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly"
fi
continue
fi
if [[ "$result" == "clean" ]]; then if [[ "$result" == "clean" ]]; then
CONSECUTIVE_BAD=0
reset_scan_strikes "$host_path" reset_scan_strikes "$host_path"
echo -e "${host_path}\t${stamp}" >> "$FRESH_CLEAN_TMP" echo -e "${host_path}\t${stamp}" >> "$FRESH_CLEAN_TMP"
[[ "$ENABLE_LOGGING" == true ]] && echo " $ICON_SUCCESS $host_path" [[ "$ENABLE_LOGGING" == true ]] && echo " $ICON_SUCCESS $host_path"
continue continue
fi fi
# corrupt:<reason> # A successful probe that merely warned. Proves the container is alive, so it clears
# the consecutive-failure tripwire, but it never becomes a strike.
if [[ "$result" == suspect:* ]]; then
CONSECUTIVE_BAD=0
(( SUSPECT_COUNT++ ))
[[ "$ENABLE_LOGGING" == true ]] && warn " ~ $host_path — ffprobe warning (exit 0), NOT corrupt: ${result#suspect:}"
continue
fi
# corrupt:<reason> — queued, NOT committed. Nothing reaches the strike file and nothing
# is deleted until this arr has been fully probed and the guards below have passed. A
# container that dies mid-scan makes every remaining file read as corrupt, and a delete
# cannot be undone — so the destructive half has to wait until the corrupt rate for the
# whole run is known. 2026-08-23: one Jellyfin restart produced 103 false positives.
reason="${result#corrupt:}" reason="${result#corrupt:}"
(( CORRUPT_COUNT++ )) (( CORRUPT_COUNT++ ))
strikes=$(increment_scan_strikes "$host_path") (( CONSECUTIVE_BAD++ ))
prev_strikes=$(get_scan_strikes "$host_path")
prev_strikes="${prev_strikes//[^0-9]/}"
strikes=$(( ${prev_strikes:-0} + 1 ))
QUEUE_PATH+=("$host_path")
QUEUE_STRIKES+=("$strikes")
QUEUE_ITEM+=("$item")
echo " $ICON_ERROR CORRUPT: $host_path (strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT)" echo " $ICON_ERROR CORRUPT: $host_path (strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT)"
[[ "$ENABLE_LOGGING" == true ]] && echo " $reason" [[ "$ENABLE_LOGGING" == true ]] && echo " $reason"
if [[ "$REMEDIATE" != true ]]; then if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then
continue abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly"
fi fi
done < <(echo "$ITEMS" | jq -c '.[]')
# ━━━ False-positive guards — run before anything is committed ━━━
if (( CORRUPT_COUNT > 0 )); then
# The pre-flight check only proves the container was up when the scan started.
# Re-check now: a mid-scan death is exactly what this guard exists to catch.
check_container_health "$FFPROBE_CONTAINER" "${DOCKER_TIMEOUT:-30}" "Arr Corruption Scan"
if (( SCANNED >= CORRUPTION_SCAN_GUARD_MIN_SCANNED )); then
corrupt_pct=$(( CORRUPT_COUNT * 100 / SCANNED ))
if (( corrupt_pct >= CORRUPTION_SCAN_MAX_CORRUPT_PCT )); then
abort_scan "$CORRUPT_COUNT of $SCANNED probed files (${corrupt_pct}%) read as corrupt — at or above the ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% ceiling"
fi
fi
fi
# ━━━ Guards passed — commit strikes, then remediate whatever reached the limit ━━━
for _q in "${!QUEUE_PATH[@]}"; do
host_path="${QUEUE_PATH[$_q]}"
strikes="${QUEUE_STRIKES[$_q]}"
item="${QUEUE_ITEM[$_q]}"
set_scan_strikes "$host_path" "$strikes"
[[ "$REMEDIATE" != true ]] && continue
if (( strikes < CORRUPTION_SCAN_STRIKE_LIMIT )); then if (( strikes < CORRUPTION_SCAN_STRIKE_LIMIT )); then
warn " $host_path — strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT, not yet remediating (needs repeat confirmation)" warn " $host_path — strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT, not yet remediating (needs repeat confirmation)"
@@ -591,6 +697,8 @@ for arr in sonarr radarr; do
fi fi
reset_scan_strikes "$host_path" reset_scan_strikes "$host_path"
file_id=$(echo "$item" | jq -r '.file_id')
parent_id=$(echo "$item" | jq -r '.parent_id')
title=$(echo "$item" | jq -r '.title') title=$(echo "$item" | jq -r '.title')
http_code=$(curl -sf -o /dev/null -w "%{http_code}" -X DELETE \ http_code=$(curl -sf -o /dev/null -w "%{http_code}" -X DELETE \
@@ -625,7 +733,7 @@ for arr in sonarr radarr; do
warn " $title — deleted and verified, but search trigger returned HTTP $search_code" warn " $title — deleted and verified, but search trigger returned HTTP $search_code"
(( REMEDIATE_FAILED++ )) (( REMEDIATE_FAILED++ ))
fi fi
done < <(echo "$ITEMS" | jq -c '.[]') done
merge_clean_state "$FRESH_CLEAN_TMP" merge_clean_state "$FRESH_CLEAN_TMP"
rm -f "$FRESH_CLEAN_TMP" rm -f "$FRESH_CLEAN_TMP"
@@ -634,6 +742,8 @@ for arr in sonarr radarr; do
ARR_SKIPPED_CACHED[$arr]=$SKIPPED_CACHED ARR_SKIPPED_CACHED[$arr]=$SKIPPED_CACHED
ARR_SKIPPED_UNMAPPED[$arr]=$SKIPPED_UNMAPPED ARR_SKIPPED_UNMAPPED[$arr]=$SKIPPED_UNMAPPED
ARR_CORRUPT[$arr]=$CORRUPT_COUNT ARR_CORRUPT[$arr]=$CORRUPT_COUNT
ARR_SUSPECT[$arr]=$SUSPECT_COUNT
ARR_PROBE_ERRORS[$arr]=$PROBE_ERRORS
ARR_STRIKE_HELD[$arr]=$STRIKE_HELD ARR_STRIKE_HELD[$arr]=$STRIKE_HELD
ARR_REMEDIATED[$arr]=$REMEDIATED ARR_REMEDIATED[$arr]=$REMEDIATED
ARR_REMEDIATE_FAILED[$arr]=$REMEDIATE_FAILED ARR_REMEDIATE_FAILED[$arr]=$REMEDIATE_FAILED
@@ -658,6 +768,8 @@ for arr in sonarr radarr; do
echo " $ICON_SUCCESS Skipped (cached): ${ARR_SKIPPED_CACHED[$arr]}" echo " $ICON_SUCCESS Skipped (cached): ${ARR_SKIPPED_CACHED[$arr]}"
echo " $ICON_WARN Skipped (unmapped): ${ARR_SKIPPED_UNMAPPED[$arr]}" echo " $ICON_WARN Skipped (unmapped): ${ARR_SKIPPED_UNMAPPED[$arr]}"
echo " $ICON_ERROR Corrupt found: ${ARR_CORRUPT[$arr]}" echo " $ICON_ERROR Corrupt found: ${ARR_CORRUPT[$arr]}"
echo " $ICON_WARN Warnings (exit 0): ${ARR_SUSPECT[$arr]} (reported, never remediated)"
echo " $ICON_WARN Probe errors: ${ARR_PROBE_ERRORS[$arr]} (not counted as corrupt)"
if [[ "$REMEDIATE" == true ]]; then if [[ "$REMEDIATE" == true ]]; then
echo " $ICON_WARN Held (strikes): ${ARR_STRIKE_HELD[$arr]}" echo " $ICON_WARN Held (strikes): ${ARR_STRIKE_HELD[$arr]}"
echo " $ICON_SUCCESS Remediated: ${ARR_REMEDIATED[$arr]}" echo " $ICON_SUCCESS Remediated: ${ARR_REMEDIATED[$arr]}"
Executable → Regular
+70 -8
View File
@@ -457,6 +457,30 @@ NOW=$(date +%s)
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt" TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
> "$TO_DELETE_FILE" > "$TO_DELETE_FILE"
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
# Same contract as radarr_cleanup.sh: a file must classify for deletion on
# LIDARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is removed. Covers the partial
# classification failure that is too small to trip the tracked-count floor above. The file is
# rebuilt from each run rather than edited, which is what prunes it.
LIDARR_ORPHAN_STRIKE_LIMIT="${LIDARR_ORPHAN_STRIKE_LIMIT:-2}"
STRIKES_FILE="${LIDARR_ORPHAN_STRIKES_FILE:-$DB_DIR/lidarr_orphan_strikes.tsv}"
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
touch "$STRIKES_FILE" 2>/dev/null || true
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
> "$STRIKES_NEW"
HELD_COUNT=0
HELD_BYTES=0
orphan_strike_ok() {
local path="$1" prev strikes
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
strikes=$(( ${prev:-0} + 1 ))
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
(( strikes >= LIDARR_ORPHAN_STRIKE_LIMIT )) && return 0
warn " strike $strikes/$LIDARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
return 1
}
while read -r FILE_SIZE FILE_CTIME filepath; do while read -r FILE_SIZE FILE_CTIME filepath; do
[[ -z "$filepath" ]] && continue [[ -z "$filepath" ]] && continue
FILE_CTIME="${FILE_CTIME%%.*}" FILE_CTIME="${FILE_CTIME%%.*}"
@@ -493,12 +517,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
warn "$ICON_TRASH ORPHAN: $filepath" warn "$ICON_TRASH ORPHAN: $filepath"
(( ORPHAN_COUNT++ )) (( ORPHAN_COUNT++ ))
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE )) ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
echo "$filepath" >> "$TO_DELETE_FILE" if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
else else
log "JUNK: $filepath" log "JUNK: $filepath"
(( JUNK_COUNT++ )) (( JUNK_COUNT++ ))
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE )) JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
echo "$filepath" >> "$TO_DELETE_FILE" if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
fi fi
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a # -printf gets size + mtime directly from find's own stat() during the walk, instead of a
@@ -506,13 +532,43 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
# 4.3ms), since find already has to stat() every entry anyway to know it's -type f. # 4.3ms), since find already has to stat() every entry anyway to know it's -type f.
done < <(find "$LIDARR_MUSIC_ROOT" -type f -printf '%s %C@ %p\n' 2>/dev/null) done < <(find "$LIDARR_MUSIC_ROOT" -type f -printf '%s %C@ %p\n' 2>/dev/null)
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES )) # Eligible, not classified: a file still serving its strikes is an orphan but is not queued this
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) # run, so it must not appear in the denominator the budget reports against.
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
# Rebuilt, never edited. Skipped on a dry run: a preview that advanced real counters would make
# the next real run delete a run early.
if [[ "$DRY_RUN" == false ]]; then
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
fi
# ============================================================================================== # ==============================================================================================
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━ # ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
# ============================================================================================== # ==============================================================================================
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$LIDARR_MAX_DELETE_GB" "Lidarr Cleanup" # A per-run budget, not a veto — see apply_delete_budget() in common.sh. The ceiling still caps
# any single run; it just no longer deadlocks on a backlog larger than itself.
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
if [[ "$I_KNOW" == true ]]; then
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
else
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$LIDARR_MAX_DELETE_GB"
if [[ -n "$_BUDGET_STUCK" ]]; then
error "Single file exceeds the ${LIDARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
error " $_BUDGET_STUCK"
error "Raise LIDARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
notify "Lidarr cleanup stalled on $(hostname) — one file exceeds the ${LIDARR_MAX_DELETE_GB}GB budget" \
"Lidarr Cleanup" "warning"
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
warn "Budget ${LIDARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
notify "Lidarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname)$_BUDGET_DEFERRED_COUNT file(s) deferred" \
"Lidarr Cleanup" "normal"
fi
fi
# ── Execute Deletions ───────────────────────────────────────────────────────────────────────── # ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
# All safety layers passed — delete orphans and junk. Reuses TO_DELETE_FILE from the # All safety layers passed — delete orphans and junk. Reuses TO_DELETE_FILE from the
@@ -521,7 +577,7 @@ if [[ "$DRY_RUN" == false ]]; then
while IFS= read -r filepath; do while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue [[ -z "$filepath" ]] && continue
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath" rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
done < "$TO_DELETE_FILE" done < "$BUDGET_FILE"
info "Cleaning up empty folders..." info "Cleaning up empty folders..."
find "$LIDARR_MUSIC_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null find "$LIDARR_MUSIC_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null
@@ -546,6 +602,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (cover art, metadata
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)" echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)" echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${LIDARR_ORPHAN_AGE} days)" echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${LIDARR_ORPHAN_AGE} days)"
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${LIDARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${LIDARR_MAX_DELETE_GB}GB run budget"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo "" echo ""
@@ -554,8 +614,10 @@ if [[ "$DRY_RUN" == true ]]; then
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
echo "$ICON_DONE Clean — nothing to remove" echo "$ICON_DONE Clean — nothing to remove"
else else
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" # What was actually removed, not what was classified. With strikes and a budget in force those
notify "Lidarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "warning" # differ, and reporting the classification as the outcome is the oldest bug shape here.
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
notify "Lidarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "warning"
# Notify Emby to clean missing files — removes ghost entries immediately # Notify Emby to clean missing files — removes ghost entries immediately
notify_emby_scan notify_emby_scan
fi fi
Executable → Regular
+134 -8
View File
@@ -450,9 +450,46 @@ NOW=$(date +%s)
# just delete them directly instead of re-walking and re-classifying every SCAN_ROOTS entry a # just delete them directly instead of re-walking and re-classifying every SCAN_ROOTS entry a
# second time (2026-07-17) — the size-threshold check below needs to know the total before # second time (2026-07-17) — the size-threshold check below needs to know the total before
# deleting anything, not before knowing what to delete. # deleting anything, not before knowing what to delete.
# Carries size and ctime alongside the path now, because the budget pass below has to order by
# age and stop at a byte ceiling — neither of which a bare path list can answer.
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt" TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
> "$TO_DELETE_FILE" > "$TO_DELETE_FILE"
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
# A file must classify for deletion on RADARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is
# actually removed. Gate 6 already refuses a run whose tracked count collapsed; this covers the
# partial failure underneath that threshold — one root folder failing to enumerate makes its
# movies look orphaned while the overall percentage still looks fine, and a transient fault will
# not reproduce on the next run.
#
# The file is REBUILT from this run's classifications rather than edited in place, which is what
# prunes it: anything that stopped being an orphan simply is not written again, so a file that
# Radarr re-adopts loses its strikes without needing a reset pass to find it.
#
# Keyed by host path, which is why this could not have worked before 2026-08-26 — wd_state_set
# built a regex from the key, and a release tag like [Bluray-1080p] holds the reversed range 1-0,
# so every write truncated the store to one line. See common.sh.
RADARR_ORPHAN_STRIKE_LIMIT="${RADARR_ORPHAN_STRIKE_LIMIT:-2}"
STRIKES_FILE="${RADARR_ORPHAN_STRIKES_FILE:-$DB_DIR/radarr_orphan_strikes.tsv}"
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
touch "$STRIKES_FILE" 2>/dev/null || true
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
> "$STRIKES_NEW"
HELD_COUNT=0
HELD_BYTES=0
# Records this run's strike for a file and says whether it has served enough of them.
# Returns 0 when the file may be deleted, 1 when it is still accruing.
orphan_strike_ok() {
local path="$1" prev strikes
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
strikes=$(( ${prev:-0} + 1 ))
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
(( strikes >= RADARR_ORPHAN_STRIKE_LIMIT )) && return 0
warn " strike $strikes/$RADARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
return 1
}
while read -r FILE_SIZE FILE_CTIME filepath; do while read -r FILE_SIZE FILE_CTIME filepath; do
[[ -z "$filepath" ]] && continue [[ -z "$filepath" ]] && continue
FILE_CTIME="${FILE_CTIME%%.*}" FILE_CTIME="${FILE_CTIME%%.*}"
@@ -487,12 +524,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
warn "$ICON_TRASH ORPHAN: $filepath" warn "$ICON_TRASH ORPHAN: $filepath"
(( ORPHAN_COUNT++ )) (( ORPHAN_COUNT++ ))
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE )) ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
echo "$filepath" >> "$TO_DELETE_FILE" if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
else else
log "JUNK: $filepath" log "JUNK: $filepath"
(( JUNK_COUNT++ )) (( JUNK_COUNT++ ))
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE )) JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
echo "$filepath" >> "$TO_DELETE_FILE" if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
fi fi
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a # -printf gets size + mtime directly from find's own stat() during the walk, instead of a
@@ -504,13 +543,94 @@ done < <(
done | sort -u done | sort -u
) )
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES )) # Eligible, not classified. A file still serving its strikes was counted as an orphan above — it
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) # is one — but it is not going to be deleted this run, so it must not appear in the denominator
# the budget reports against or the run claims to have skipped work it never queued.
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
# Rebuilt, never edited: a path absent from this run is absent from the file, so a file Radarr
# re-adopts drops its strikes with no reset pass needed. Skipped on a dry run — a preview that
# advanced real strike counters would make the next real run delete a run early.
if [[ "$DRY_RUN" == false ]]; then
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
fi
# ============================================================================================== # ==============================================================================================
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━ # ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
# ============================================================================================== # ==============================================================================================
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$RADARR_MAX_DELETE_GB" "Radarr Cleanup" # The ceiling is a per-run budget, not a veto. It still means what it always meant — no single run
# removes more than RADARR_MAX_DELETE_GB — but a backlog larger than the ceiling now drains over
# consecutive nights instead of failing the orchestrator forever on a queue it cannot clear.
# ── AI note (AI_ASSIST_CLEANUP) ───────────────────────────────────────────────────────────────
# Describes the shape of what was classified. It decides nothing: the eligible set, the budget and
# the strikes are all settled above and none of them read this. Switch AI_ASSIST_CLEANUP off and
# the run removes exactly the same files — the log just loses a paragraph.
#
# ctime clustering is the signal worth surfacing. A normal upgrade cycle dribbles in over weeks; a
# lump sharing one narrow ctime window with mtimes spread across months is a bulk write-back, which
# is what a partnership merge against a partner holding older copies produces. That distinction
# took a person an evening on 2026-08-26 and is the whole reason this note exists.
if [[ "$ORPHAN_COUNT" -gt 0 ]] && [[ -s "$TO_DELETE_FILE" ]]; then
_ai_ev=$(awk -F'\t' '
{ n++; bytes += $1
c = int($2)
if (cmin == 0 || c < cmin) cmin = c
if (c > cmax) cmax = c
bucket[int(c / 21600)]++ }
END {
for (b in bucket) if (bucket[b] > top) { top = bucket[b] }
printf "files=%d bytes_gb=%.1f ctime_span_hours=%.1f largest_6h_ctime_bucket=%d\n",
n, bytes/1073741824, (cmax-cmin)/3600, top
}' "$TO_DELETE_FILE")
_ai_mt=$(cut -d"$(printf '\t')" -f3 "$TO_DELETE_FILE" | head -8 \
| while IFS= read -r p; do [[ -f "$p" ]] && \
printf '%s %s\n' "$(stat -c %y "$p" 2>/dev/null | cut -c1-7)" "$(basename "$p")"; done)
_ai_note=$(ai_assist_note AI_ASSIST_CLEANUP "You are looking at files an automated media-library cleanup has classified for deletion on an Unraid server. They are files on disk that the Radarr database no longer references.
EVIDENCE
$_ai_ev
sample (modification month, then path):
$_ai_mt
A normal quality-upgrade cycle produces orphans whose ctimes are spread out over weeks, because each upgrade happens on its own day. A bulk event - a sync or restore writing files back onto this host - produces orphans sharing one narrow ctime window while their modification times stay spread across months, because the copy preserves modification time but resets ctime.
In no more than three sentences, say which of those two this looks like and name the numbers above that support it. Do not recommend an action. Do not speculate beyond the evidence given.") || _ai_note=""
if [[ -n "$_ai_note" ]]; then
echo ""
echo "━━━ $ICON_GEAR AI note on this classification ━━━"
printf '%s\n' "$_ai_note"
fi
unset _ai_ev _ai_mt
fi
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
if [[ "$I_KNOW" == true ]]; then
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
else
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$RADARR_MAX_DELETE_GB"
if [[ -n "$_BUDGET_STUCK" ]]; then
# One file larger than the whole budget can never fit, so it would be re-found and
# re-deferred every night. Name it rather than loop on it silently.
error "Single file exceeds the ${RADARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
error " $_BUDGET_STUCK"
error "Raise RADARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
notify "Radarr cleanup stalled on $(hostname) — one file exceeds the ${RADARR_MAX_DELETE_GB}GB budget" \
"Radarr Cleanup" "warning"
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
warn "Budget ${RADARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
warn "Oldest first — the deferred files are the newest and are re-evaluated tomorrow"
notify "Radarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname)$_BUDGET_DEFERRED_COUNT file(s) deferred to the next run" \
"Radarr Cleanup" "normal"
fi
fi
# ── Execute Deletions ───────────────────────────────────────────────────────────────────────── # ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
# Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and # Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and
@@ -519,7 +639,7 @@ if [[ "$DRY_RUN" == false ]]; then
while IFS= read -r filepath; do while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue [[ -z "$filepath" ]] && continue
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath" rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
done < "$TO_DELETE_FILE" done < "$BUDGET_FILE"
info "Cleaning up empty folders..." info "Cleaning up empty folders..."
for host_path in "${SCAN_ROOTS[@]}"; do for host_path in "${SCAN_ROOTS[@]}"; do
@@ -545,6 +665,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles,
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)" echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)" echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${RADARR_ORPHAN_AGE} days)" echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${RADARR_ORPHAN_AGE} days)"
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${RADARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${RADARR_MAX_DELETE_GB}GB run budget"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo "" echo ""
@@ -553,8 +677,10 @@ if [[ "$DRY_RUN" == true ]]; then
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
echo "$ICON_DONE Clean — nothing to remove" echo "$ICON_DONE Clean — nothing to remove"
else else
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" # What was actually removed, not what was classified. With a budget in force those differ, and
notify "Radarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \ # reporting the classification as the outcome is the oldest bug shape in this codebase.
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED classified files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
notify "Radarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED classified files ($(format_bytes "$_BUDGET_KEPT_BYTES"))$([[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && echo ", $_BUDGET_DEFERRED_COUNT deferred")" \
"Radarr Cleanup" "warning" "Radarr Cleanup" "warning"
# Notify Emby to clean missing files — removes ghost entries immediately # Notify Emby to clean missing files — removes ghost entries immediately
notify_emby_scan notify_emby_scan
Executable → Regular
+70 -8
View File
@@ -440,6 +440,30 @@ NOW=$(date +%s)
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt" TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
> "$TO_DELETE_FILE" > "$TO_DELETE_FILE"
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
# Same contract as radarr_cleanup.sh: a file must classify for deletion on
# SONARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is removed. Covers the partial
# classification failure that is too small to trip the tracked-count floor above. The file is
# rebuilt from each run rather than edited, which is what prunes it.
SONARR_ORPHAN_STRIKE_LIMIT="${SONARR_ORPHAN_STRIKE_LIMIT:-2}"
STRIKES_FILE="${SONARR_ORPHAN_STRIKES_FILE:-$DB_DIR/sonarr_orphan_strikes.tsv}"
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
touch "$STRIKES_FILE" 2>/dev/null || true
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
> "$STRIKES_NEW"
HELD_COUNT=0
HELD_BYTES=0
orphan_strike_ok() {
local path="$1" prev strikes
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
strikes=$(( ${prev:-0} + 1 ))
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
(( strikes >= SONARR_ORPHAN_STRIKE_LIMIT )) && return 0
warn " strike $strikes/$SONARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
return 1
}
while read -r FILE_SIZE FILE_CTIME filepath; do while read -r FILE_SIZE FILE_CTIME filepath; do
[[ -z "$filepath" ]] && continue [[ -z "$filepath" ]] && continue
FILE_CTIME="${FILE_CTIME%%.*}" FILE_CTIME="${FILE_CTIME%%.*}"
@@ -474,12 +498,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
warn "$ICON_TRASH ORPHAN: $filepath" warn "$ICON_TRASH ORPHAN: $filepath"
(( ORPHAN_COUNT++ )) (( ORPHAN_COUNT++ ))
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE )) ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
echo "$filepath" >> "$TO_DELETE_FILE" if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
else else
log "JUNK: $filepath" log "JUNK: $filepath"
(( JUNK_COUNT++ )) (( JUNK_COUNT++ ))
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE )) JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
echo "$filepath" >> "$TO_DELETE_FILE" if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
fi fi
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a # -printf gets size + mtime directly from find's own stat() during the walk, instead of a
@@ -491,13 +517,43 @@ done < <(
done | sort -u done | sort -u
) )
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES )) # Eligible, not classified: a file still serving its strikes is an orphan but is not queued this
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) # run, so it must not appear in the denominator the budget reports against.
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
# Rebuilt, never edited. Skipped on a dry run: a preview that advanced real counters would make
# the next real run delete a run early.
if [[ "$DRY_RUN" == false ]]; then
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
fi
# ============================================================================================== # ==============================================================================================
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━ # ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
# ============================================================================================== # ==============================================================================================
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$SONARR_MAX_DELETE_GB" "Sonarr Cleanup" # A per-run budget, not a veto — see apply_delete_budget() in common.sh. The ceiling still caps
# any single run; it just no longer deadlocks on a backlog larger than itself.
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
if [[ "$I_KNOW" == true ]]; then
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
else
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$SONARR_MAX_DELETE_GB"
if [[ -n "$_BUDGET_STUCK" ]]; then
error "Single file exceeds the ${SONARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
error " $_BUDGET_STUCK"
error "Raise SONARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
notify "Sonarr cleanup stalled on $(hostname) — one file exceeds the ${SONARR_MAX_DELETE_GB}GB budget" \
"Sonarr Cleanup" "warning"
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
warn "Budget ${SONARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
notify "Sonarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname)$_BUDGET_DEFERRED_COUNT file(s) deferred" \
"Sonarr Cleanup" "normal"
fi
fi
# ── Execute Deletions ───────────────────────────────────────────────────────────────────────── # ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
# Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and # Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and
@@ -506,7 +562,7 @@ if [[ "$DRY_RUN" == false ]]; then
while IFS= read -r filepath; do while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue [[ -z "$filepath" ]] && continue
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath" rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
done < "$TO_DELETE_FILE" done < "$BUDGET_FILE"
info "Cleaning up empty folders..." info "Cleaning up empty folders..."
for host_path in "${SCAN_ROOTS[@]}"; do for host_path in "${SCAN_ROOTS[@]}"; do
@@ -532,6 +588,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles,
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)" echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)" echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${SONARR_ORPHAN_AGE} days)" echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${SONARR_ORPHAN_AGE} days)"
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${SONARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${SONARR_MAX_DELETE_GB}GB run budget"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo "" echo ""
@@ -540,8 +600,10 @@ if [[ "$DRY_RUN" == true ]]; then
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
echo "$ICON_DONE Clean — nothing to remove" echo "$ICON_DONE Clean — nothing to remove"
else else
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" # What was actually removed, not what was classified. With strikes and a budget in force those
notify "Sonarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \ # differ, and reporting the classification as the outcome is the oldest bug shape here.
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
notify "Sonarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
"Sonarr Cleanup" "warning" "Sonarr Cleanup" "warning"
# Notify Emby to clean missing files — removes ghost entries immediately # Notify Emby to clean missing files — removes ghost entries immediately
notify_emby_scan notify_emby_scan
+27 -1
View File
@@ -1268,6 +1268,8 @@
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day # protects against API returning partial data on a bad day
LIDARR_TRACKED_COUNT_FILE="${DB_DIR}/lidarr_tracked.count" LIDARR_TRACKED_COUNT_FILE="${DB_DIR}/lidarr_tracked.count"
LIDARR_ORPHAN_STRIKES_FILE="${DB_DIR}/lidarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
LIDARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify before it is removed
LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
# Lidarr tracked-data cache — shared by lidarr_cleanup.sh, lidarr_duplicate_artist_cleanup.sh, # Lidarr tracked-data cache — shared by lidarr_cleanup.sh, lidarr_duplicate_artist_cleanup.sh,
@@ -1343,6 +1345,8 @@
SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day # protects against API returning partial data on a bad day
SONARR_TRACKED_COUNT_FILE="${DB_DIR}/sonarr_tracked.count" SONARR_TRACKED_COUNT_FILE="${DB_DIR}/sonarr_tracked.count"
SONARR_ORPHAN_STRIKES_FILE="${DB_DIR}/sonarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
SONARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify before it is removed
SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
SONARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveSeries command to SONARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveSeries command to
# reach "completed" — generous because a large series can sit # reach "completed" — generous because a large series can sit
@@ -1355,6 +1359,15 @@
# against a one-off ffprobe hiccup (mid-write file, NFS blip) # against a one-off ffprobe hiccup (mid-write file, NFS blip)
# triggering an unnecessary delete. Resets to 0 the moment a # triggering an unnecessary delete. Resets to 0 the moment a
# file probes clean again. # file probes clean again.
CORRUPTION_SCAN_MAX_CORRUPT_PCT=10 # abort the run, committing nothing, if this share of
# newly-scanned files reads as corrupt. A healthy library sits
# near zero; a high rate means the detector is wrong, not the
# library. Only counts ffprobe exit != 0.
CORRUPTION_SCAN_MAX_CONSECUTIVE=15 # abort after this many files in a row fail to probe
# cleanly — catches the ffprobe container dying mid-scan,
# which the pre-flight health check cannot see.
CORRUPTION_SCAN_GUARD_MIN_SCANNED=20 # below this many newly-scanned files the percentage
# ceiling is not applied — too small a sample to judge.
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
SONARR_PROTECTED_PATTERNS=( SONARR_PROTECTED_PATTERNS=(
# Subtitles # Subtitles
@@ -1381,6 +1394,12 @@
RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day # protects against API returning partial data on a bad day
RADARR_TRACKED_COUNT_FILE="${DB_DIR}/radarr_tracked.count" RADARR_TRACKED_COUNT_FILE="${DB_DIR}/radarr_tracked.count"
RADARR_ORPHAN_STRIKES_FILE="${DB_DIR}/radarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
RADARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify for deletion before it is
# removed. Gate 6 already catches an API returning far too few
# tracked files; this catches the partial failure too small to trip
# that percentage — one root folder failing to enumerate makes its
# movies look orphaned, and a transient one will not repeat.
RADARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan RADARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
RADARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveMovie command to RADARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveMovie command to
# reach "completed" — mirrors SONARR_MOVE_POLL_TIMEOUT # reach "completed" — mirrors SONARR_MOVE_POLL_TIMEOUT
@@ -1647,6 +1666,10 @@
# HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES # HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES
BACKUP_VERIFY_SAMPLE=10 # random files to check per share BACKUP_VERIFY_SAMPLE=10 # random files to check per share
BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample
BACKUP_VERIFY_MD5_TIMEOUT_MAX=600 # ceiling for one remote checksum. The per-file budget
# scales with size (~50MB/s); this caps it. A fixed
# connect-sized timeout killed multi-GB checksums and
# the empty result was then reported as MISSING.
# ━━━ SMART Health ━━━ # ━━━ SMART Health ━━━
# Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*. # Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*.
@@ -2025,7 +2048,10 @@
AI_ASSIST_REPORTS=false # tier 1 — digest / coffee report narration AI_ASSIST_REPORTS=false # tier 1 — digest / coffee report narration
AI_ASSIST_WATCHDOG=false # tier 2 — file a finding when a watchdog counter passes its limit (needs AI_REPAIR_ENABLED) AI_ASSIST_WATCHDOG=false # tier 2 — file a finding when a watchdog counter passes its limit (needs AI_REPAIR_ENABLED)
AI_ASSIST_DISCOVERY=false # tier 2 — discovery / classification judgement calls AI_ASSIST_DISCOVERY=false # tier 2 — discovery / classification judgement calls
AI_ASSIST_CLEANUP=false # tier 2 — orphan and stuck-import triage AI_ASSIST_CLEANUP=false # tier 2 — orphan and stuck-import triage. Describes the shape of a
# classification in the log; decides nothing. Off = identical deletions.
AI_ASSIST_TIMEOUT=45 # seconds any single assist may take. An assist that can stall a
# nightly cleanup is not an assist — it is silently skipped past this.
AI_ASSIST_ONBOARD=false # tier 3 — onboarding / settings assistance AI_ASSIST_ONBOARD=false # tier 3 — onboarding / settings assistance
# ━━━ AI Repair ━━━ # ━━━ AI Repair ━━━
+55
View File
@@ -28,6 +28,42 @@
# pointing at a directory the data is not in is worse than not having started. # pointing at a directory the data is not in is worse than not having started.
# #
# ============================================================================================== # ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Two halves, in order: move the files, then rewrite the conf keys that point at them. Doing it
# the other way round would leave every path variable naming a location nothing had reached yet,
# and any script that ran in between would create the old layout again underneath the new one.
#
# Idempotent. A path already under DATA_DIR is left alone, so a re-run after a partial migration
# finishes the job rather than moving things twice or failing on what is already done.
#
# One-time by intent, not by a marker file. There is no "already migrated" flag — the check is
# whether each individual path is already where it belongs, which is also what makes an
# interrupted run safe to repeat.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Existing keys are rewritten, which is why conf_upgrade cannot do this.
# conf_upgrade adds keys the template has and the installation does not, and never rewrites a
# value the operator already holds — correct for it, and exactly why it is the wrong tool here.
# STATE_DIR, BANDWIDTH_LOG, AI_INDEX_DB and two dozen more are existing keys whose values must
# change, or they would go on naming the old layout forever while the new directory variables
# sat beside them unused.
#
# Move, never copy-and-hope.
# The data being relocated is the only copy — statistics, histories, the AI index, arr caches.
# Everything is moved and the source is gone afterwards, so there is no second location that
# might still be written to by something that missed the change.
#
# The conf rewrite is the last thing, and the riskiest thing.
# Until it happens the installation still works from the old layout. That ordering means an
# abort partway through leaves a system that runs, rather than one whose paths point at
# nothing.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS # OPERATIONAL SAFEGUARDS
# ============================================================================================== # ==============================================================================================
# #
@@ -49,6 +85,25 @@
# check costs nothing and the failure is silent otherwise. # check costs nothing and the failure is silent otherwise.
# #
# ============================================================================================== # ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# This script reads conf to find the old locations and rewrites conf to record the new ones. It
# is the one script here whose purpose is to change these values rather than obey them.
#
# Read to locate what moves
# STATE_DIR, BANDWIDTH_LOG, AI_INDEX_DB, AI_MEMORY_FILE, AI_TOKEN_DB, ARR_CLEANUP_STATS,
# ARR_SYNC_BLOCKLIST, CORRUPTION_SCAN_STATE_FILE, LIDARR_CACHE_FILE, ZFS_REPORT_LOG and the
# rest of the per-script path keys — roughly two dozen in total.
#
# Written as the new roots
# DATA_DIR and the directories beneath it: DB_DIR, STATE_DIR, AI_DATA_DIR,
# CACHE_BACKUP_DIR, ARR_CACHE_BACKUP_DIR, CONF_CACHE_BACKUP_DIR, LOG_ARCHIVE_DIR.
#
# Every rewritten value is expressed as ${DB_DIR}/… rather than an absolute path, so a later
# storage-mode migration moves them again by changing one variable.
#
# ==============================================================================================
# RUNTIME MODES # RUNTIME MODES
# ============================================================================================== # ==============================================================================================
# #
+37 -1
View File
@@ -42,7 +42,43 @@
# my-Foo.xml routinely holds a container called something else. Matching on the filename # my-Foo.xml routinely holds a container called something else. Matching on the filename
# silently pushes the wrong template, or nothing at all. # silently pushes the wrong template, or nothing at all.
# #
# USAGE # OPERATIONAL SAFEGUARDS
# Only in NORMAL state. FALLBACK_STATE_FILE is read before anything is pushed or removed, and
# any other state refuses the action. A push during a live failover would deploy a second copy
# of a container the partner is currently running on our behalf; a remove would delete the one
# doing the covering.
#
# --status is exempt from that gate, because it only reports. Refusing to answer "what is
# deployed over there" during a failover would withhold the information precisely when it is
# most wanted.
#
# Every deploy is verified stopped, and a container that will not stay stopped is an error
# rather than a warning — see DESIGN PRINCIPLES. A second live instance against the same data
# is the failure this whole script exists inside.
#
# Push and remove are explicit modes with no default. Running the script with no flag does
# nothing; neither action can be reached by accident, and neither is a side effect of editing
# the tier list.
#
# --dry-run works in every mode and touches nothing on either host — no container is built,
# started, stopped or removed, and no template is written or deleted.
#
# Remove deletes the container's appdata on the partner as well. That is deliberate and is the
# most destructive thing here; the NORMAL-state gate above is what keeps it away from a
# partner that is mid-handback.
#
# CONFIGURATION
# master.conf
# FALLBACK_<HOST>_TIER1..N the covered container names — what --push deploys and --status
# reports on. This script reads that list; it never edits it.
#
# host*.conf
# FALLBACK_STATE_FILE overrides where fallback.sh's state is read from. Defaults to
# STATE_DIR/fallback_state.db. A missing file reads as NORMAL,
# which is the correct default on a host where fallback has never
# run.
#
# RUNTIME MODES
# coverage_deploy.sh --push deploy every covered container onto the partner (stopped) # coverage_deploy.sh --push deploy every covered container onto the partner (stopped)
# coverage_deploy.sh --remove stop, remove, and delete the pushed template on the partner # coverage_deploy.sh --remove stop, remove, and delete the pushed template on the partner
# coverage_deploy.sh --status report, per covered container, whether it exists there # coverage_deploy.sh --status report, per covered container, whether it exists there
Executable → Regular
+46 -10
View File
@@ -122,6 +122,7 @@ source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@" parse_args "$@"
SSH_TIMEOUT=15 SSH_TIMEOUT=15
BACKUP_VERIFY_MD5_TIMEOUT_MAX="${BACKUP_VERIFY_MD5_TIMEOUT_MAX:-600}"
# ============================================================================================== # ==============================================================================================
# ━━━ Setup ━━━ # ━━━ Setup ━━━
@@ -228,6 +229,7 @@ TOTAL_CHECKED=0
TOTAL_MATCH=0 TOTAL_MATCH=0
TOTAL_MISMATCH=0 TOTAL_MISMATCH=0
TOTAL_MISSING=0 TOTAL_MISSING=0
TOTAL_UNVERIFIED=0
SHARES_WITH_ISSUES=() SHARES_WITH_ISSUES=()
for share in "${VERIFY_SHARES[@]}"; do for share in "${VERIFY_SHARES[@]}"; do
@@ -265,6 +267,7 @@ for share in "${VERIFY_SHARES[@]}"; do
SHARE_MATCH=0 SHARE_MATCH=0
SHARE_MISMATCH=0 SHARE_MISMATCH=0
SHARE_MISSING=0 SHARE_MISSING=0
SHARE_UNVERIFIED=0
for local_file in "${SAMPLE_FILES[@]}"; do for local_file in "${SAMPLE_FILES[@]}"; do
[[ -z "$local_file" ]] && continue [[ -z "$local_file" ]] && continue
@@ -276,19 +279,49 @@ for share in "${VERIFY_SHARES[@]}"; do
continue continue
fi fi
# Remote checksum via SSH — timeout protected # The path is interpolated into a remote shell command, so it must be escaped for
remote_md5=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ # reuse as one word. A bare '$local_file' inside single quotes breaks on the first
# apostrophe — "Frieren - Beyond Journey's End" ended the quote early, md5sum fell
# back to reading stdin, and the empty-input hash d41d8cd9... was reported as a
# MISMATCH against a file that is byte-identical on the remote.
printf -v remote_q '%q' "$local_file"
# Existence and content are separate questions. Asking them together means a slow
# checksum is indistinguishable from an absent file.
remote_exists=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" \ -o ConnectTimeout="$SSH_TIMEOUT" \
-o StrictHostKeyChecking=no \ -o StrictHostKeyChecking=no \
root@"$REMOTE_SERVER" \ root@"$REMOTE_SERVER" \
"md5sum '$local_file' 2>/dev/null | awk '{print \$1}'" 2>/dev/null) "test -f $remote_q && echo yes" 2>/dev/null </dev/null)
(( TOTAL_CHECKED++ )) (( TOTAL_CHECKED++ ))
if [[ -z "$remote_md5" ]]; then if [[ "$remote_exists" != "yes" ]]; then
warn "$ICON_ERROR MISSING: $(basename "$local_file")" warn "$ICON_ERROR MISSING: $(basename "$local_file")"
(( SHARE_MISSING++ )) (( SHARE_MISSING++ ))
(( TOTAL_MISSING++ )) (( TOTAL_MISSING++ ))
continue
fi
# md5sum of a multi-GB file cannot finish inside a connect-sized timeout. Budget by
# size — a 5.9GB file needs ~30s and was being killed at 15s, then counted MISSING
# even though it was present and correct.
local_size=$(stat -c%s "$local_file" 2>/dev/null || echo 0)
md5_timeout=$(( local_size / 52428800 + SSH_TIMEOUT ))
(( md5_timeout > BACKUP_VERIFY_MD5_TIMEOUT_MAX )) && md5_timeout=$BACKUP_VERIFY_MD5_TIMEOUT_MAX
remote_md5=$(timeout "$md5_timeout" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" \
-o StrictHostKeyChecking=no \
root@"$REMOTE_SERVER" \
"md5sum $remote_q 2>/dev/null | awk '{print \$1}'" 2>/dev/null </dev/null)
if [[ -z "$remote_md5" ]]; then
# Present but unreadable within budget. Reporting this as a mismatch or a miss
# would be a claim the run did not earn.
warn "$ICON_WARN UNVERIFIED (checksum timed out after ${md5_timeout}s): $(basename "$local_file")"
(( SHARE_UNVERIFIED++ ))
(( TOTAL_UNVERIFIED++ ))
elif [[ "$local_md5" == "$remote_md5" ]]; then elif [[ "$local_md5" == "$remote_md5" ]]; then
log "MATCH: $(basename "$local_file")" log "MATCH: $(basename "$local_file")"
(( SHARE_MATCH++ )) (( SHARE_MATCH++ ))
@@ -303,8 +336,8 @@ for share in "${VERIFY_SHARES[@]}"; do
done done
# Per-share result — only visible if issues found # Per-share result — only visible if issues found
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 ]]; then if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 || "$SHARE_UNVERIFIED" -gt 0 ]]; then
warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH" warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH unverified: $SHARE_UNVERIFIED"
SHARES_WITH_ISSUES+=("$SHARE_NAME") SHARES_WITH_ISSUES+=("$SHARE_NAME")
else else
echo "$SHARE_NAME — all $SHARE_MATCH files match ✅" echo "$SHARE_NAME — all $SHARE_MATCH files match ✅"
@@ -325,10 +358,11 @@ echo "$ICON_VERIFY Checked: $TOTAL_CHECKED files"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo "" echo ""
if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 || "$TOTAL_UNVERIFIED" -gt 0 ]]; then
echo "$ICON_SUCCESS Match: $TOTAL_MATCH" echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
warn "Missing: $TOTAL_MISSING" warn "Missing: $TOTAL_MISSING"
[[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH" [[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
[[ "$TOTAL_UNVERIFIED" -gt 0 ]] && warn "Unverified: $TOTAL_UNVERIFIED (present, checksum timed out)"
fi fi
if [[ "$DRY_RUN" == true ]]; then if [[ "$DRY_RUN" == true ]]; then
@@ -337,6 +371,8 @@ elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention: ${SHARES_WITH_ISSUES[*]}" echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention: ${SHARES_WITH_ISSUES[*]}"
notify "Backup verify FAILED on $(hostname)$REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" \ notify "Backup verify FAILED on $(hostname)$REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" \
"Backup Verify" "warning" "Backup Verify" "warning"
elif [[ "$TOTAL_UNVERIFIED" -gt 0 ]]; then
warn "Status: $TOTAL_MATCH verified, $TOTAL_UNVERIFIED could not be checksummed in time — NOT a clean run"
else else
echo "$ICON_DONE Status: all $TOTAL_CHECKED files match across ${#VERIFY_SHARES[@]} shares ✅" echo "$ICON_DONE Status: all $TOTAL_CHECKED files match across ${#VERIFY_SHARES[@]} shares ✅"
fi fi
+36
View File
@@ -20,6 +20,42 @@
# decision to notify is the exit code rather than this script parsing the text it just printed. # decision to notify is the exit code rather than this script parsing the text it just printed.
# #
# ============================================================================================== # ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Silence is the normal output.
# A report that always says something is a report nobody reads. A perfect week prints nothing
# and notifies nothing, so anything that does appear in the Sunday report is worth the glance.
#
# The exit code is the decision, not the text.
# uptime_probe.php --report exits 1 when it has something to say and 0 when it does not. This
# script never parses the output it just printed to work out whether to notify — a report whose
# wording changed would otherwise silently stop notifying.
#
# It reads; it never probes.
# The measurements are already taken, once a minute, by Tools/uptime_probe.sh. Re-probing at
# report time would describe Sunday morning rather than the week being reported on.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Read-only. Reads the stored history and prints; records nothing, and cannot alter the data it
# is reporting on.
#
# UPTIME_PROBE_ENABLED gates the whole run — with the probe off there is no history worth
# reporting, and this says nothing rather than reporting an empty week as a perfect one.
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# uptime_report.sh the weekly read. Silent when every domain was 100%.
#
# Called from COFFEE_REPORT_SCRIPTS; takes no arguments and has no other mode. For live figures
# or a per-domain table, use Tools/uptime_probe.sh --status.
#
# ==============================================================================================
# CONFIGURATION # CONFIGURATION
# ============================================================================================== # ==============================================================================================
# #
+5 -4
View File
@@ -110,10 +110,11 @@ naming every card on the Monitor grid, its width and its order, and generates th
from it — the column ladder, the span clamps at each width, the row-height cap, and the from it — the column ladder, the span clamps at each width, the row-height cap, and the
compensation when a conditional card is absent. Column counts are 8/4/2/1 and spans are 1/2/4/8, compensation when a conditional card is absent. Column counts are 8/4/2/1 and spans are 1/2/4/8,
which is what lets the board re-cut itself at any width with no holes and no hand-placed card. which is what lets the board re-cut itself at any width with no holes and no hand-placed card.
Breakpoints are arithmetic over `VV_MON_CARD_FLOOR`, never chosen by eye. `pages/monitor.php` Breakpoints are arithmetic over `VV_MON_CARD_FLOOR`, never chosen by eye, and the row-height cap
carries the card bodies and nothing about where they go; moving a card is moving a line in that is conditional on the same arithmetic — it applies only where a rung fits one screen.
array. The page cross-checks the declaration against the cards that actually rendered and says so `pages/monitor.php` carries the card bodies and nothing about where they go; moving a card is
in the browser if they disagree. moving a line in that array. The page cross-checks the declaration against the cards that
actually rendered and says so in the browser if they disagree.
**Caching.** Several endpoints serve from `$VV_CACHE_DIR` (`/tmp/varaverk/api`, tmpfs) rather than hitting live **Caching.** Several endpoints serve from `$VV_CACHE_DIR` (`/tmp/varaverk/api`, tmpfs) rather than hitting live
APIs on every page view, refreshed by `Tools/api_cache_writer.sh`. `?live=1` bypasses the APIs on every page view, refreshed by `Tools/api_cache_writer.sh`. `?live=1` bypasses the
+28 -4
View File
@@ -8,10 +8,34 @@
// Reads ai_log_fixtures.txt and this host's /var/log/syslog*. Files nothing, writes nothing, // Reads ai_log_fixtures.txt and this host's /var/log/syslog*. Files nothing, writes nothing,
// and calls no part of the sweep beyond vv_ai_syslog_findings() on lines it supplies itself. // and calls no part of the sweep beyond vv_ai_syslog_findings() on lines it supplies itself.
// //
// EXIT // DESIGN PRINCIPLES
// 0 when every fixture is recognised as written. Precision findings are reported but never // Only recall can fail the run.
// fail the run: what a real syslog contains is a fact about the machine, not about the // A missed fixture is a fact about the patterns and is always a defect. A precision hit is
// patterns, and a genuinely failing disk should not turn this into a red test. // a fact about this machine — a genuinely failing disk should not turn this red, and if it
// did, the honest fix would be to stop having a failing disk rather than to edit a pattern.
//
// Precision is replayed against real history, not a sample.
// The patterns that cause damage are the ones matching ordinary operation, and ordinary
// operation is exactly what a hand-written fixture file never contains. Only the machine's
// own syslog can show what a pattern fires on when nothing is wrong.
//
// The sweep is never invoked, only its matcher.
// vv_ai_syslog_findings() is called on lines this file supplies. Running the real sweep
// would file findings, and a test that has to be cleaned up afterwards stops being run.
//
// OPERATIONAL SAFEGUARDS
// Read-only. Reads ai_log_fixtures.txt and this host's /var/log/syslog*; files no finding,
// writes no store, and touches no conf beyond the enable flag.
//
// Exit 0 when every fixture is recognised as written. Precision findings are reported but
// never fail the run — see DESIGN PRINCIPLES.
//
// RUNTIME MODES
// php ai_log_check.php both checks
// php ai_log_check.php --recall fixtures only
// php ai_log_check.php --precision replay this host's syslog history only
//
// Not scheduled, and deliberately so. Run it after touching VV_AI_SYSLOG_PATTERNS.
// ═══════════════════════════════════════════════════════════════════════════════════════════════ // ═══════════════════════════════════════════════════════════════════════════════════════════════
require_once dirname(__DIR__) . '/include/ai_repair.php'; require_once dirname(__DIR__) . '/include/ai_repair.php';
+41
View File
@@ -15,6 +15,47 @@
# #
# Run it after touching VV_AI_SYSLOG_PATTERNS. Nothing here writes: no findings are filed, no # Run it after touching VV_AI_SYSLOG_PATTERNS. Nothing here writes: no findings are filed, no
# conf is read for anything but the enable flag, and the sweep is never invoked. # conf is read for anything but the enable flag, and the sweep is never invoked.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# A wrapper. The work is in ai_log_check.php, next to the sweep's own matcher — the patterns and
# vv_ai_syslog_findings() live in include/ai_repair.php, and a bash reimplementation of the
# matching would be a second set of regexes to keep in step with the first.
#
# Flags are forwarded verbatim; nothing is interpreted here.
#
# Not scheduled and in no orchestrator. This is a development check that runs when the patterns
# change, not on a timer — nothing on the running system depends on it.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Two checks, because the failure modes are opposite.
# Recall catches a pattern that stopped matching; precision catches one that matches too much.
# A single test would catch one and silently permit the other, and the second is the one that
# fills the findings store with noise until the operator stops reading it.
#
# Precision is measured against this machine's real history.
# A hand-written fixture file can show that a pattern matches what it should. Only a real
# syslog can show what it also matches when nothing is wrong.
#
# Only recall fails the run.
# What a real syslog contains is a fact about the machine, not about the patterns. A genuinely
# failing disk should not turn this red.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Read-only. No finding is filed, no store is written, and the repair sweep itself is never run
# — only its matcher, on lines this check supplies.
#
# Safe to run on a live host at any time, including one that is currently faulting. It observes
# the syslog it replays and changes nothing about it.
#
# ============================================================================================== # ==============================================================================================
# RUNTIME MODES # RUNTIME MODES
# ============================================================================================== # ==============================================================================================
+32 -9
View File
@@ -13,16 +13,39 @@
// php Tools/conf_widget_check.php assertions, then the live summary // php Tools/conf_widget_check.php assertions, then the live summary
// php Tools/conf_widget_check.php --list every live field and its inferred control // php Tools/conf_widget_check.php --list every live field and its inferred control
// //
// WHY IT ASSERTS AGAINST SNIPPETS AND NOT THE LIVE CONF // DESIGN PRINCIPLES
// The live conf is the thing being described, so it cannot also be the thing that proves the // Assertions run against snippets, never against the live conf.
// description right — an inference rule that silently stopped matching would keep passing as // The live conf is the thing being described, so it cannot also be the thing that proves
// the conf drifted to suit it. The snippets are frozen copies of each convention as written, // the description right — an inference rule that silently stopped matching would keep
// so a rule change that breaks one shows up here rather than as a wrong control on a page. // passing as the conf drifted to suit it. The snippets are frozen copies of each
// convention as written, so a rule change that breaks one shows up here rather than as a
// wrong control on a page.
// //
// WHAT AN INFERENCE IS NOT // An inference is a drawing decision, not a validation.
// Consistent with confform.php, none of this validates. A number field carrying min and max is // Consistent with confform.php, none of this validates. A number field carrying min and
// a courtesy to whoever is typing, not a promise the value is sensible — the consuming script // max is a courtesy to whoever is typing, not a promise the value is sensible — the
// still owns that question. // consuming script still owns that question.
//
// The live pass reports, it does not assert.
// What this host's master.conf infers to is a description of that file, not a verdict on
// it. Turning the live summary into pass/fail would make an unusual but legitimate
// setting look like a defect.
//
// OPERATIONAL SAFEGUARDS
// Read-only. Parses conf and reports; writes no conf, no store and no page.
//
// Exits non-zero only when a snippet assertion fails, so it can gate a commit without a real
// conf's contents ever being able to break the build.
//
// Never renders. It reports which control would be drawn; the drawing stays in confform.php,
// so this cannot disagree with the page by construction.
//
// RUNTIME MODES
// php Tools/conf_widget_check.php assertions, then the live summary
// php Tools/conf_widget_check.php --list every live field and its inferred control
//
// Hand-run. Not scheduled and in no orchestrator — run it after touching _vv_conf_widget(),
// after adding a conf convention, or when a setting draws as the wrong control.
// //
// DEPENDS ON // DEPENDS ON
// include/confform.php _vv_conf_parse_field_range(), vv_conf_key_is_secret() // include/confform.php _vv_conf_parse_field_range(), vv_conf_key_is_secret()
+40 -22
View File
@@ -5,31 +5,49 @@
// to reach it. Generated so the assistant can answer "how do I change X" with a path through // to reach it. Generated so the assistant can answer "how do I change X" with a path through
// the pages instead of an instruction to open master.conf. // the pages instead of an instruction to open master.conf.
// //
// WHY THE ASSISTANT NEEDS THIS AT ALL
// The retrieval index reads git-tracked files. PHP body markup is not indexed and would be
// useless if it were — a page is a pile of divs, not a description of itself — so the assistant
// has never had any way to know the UI exists. It could name a conf key and nothing more.
// pages/readme/*.md is the one directory the chunker classifies as kind='ui', which is why the
// output lands there and not in docs/.
//
// WHY IT IS GENERATED
// A hand-written map is a second description of the pages, and the moment a card moves it
// starts lying with total confidence — which is worse than saying nothing, because the
// assistant will repeat it. Everything here is derived from the same registries the pages
// themselves are built from: VV_SCRIPT_CONF_SECTIONS for what the Scheduler shows per script,
// VV_UI_SECTION_SURFACES for the pages that show sections by subject, and the conf files for
// the settings and their controls.
//
// OPERATIONAL MODEL // OPERATIONAL MODEL
// Hand-run, and re-run after adding a conf section, a script mapping or a settings surface. // Reads the section registries and the conf files, resolves each setting to the page and card
// Writes exactly one file and nothing else. // that renders it, and writes the whole map in one pass. Nothing is merged with what is
// already there — the output is derived entirely from the registries, so a stale entry cannot
// survive a rebuild.
// //
// php Tools/ui_map_build.php write the map // DESIGN PRINCIPLES
// php Tools/ui_map_build.php --check report what it would change, write nothing // The assistant cannot see the UI any other way.
// The retrieval index reads git-tracked files. PHP body markup is not indexed and would be
// useless if it were — a page is a pile of divs, not a description of itself — so the
// assistant has never had any way to know the UI exists. It could name a conf key and
// nothing more. pages/readme/*.md is the one directory the chunker classifies as
// kind='ui', which is why the output lands there and not in docs/.
// //
// Only sections that are genuinely reachable are listed. A section no page renders is reported // Generated, because a hand-written map lies with confidence.
// at the end as unreachable rather than silently omitted — a setting with no route is a real // A second description of the pages starts being wrong the moment a card moves, and that
// finding, and the map is the only place that would notice. // is worse than saying nothing, because the assistant will repeat it. Everything here is
// derived from the same registries the pages themselves are built from:
// VV_SCRIPT_CONF_SECTIONS for what the Scheduler shows per script, VV_UI_SECTION_SURFACES
// for the pages that show sections by subject, and the conf files for the settings and
// their controls.
//
// An unreachable section is reported, never dropped.
// A section no page renders is listed at the end rather than silently omitted. A setting
// with no route through the UI is a real finding, and this map is the only thing that
// would ever notice.
//
// OPERATIONAL SAFEGUARDS
// Writes exactly one file, pages/readme/ui-map.md, and nothing else. No conf is modified, no
// page is touched, and the registries it reads are only read.
//
// --check reports what would change and writes nothing, so the map can be verified current in
// a commit without regenerating it.
//
// Generated output only. Nothing hand-edited belongs in ui-map.md — an edit there is lost on
// the next run, which is the correct behaviour for a derived file and the reason the header
// says so.
//
// RUNTIME MODES
// php Tools/ui_map_build.php write the map
// php Tools/ui_map_build.php --check report what it would change, write nothing
//
// Hand-run. Re-run after adding a conf section, a script mapping or a settings surface.
// //
// DEPENDS ON // DEPENDS ON
// include/confform.php the section registries, the parser, and the inferred controls // include/confform.php the section registries, the parser, and the inferred controls
+28 -4
View File
@@ -235,6 +235,27 @@ body.vv-fullscreen #displaybox { padding-left: 1rem !important; padding-top: .5r
/* Hide scrollbars on any nested scrollable div inside monitor cards */ /* Hide scrollbars on any nested scrollable div inside monitor cards */
#vv-monitor .vv-card div::-webkit-scrollbar { display: none; } #vv-monitor .vv-card div::-webkit-scrollbar { display: none; }
/* System card, held to the Network card's height.
At eight columns these two share a row (system 1 + ups 1 + cpu 2 + memory 2 + network 2), so
the grid already stretches them to the same height and nothing here is needed. Below eight the
board re-cuts into rows of four: System lands in row 1 with UPS and CPU, Network in row 2 with
Memory, and once rows are sized to content (vv_mon_rung_overflows) the two are free to differ.
System is the taller — it carries a host block and a Varaverk block — so it is the one capped.
222px is Network measured rather than guessed: 2 border + 24 padding + 25 h3 + 61 header block
+ 110 canvas. The canvas is a fixed 110px, which is what makes the number worth writing down —
most of Network's height cannot drift. What can is the IP list (LAN/EXT/TS), roughly 18px a row,
so a host resolving fewer of the three leaves System slightly tall against it.
Two ids on the selectors, to outrank #vv-monitor .vv-card from the auto-rows hatch above. That
hatch lifts overflow off every card so content-sized rows can work; this card is the one place
the clamps have to go back on, or a max-height with overflow:visible would draw straight
through the card's own border. */
@media (max-width: 1383px), (max-height: 700px) {
#vv-monitor #vv-system { max-height: 222px; overflow: hidden; }
#vv-monitor #vv-system > div { overflow-y: auto; min-height: 0; }
}
/* Dynamic row heights capped per screen tier — rows size to content, never exceed the cap. /* Dynamic row heights capped per screen tier — rows size to content, never exceed the cap.
minmax(0, Xpx): track is content-driven but capped; align-items:stretch makes all cards minmax(0, Xpx): track is content-driven but capped; align-items:stretch makes all cards
in a row fill the track, so short cards (Pools, Watchdog) match tall ones (Array). in a row fill the track, so short cards (Pools, Watchdog) match tall ones (Array).
@@ -264,10 +285,13 @@ body.vv-fullscreen #displaybox { padding-left: 1rem !important; padding-top: .5r
worked, while landscape (851x393) missed it and inherited the four-row cap — 58px cards with worked, while landscape (851x393) missed it and inherited the four-row cap — 58px cards with
overflow:hidden and scrollbars disabled. Nothing was visible and nothing said so. overflow:hidden and scrollbars disabled. Nothing was visible and nothing said so.
The width half is the two-column rung from monitor_board.php (below 696px). It is written as a Height only now. The width half was a literal 695 that had to be kept in step with
literal here because a media query cannot read a custom property; if VV_MON_CARD_FLOOR changes, VV_MON_CARD_FLOOR by hand; monitor_board.php emits these same three rules for every rung whose
this number and the AI row's two below are the only three places that have to follow it. */ row count exceeds VV_MON_ROWS_PER_SCREEN — see vv_mon_rung_overflows() — which covers that band
@media (max-width: 695px), (max-height: 700px) { and the four-column one above it from the arithmetic instead. What is left here is the case the
ladder cannot see: a window wide enough for a rung that does fit a screen, on a screen too short
to give those rows a usable height. */
@media (max-height: 700px) {
#vv-monitor { grid-auto-rows: auto; } #vv-monitor { grid-auto-rows: auto; }
#vv-monitor .vv-card { overflow: visible; } #vv-monitor .vv-card { overflow: visible; }
#vv-monitor .vv-card > div { overflow-y: visible; min-height: auto; } #vv-monitor .vv-card > div { overflow-y: visible; min-height: auto; }
+9
View File
@@ -1363,6 +1363,15 @@ vv_ai_profiles_script();
profile: profile, profile: profile,
question: q, question: q,
history: JSON.stringify(sendable()), history: JSON.stringify(sendable()),
// Where the caller is standing, so a question can say "this setting" and mean it. The
// backend whitelists it and passes it to the worker, which turns it into a location
// line; a page that sets no scope sends '' and the worker omits that line.
//
// Keep this here. The scope-aware dock in 185abdb sent it from the page's own ask call;
// 8eeb4c3 folded that dock into this component and the line did not come with it, so
// for the whole of that window Scheduler computed a scope, re-read it at send time, and
// the model never saw it. Nothing errored — the answer just arrived ungrounded.
scope: (typeof o.scope === 'function' ? o.scope() : (o.scope || '')),
kind: (PROFILES[profile].kind && kindEl) ? kindEl.value : '', kind: (PROFILES[profile].kind && kindEl) ? kindEl.value : '',
// A checkbox where the page offers one, otherwise whatever the page decides from the // A checkbox where the page offers one, otherwise whatever the page decides from the
// profile, otherwise on. The Scheduler reasons only when diagnosing: working out what a // profile, otherwise on. The Scheduler reasons only when diagnosing: working out what a
+44 -8
View File
@@ -13,9 +13,14 @@
// Powers of two, all the way down. // Powers of two, all the way down.
// Column counts are 8, 4, 2, 1 and every span is 1, 2, 4 or 8. That pairing is what makes // Column counts are 8, 4, 2, 1 and every span is 1, 2, 4 or 8. That pairing is what makes
// the board tile with no holes at every width without a single hand-placed card: 32 span // the board tile with no holes at every width without a single hand-placed card: 32 span
// units divide into 8 columns as 4 rows, into 4 as 8 rows, into 2 as 16, into 1 as 32. // units divide into 8 columns as 4 rows and into 4 as 8. A rung of 6 or 10 columns does
// A rung of 6 or 10 columns does not divide a span of 4, which is why the previous ladder // not divide a span of 4, which is why the previous ladder needed a per-breakpoint
// needed a per-breakpoint override for every wide card and still left holes. // override for every wide card and still left holes.
//
// Below four columns the arithmetic stops being a division. Spans clamp to the column
// count, so a span-4 card contributes 2 at two columns and 1 at one, and the board is 13
// rows and then 18 rather than the 16 and 32 a naive division predicts. It still tiles
// without holes, for a different reason: a clamped card fills its row outright.
// //
// Breakpoints are arithmetic, not taste. // Breakpoints are arithmetic, not taste.
// A column count is viable exactly when the cards still fit: // A column count is viable exactly when the cards still fit:
@@ -50,9 +55,15 @@
// — the one array every other function here reads. Change the layout by editing // — the one array every other function here reads. Change the layout by editing
// that declaration, never by editing what follows. // that declaration, never by editing what follows.
// Geometry vv_mon_rung_min(), vv_mon_rung_rows(), vv_mon_rung_rowdiv(), // Geometry vv_mon_rung_min(), vv_mon_rung_rows(), vv_mon_rung_rowdiv(),
// vv_mon_absent_class() // vv_mon_rung_overflows(), vv_mon_absent_class()
// — derive the column ladder and each card's span from the declaration. // — derive the column ladder and each card's span from the declaration.
// Emitters vv_mon_board_css(), vv_mon_absorb_css(), vv_mon_board_js() // vv_mon_rung_overflows() is the one that asks whether a rung is already taller
// than one screen, which is when the row-height cap stops helping and starts
// clipping.
// Emitters vv_mon_board_css(), vv_mon_absorb_css(), vv_mon_autorows_css(),
// vv_mon_board_js()
// — vv_mon_autorows_css() is the hatch for an overflowing rung: rows sized to
// content, and the overflow clamps lifted off the cards so they can use it.
// Validation vv_mon_board_check() // Validation vv_mon_board_check()
// — the only one that reports rather than renders; see OPERATIONAL SAFEGUARDS. // — the only one that reports rather than renders; see OPERATIONAL SAFEGUARDS.
// ═══════════════════════════════════════════════════════════════════════════════════════════════ // ═══════════════════════════════════════════════════════════════════════════════════════════════
@@ -74,9 +85,13 @@ const VV_MON_PAD = 20; // horizontal padding the Unraid page wrapper takes off
// The column ladder, widest first. // The column ladder, widest first.
const VV_MON_RUNGS = [8, 4, 2, 1]; const VV_MON_RUNGS = [8, 4, 2, 1];
// Cap on how many rows may share one screen height. The row-height cap divides the viewport by // Cap on how many rows may share one screen height — and the line between two behaviours, not one
// this, so a card can never grow past a quarter of the screen and push the rest off it. Rungs // rule. A rung with no more rows than this fits a screen: the row-height cap divides the viewport
// with fewer rows than this divide by their own row count instead and fill the screen. // by the rung's own row count, or by this when the rung has more, so the board fills the screen
// and no card grows past its share. A rung with MORE rows than this cannot fit however its rows
// are sized, so the cap is dropped there and rows size to content instead — see
// vv_mon_rung_overflows(). Capping a board that scrolls regardless buys nothing and clips every
// card to pay for it. This board is 4 rows at eight columns and 8 at four, so both cases are live.
const VV_MON_ROWS_PER_SCREEN = 4; const VV_MON_ROWS_PER_SCREEN = 4;
// ── The board ───────────────────────────────────────────────────────────────────────────────── // ── The board ─────────────────────────────────────────────────────────────────────────────────
@@ -164,6 +179,25 @@ function vv_mon_rung_rowdiv(int $cols): int {
return min(vv_mon_rung_rows($cols), VV_MON_ROWS_PER_SCREEN); return min(vv_mon_rung_rows($cols), VV_MON_ROWS_PER_SCREEN);
} }
// Whether this rung is already taller than one screen. The row-height cap means "no card taller
// than 1/VV_MON_ROWS_PER_SCREEN of the screen", which is only worth paying for on a rung that
// fits a screen — there it is what keeps the board from scrolling. On a rung with more rows than
// the cap the page scrolls no matter what, so the cap buys nothing and only cuts each card's
// content off. At four columns this board is eight rows: an 11" tablet in landscape was being
// given 126px cards on a board two screens tall.
function vv_mon_rung_overflows(int $cols): bool {
return vv_mon_rung_rows($cols) > VV_MON_ROWS_PER_SCREEN;
}
// Rows sized by content instead of by the cap, for a rung that overflows the screen anyway.
// Mirrors the phone hatch in css/varaverk.css: the cards have to give up overflow:hidden too, or
// they keep clipping at a height nothing is constraining any more.
function vv_mon_autorows_css(): string {
return '#vv-monitor{grid-auto-rows:auto;}'
. '#vv-monitor .vv-card{overflow:visible;}'
. '#vv-monitor .vv-card>div{overflow-y:visible;min-height:auto;}';
}
// The class the page puts on #vv-monitor when a card is not rendered, so the generated absorb // The class the page puts on #vv-monitor when a card is not rendered, so the generated absorb
// rules can fire. Derived from the id on both sides — PHP writes the rule, JS writes the class — // rules can fire. Derived from the id on both sides — PHP writes the rule, JS writes the class —
// so the two can never drift apart by a typo. // so the two can never drift apart by a typo.
@@ -216,6 +250,7 @@ function vv_mon_board_css(): string {
$out = ''; $out = '';
$out .= "#vv-monitor{--vv-cols:$top;--vv-rowdiv:" . vv_mon_rung_rowdiv($top) . ";}\n"; $out .= "#vv-monitor{--vv-cols:$top;--vv-rowdiv:" . vv_mon_rung_rowdiv($top) . ";}\n";
if (vv_mon_rung_overflows($top)) $out .= vv_mon_autorows_css() . "\n";
$order = 0; $order = 0;
foreach ($cards as $card) { foreach ($cards as $card) {
@@ -245,6 +280,7 @@ function vv_mon_board_css(): string {
if ($wide) $out .= implode(',', $wide) . "{--vv-sp:$cols;}"; if ($wide) $out .= implode(',', $wide) . "{--vv-sp:$cols;}";
} }
if (vv_mon_rung_overflows($cols)) $out .= vv_mon_autorows_css();
$out .= vv_mon_absorb_css($cols); $out .= vv_mon_absorb_css($cols);
$out .= "}\n"; $out .= "}\n";
} }
+1
View File
@@ -1079,6 +1079,7 @@ function vvPollMonitor(live) {
<span style="color:#444;">Running</span> <span style="color:#888;">${_runningCtrs} ctr${_runningCtrs !== 1 ? 's' : ''}${_runningVMs > 0 ? ` · ${_runningVMs} VM` : ''}</span> <span style="color:#444;">Running</span> <span style="color:#888;">${_runningCtrs} ctr${_runningCtrs !== 1 ? 's' : ''}${_runningVMs > 0 ? ` · ${_runningVMs} VM` : ''}</span>
<span style="color:#444;">Version</span> <span style="color:#3a3a3a;">${ver}</span> <span style="color:#444;">Version</span> <span style="color:#3a3a3a;">${ver}</span>
</div> </div>
<!-- Varaverk's own figures, ruled off from the host's. Same grid so the labels line up, <!-- Varaverk's own figures, ruled off from the host's. Same grid so the labels line up,
separate block so it reads as a different subject rather than more of the same. --> separate block so it reads as a different subject rather than more of the same. -->
<div style="border-top:1px solid #1e1e1e;margin-top:8px;padding-top:7px;"> <div style="border-top:1px solid #1e1e1e;margin-top:8px;padding-top:7px;">
@@ -10,6 +10,22 @@ a minute, so polling faster could not make it newer — it only decides how soon
**Cards you do not have do not appear.** No GPU, no UPS, no VMs, no partner — the card is absent **Cards you do not have do not appear.** No GPU, no UPS, no VMs, no partner — the card is absent
rather than showing zeros. An empty card would be a permanent reminder of nothing. rather than showing zeros. An empty card would be a permanent reminder of nothing.
**The board re-flows on a narrow screen.** On a tablet or a phone the cards regroup into fewer,
wider columns rather than shrinking in place. Nothing is hidden and nothing is dropped — the same
cards are there in the same order, cut into different rows.
Below the full-width layout the page scrolls as a whole instead of each card scrolling inside
itself. A board that is taller than the screen has to scroll somewhere, and scrolling the page
once is better than being handed a screen of cards that are each too short to read.
**Cards in the same row share a height.** They line up along the bottom, so a card with little to
say carries empty space under it. That space means its neighbour is the taller card — not that a
reading is missing.
The System card is the one exception. On a narrow screen it stops after Version and the Varaverk
figures below scroll, so that the host itself — hostname, clock, array state, uptime — is what
you get without scrolling anything.
--- ---
## Reference — System, power, CPU, memory, network ## Reference — System, power, CPU, memory, network
+63
View File
@@ -37,6 +37,12 @@
// Step 2: auto-populate + guide + checklist. // Step 2: auto-populate + guide + checklist.
// master.conf pull (for partner servers) lives in the checklist, not here. // master.conf pull (for partner servers) lives in the checklist, not here.
// The assistant is a bonus on this page, never a dependency: vv_ai_ui_on() is false on a node
// with AI off or no model reachable, which is the normal state of the fresh install this wizard
// exists to serve. The card simply is not rendered and setup proceeds exactly as before.
require_once dirname(__DIR__) . '/include/ai_chat.php';
if (vv_ai_ui_on()) vv_ai_chat_assets();
$detectedHostname = vv_get_hostname(); $detectedHostname = vv_get_hostname();
// ── Identity already on disk? ──────────────────────────────────────────────────────────────── // ── Identity already on disk? ────────────────────────────────────────────────────────────────
@@ -313,6 +319,21 @@ hr.vv-hr { border: none; border-top: 1px solid #1e1e1e; margin: 22px 0; }
<div id="vv-onboard-panel"></div> <div id="vv-onboard-panel"></div>
<div id="vv-done-banner"></div> <div id="vv-done-banner"></div>
<?php if (vv_ai_ui_on()): ?>
<hr class="vv-hr">
<div id="vv-su-ai-card">
<?php vv_ai_chat_markup('vv-su-ai', [
'profile' => 'varaverk',
'compact' => true,
'title' => 'Setup assistant',
'height' => '260px',
'tall' => '460px',
'empty' => 'Stuck on a step? Ask what it wants and why — "what is the API key for", '
. '"why does SSH need the other host", "what does populate actually do".',
]); ?>
</div>
<?php endif; ?>
<div style="margin-top:18px;text-align:right;"> <div style="margin-top:18px;text-align:right;">
<a href="#" id="vv-exit-link" onclick="vvGoNext(event)" <a href="#" id="vv-exit-link" onclick="vvGoNext(event)"
style="font-size:11px;color:#3a3a3a;text-decoration:none;">Skip →</a> style="font-size:11px;color:#3a3a3a;text-decoration:none;">Skip →</a>
@@ -443,6 +464,7 @@ function vvShowStep2(redirect, apiKey) {
_vvRedirect = redirect || '?tab=scheduler'; _vvRedirect = redirect || '?tab=scheduler';
document.getElementById('vv-step1').style.display = 'none'; document.getElementById('vv-step1').style.display = 'none';
document.getElementById('vv-step2').style.display = 'block'; document.getElementById('vv-step2').style.display = 'block';
vvSetupAiInit(); // card is visible now, so it can measure itself correctly
if (apiKey && apiKey.ok) { if (apiKey && apiKey.ok) {
const btn = document.getElementById('vv-key-btn'); const btn = document.getElementById('vv-key-btn');
const status = document.getElementById('vv-key-status'); const status = document.getElementById('vv-key-status');
@@ -676,6 +698,47 @@ function vvStartPhase2Watch(sinceMs) {
_vvPartnerPoll = setInterval(check, 6000); _vvPartnerPoll = setInterval(check, 6000);
} }
// ── Setup assistant ───────────────────────────────────────────────────────────
// Scoped to the step the operator is actually stuck on, so "what does this want" resolves without
// them naming it. The worker turns a non-empty scope into a line saying what is open in the
// WebGUI; the varaverk profile holds no scoped_log capability, so a checklist id never triggers a
// log lookup and cannot produce a "log missing" note for something that was never a script.
//
// Re-read at send time, not captured: the checklist re-polls while the page is open and the step
// they are on can change between opening the composer and pressing Ask.
//
// First not-ok, not-deferred item — deferred means "I have decided to skip this", which is not
// where they are stuck. Falls back to the bare page when the list is clean or has not loaded.
function vvSetupScope() {
const d = _vvLastChecklist;
if (!d || !Array.isArray(d.items)) return 'Setup';
const stuck = d.items.find(i => !i.ok && !i.deferred);
return stuck && stuck.id ? 'Setup/' + stuck.id : 'Setup';
}
// Mounted when Step 2 is revealed, not at parse time. This is the only page where the card starts
// inside a display:none block, and a chat that measured itself while hidden would come up wrong
// with nothing to correct it — the component carries no resize observer. Idempotent because
// vvShowStep2 is reachable more than once.
let _vvSetupAiUp = false;
function vvSetupAiInit() {
if (_vvSetupAiUp) return;
if (typeof VvAiChat !== 'function') return;
if (!document.getElementById('vv-su-ai-chat')) return; // AI off — card was never rendered
_vvSetupAiUp = true;
VvAiChat({
prefix: 'vv-su-ai',
profile: 'varaverk',
// Pinned, for the same reason the Partnership card pins it: resuming whatever thread was last
// touched anywhere in the UI could land a first-run operator mid-way through someone else's
// Scheduler conversation, on a card with no picker to get back from.
resumeProfile: 'varaverk',
scope: vvSetupScope,
scopeLabel: 'Setup',
empty: 'Stuck on a step? Ask what it wants and why.',
});
}
function vvLoadChecklist() { function vvLoadChecklist() {
fetch('/plugins/varaverk/api/checklist.php?_=' + Date.now()) fetch('/plugins/varaverk/api/checklist.php?_=' + Date.now())
.then(r => r.json()).then(d => { .then(r => r.json()).then(d => {
+19 -1
View File
@@ -23,6 +23,7 @@
# rsync would then stop nothing before copying a live database. They are excluded by name and # rsync would then stop nothing before copying a live database. They are excluded by name and
# must stay excluded. # must stay excluded.
# #
# OPERATIONAL SAFEGUARDS
# A docker that cannot be read is not a docker with nothing in it. # A docker that cannot be read is not a docker with nothing in it.
# If `docker ps -a` fails, times out, or returns nothing on a host that is meant to run # If `docker ps -a` fails, times out, or returns nothing on a host that is meant to run
# containers, EVERY name looks missing and one run would strike the entire configuration. # containers, EVERY name looks missing and one run would strike the entire configuration.
@@ -34,7 +35,24 @@
# verified by sourcing the result in a subshell and confirming the key still parses as an # verified by sourcing the result in a subshell and confirming the key still parses as an
# array with exactly one fewer element. # array with exactly one fewer element.
# #
# USAGE # Each removal is its own verified rewrite.
# Several names can reach the limit in one run, and each is removed and re-verified
# independently rather than batched into a single edit. A rewrite that fails verification
# therefore costs that one entry, not every entry the run intended to prune.
#
# --dry-run records no strike. A dry run that advanced the counter would eventually prune
# through repetition alone, which is the opposite of what it is for.
#
# CONFIGURATION
# master.conf
# CONF_PRUNE_STRIKE_LIMIT consecutive runs a name must be missing before it is removed.
# Seeing the container again resets its strike to zero immediately,
# so a rebuild costs one strike at most.
#
# The keys this may prune are an explicit allow list in the script, deliberately not a conf
# value — see DESIGN PRINCIPLES for why a pattern is the wrong shape here.
#
# RUNTIME MODES
# conf_container_prune.sh strike, and prune anything at the limit # conf_container_prune.sh strike, and prune anything at the limit
# conf_container_prune.sh --dry-run report what would be struck and pruned, write nothing # conf_container_prune.sh --dry-run report what would be struck and pruned, write nothing
# conf_container_prune.sh --status show current strikes and stop # conf_container_prune.sh --status show current strikes and stop
+11 -1
View File
@@ -270,7 +270,17 @@ log "$ICON_SUCCESS Internet reachable"
# ============================================================================================== # ==============================================================================================
if [[ -n "$DDNS_DOMAIN" ]] && [[ -n "$DDNS_CONTAINER" ]]; then if [[ -n "$DDNS_DOMAIN" ]] && [[ -n "$DDNS_CONTAINER" ]]; then
PUBLIC_IP=$(curl -sf --max-time 5 https://ifconfig.me 2>/dev/null | tr -d '[:space:]') PUBLIC_IP=$(curl -sf --max-time 5 https://ifconfig.me 2>/dev/null | tr -d '[:space:]')
DNS_IP=$(dig +short "$DDNS_DOMAIN" @1.1.1.1 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1) # dig's failure text names the resolver it could not reach (";; communications error to
# 1.1.1.1#53: timed out"), so scraping its output for an address yields the server, not the
# answer — a guaranteed mismatch that restarts DDNS over what is only a DNS timeout. Trust
# the exit status, and anchor the match so only a bare answer line counts. NXDOMAIN exits 0
# with no output and correctly falls through to the "could not resolve" branch below.
if DNS_ANSWER=$(dig +short "$DDNS_DOMAIN" @1.1.1.1 2>/dev/null); then
DNS_IP=$(printf '%s\n' "$DNS_ANSWER" \
| grep -Eox '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1)
else
DNS_IP=""
fi
if [[ -z "$PUBLIC_IP" ]]; then if [[ -z "$PUBLIC_IP" ]]; then
warn "Could not determine public IP — skipping DDNS check" warn "Could not determine public IP — skipping DDNS check"
Executable → Regular
+112 -6
View File
@@ -2273,8 +2273,10 @@ acquire_rsync_lock() {
# ============================================================================================== # ==============================================================================================
# Generic get/set for flat "key<sep>value" state files (one entry per line) — the pattern # Generic get/set for flat "key<sep>value" state files (one entry per line) — the pattern
# every watchdog's strike-tracking and state-file logic was independently reimplementing. # every watchdog's strike-tracking and state-file logic was independently reimplementing.
# grep -v + tempfile-swap on write, not sed -i in place — avoids sed treating a key containing # Keys are matched as literal prefixes via awk substr(), never as regexes. A key is often a
# regex metacharacters (container names, etc.) as part of the substitution pattern. # media file path, and a release tag like [Bluray-1080p] is a valid-looking bracket expression
# holding the reversed range 1-0 — grep -E rejects it, exits 2, and the tempfile-swap then
# commits an empty file, silently wiping every other entry.
# #
# Usage: wd_state_get "$key" "$file" [sep=:] # Usage: wd_state_get "$key" "$file" [sep=:]
# wd_state_set "$key" "$value" "$file" [sep=:] # wd_state_set "$key" "$value" "$file" [sep=:]
@@ -2284,14 +2286,24 @@ acquire_rsync_lock() {
# keep their own thin same-named wrapper around these rather than changing call sites. # keep their own thin same-named wrapper around these rather than changing call sites.
wd_state_get() { wd_state_get() {
local key="$1" file="$2" sep="${3:-:}" local key="$1" file="$2" sep="${3:-:}"
grep -E "^${key}${sep}" "$file" 2>/dev/null | cut -d"$sep" -f2- [[ -f "$file" ]] || return 0
awk -v pfx="${key}${sep}" \
'substr($0, 1, length(pfx)) == pfx { print substr($0, length(pfx) + 1); exit }' \
"$file" 2>/dev/null
} }
# Returns 1 without touching the state file if the rewrite fails, so a read error costs the
# caller one update rather than the whole file.
wd_state_set() { wd_state_set() {
local key="$1" value="$2" file="$3" sep="${4:-:}" local key="$1" value="$2" file="$3" sep="${4:-:}"
grep -vE "^${key}${sep}" "$file" 2>/dev/null > "${file}.tmp" local tmp="${file}.tmp"
echo "${key}${sep}${value}" >> "${file}.tmp" : > "$tmp" || return 1
mv "${file}.tmp" "$file" if [[ -f "$file" ]]; then
awk -v pfx="${key}${sep}" \
'substr($0, 1, length(pfx)) != pfx' "$file" > "$tmp" || { rm -f "$tmp"; return 1; }
fi
echo "${key}${sep}${value}" >> "$tmp"
mv "$tmp" "$file"
} }
# ============================================================================================== # ==============================================================================================
@@ -2722,6 +2734,100 @@ check_delete_size_threshold() {
fi fi
} }
# Applies a per-run delete budget instead of refusing the whole run. Reads "<size>\t<ctime>\t<path>"
# from $1, writes the paths that fit inside $3 GB to $2 (oldest ctime first), and reports what was
# held back in _BUDGET_*.
#
# A cap that aborts cannot drain a backlog larger than itself. Every run re-finds the same lump,
# exceeds the same ceiling and stops, so the queue is stuck permanently and only a human with
# --i-know-what-im-doing can clear it. Observed 2026-08-24..26: a partnership merge restored ten
# months of superseded Radarr files in one night, 126GB against a 100GB cap, and the daily
# orchestrator then failed three nights running on a queue it could never drain. Budgeting keeps
# the ceiling meaning exactly what it meant — no single run removes more than max_gb — while the
# backlog clears over consecutive runs with nobody touching it.
#
# Strict oldest-first, stopping at the first file that does not fit rather than skipping it for a
# smaller one further down. Skipping would drain more per run but lets a large file be passed over
# indefinitely; stopping guarantees every deferred file is newer than everything already removed,
# so nothing can starve.
#
# One case is deliberately left for a human: a single file larger than the entire budget can never
# fit, so _BUDGET_STUCK is set and the caller says so rather than looping forever on it.
apply_delete_budget() {
local src="$1" dst="$2" max_gb="$3"
local max_bytes tab
tab=$(printf '\t')
max_bytes=$(awk "BEGIN {printf \"%d\", $max_gb * 1073741824}")
_BUDGET_KEPT_BYTES=0; _BUDGET_KEPT_COUNT=0
_BUDGET_DEFERRED_BYTES=0; _BUDGET_DEFERRED_COUNT=0
_BUDGET_STUCK=""
: > "$dst"
[[ -s "$src" ]] || return 0
local out
out=$(sort -t"$tab" -k2,2n "$src" | awk -F'\t' -v max="$max_bytes" -v dst="$dst" '
{
if (stop) { defb += $1; defc++; next }
if (used + $1 > max) { stop = 1
if (kept == 0) stuck = $3
defb += $1; defc++; next }
used += $1; kept++
print $3 > dst
}
END { printf "%d\t%d\t%d\t%d\t%s", used+0, kept+0, defb+0, defc+0, stuck }
')
IFS=$'\t' read -r _BUDGET_KEPT_BYTES _BUDGET_KEPT_COUNT \
_BUDGET_DEFERRED_BYTES _BUDGET_DEFERRED_COUNT _BUDGET_STUCK <<< "$out"
}
# Asks the local generation model to characterise a block of evidence, for the one line a human
# would otherwise have to derive by hand. Prints the note on stdout and returns 0; returns 1 and
# prints nothing whenever anything at all is missing, off, slow or malformed.
#
# The contract is that failure is indistinguishable from the feature being switched off, because
# every caller must behave identically either way. An assist that can block a nightly cleanup is
# not an assist — so this never retries, never blocks longer than its timeout, and never returns
# a partial answer for a caller to interpret.
#
# Usage: note=$(ai_assist_note AI_ASSIST_CLEANUP "$prompt") && echo "$note"
# $1 name of the AI_ASSIST_* flag governing this caller — passed by name, checked here, so a
# caller cannot accidentally run an assist its own toggle says is off
# $2 the prompt, evidence included
# $3 timeout in seconds (default AI_ASSIST_TIMEOUT, else 45)
ai_assist_note() {
local flag="$1" prompt="$2" ai_timeout="${3:-${AI_ASSIST_TIMEOUT:-45}}"
[[ "${AI_ENABLED:-false}" == "true" ]] || return 1
[[ "${!flag:-false}" == "true" ]] || return 1
[[ -n "${MY_ID:-}" ]] || return 1
command -v jq >/dev/null 2>&1 || return 1
local u_var="${MY_ID}_OLLAMA_URL" m_var="${MY_ID}_OLLAMA_MODEL"
local url="${!u_var:-}" model="${!m_var:-}"
[[ -n "$url" && -n "$model" ]] || return 1
local body out
# temperature 0: this is a description of evidence, and the same evidence should not produce a
# different characterisation on a rerun.
body=$(jq -nc --arg m "$model" --arg p "$prompt" \
'{model:$m, prompt:$p, stream:false, options:{temperature:0}}' 2>/dev/null) || return 1
out=$(timeout "$ai_timeout" curl -sf --max-time "$ai_timeout" \
-H 'Content-Type: application/json' -d "$body" \
"${url%/}/api/generate" 2>/dev/null | jq -r '.response // empty' 2>/dev/null)
# Reasoning models emit a <think> block before the answer. It is not the note — a log line a
# human is meant to skim cannot open with several hundred words of the model talking itself
# through the arithmetic. Strip to the last close tag; a response with no block is unchanged.
[[ "$out" == *"</think>"* ]] && out="${out##*</think>}"
out="${out#"${out%%[![:space:]]*}"}"
[[ -n "$out" ]] || return 1
printf '%s\n' "$out"
}
# True if filepath's extension (case-insensitive) matches one of the given extensions. # True if filepath's extension (case-insensitive) matches one of the given extensions.
# Usage: has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}" # Usage: has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"
has_extension() { has_extension() {