Compare commits
19
Commits
d25b147a56
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5fa7750046 | ||
|
|
1838eed855 | ||
|
|
96d8a5e3f0 | ||
|
|
0431e720de | ||
|
|
7f4921de49 | ||
|
|
d4c19baa32 | ||
|
|
d444fd8036 | ||
|
|
d42b1e2dda | ||
|
|
9492dc4c39 | ||
|
|
172beca3c5 | ||
|
|
f0c1289519 | ||
|
|
128172d3a8 | ||
|
|
820e8325a6 | ||
|
|
74c6a0f5eb | ||
|
|
d289a9101c | ||
|
|
9553ecb16a | ||
|
|
fb104bf05a | ||
|
|
f4adc31215 | ||
|
|
c6254f2342 |
Executable → Regular
+120
-8
@@ -273,6 +273,9 @@ touch "$CORRUPTION_SCAN_STATE_FILE"
|
||||
|
||||
CORRUPTION_SCAN_STRIKES_FILE="${CORRUPTION_SCAN_STRIKES_FILE:-$DATA_DIR/corruption_scan_strikes.tsv}"
|
||||
CORRUPTION_SCAN_STRIKE_LIMIT="${CORRUPTION_SCAN_STRIKE_LIMIT:-2}"
|
||||
CORRUPTION_SCAN_MAX_CORRUPT_PCT="${CORRUPTION_SCAN_MAX_CORRUPT_PCT:-10}"
|
||||
CORRUPTION_SCAN_MAX_CONSECUTIVE="${CORRUPTION_SCAN_MAX_CONSECUTIVE:-15}"
|
||||
CORRUPTION_SCAN_GUARD_MIN_SCANNED="${CORRUPTION_SCAN_GUARD_MIN_SCANNED:-20}"
|
||||
mkdir -p "$(dirname "$CORRUPTION_SCAN_STRIKES_FILE")"
|
||||
touch "$CORRUPTION_SCAN_STRIKES_FILE"
|
||||
|
||||
@@ -301,6 +304,19 @@ reset_scan_strikes() {
|
||||
[[ -n "$current" && "$current" != "0" ]] && set_scan_strikes "$1" 0
|
||||
}
|
||||
|
||||
# Bails out of the whole run without committing anything. Safe to call at any point before
|
||||
# the commit phase: strikes are queued in memory until then, so an abort leaves the strike
|
||||
# file exactly as the previous run left it and deletes nothing.
|
||||
abort_scan() {
|
||||
local why="$1"
|
||||
error "Corruption scan ABORTED — $why"
|
||||
error "No strikes recorded and nothing remediated this run — the library was not trusted."
|
||||
[[ -n "${FRESH_CLEAN_TMP:-}" ]] && rm -f "$FRESH_CLEAN_TMP"
|
||||
notify "Corruption scan aborted on $(hostname) ($MY_ID) — $why. Nothing deleted." \
|
||||
"Arr Corruption Scan" "warning"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Per-arr API shape differences — everything else in the scan/strike/remediate loop below is
|
||||
# identical between Sonarr and Radarr.
|
||||
declare -A ARR_FILE_ENDPOINT=( [sonarr]="episodefile" [radarr]="moviefile" )
|
||||
@@ -322,6 +338,8 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo "$ICON_GEAR State file: $CORRUPTION_SCAN_STATE_FILE"
|
||||
echo "$ICON_GEAR Strike limit: $CORRUPTION_SCAN_STRIKE_LIMIT"
|
||||
echo "$ICON_GEAR Remediate: $REMEDIATE"
|
||||
echo "$ICON_GEAR Corrupt ceiling: ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% of scanned (min ${CORRUPTION_SCAN_GUARD_MIN_SCANNED} scanned)"
|
||||
echo "$ICON_GEAR Consecutive trip: $CORRUPTION_SCAN_MAX_CONSECUTIVE"
|
||||
echo "$ICON_GEAR Scan limit: ${SCAN_LIMIT:-unlimited} (per arr)"
|
||||
echo "$ICON_GEAR Path filter: ${PATH_FILTER:-none}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
@@ -357,13 +375,36 @@ ffprobe_translate_path() {
|
||||
# Probes one file. Echoes "clean" or "corrupt:<reason>". Never trusts a truncated/garbled
|
||||
# stderr as automatically corrupt — only a real non-empty ffprobe stderr counts.
|
||||
probe_file() {
|
||||
local host_path="$1" container_path output
|
||||
local host_path="$1" container_path output rc
|
||||
container_path=$(ffprobe_translate_path "$host_path") || { echo "unmapped"; return; }
|
||||
output=$(docker exec "$FFPROBE_CONTAINER" "$FFPROBE_BIN" -v error "$container_path" 2>&1)
|
||||
rc=$?
|
||||
|
||||
# docker exec writes its own failures to the same stream ffprobe uses, so a stopped
|
||||
# container or an unreachable daemon is otherwise indistinguishable from a corrupt
|
||||
# header. A stopped container exits 1 with a daemon message; a missing binary exits
|
||||
# 127 — neither is evidence about the file, so both must be caught.
|
||||
if (( rc >= 125 )) \
|
||||
|| [[ "$output" == "Error response from daemon:"* \
|
||||
|| "$output" == "Cannot connect to the Docker daemon"* \
|
||||
|| "$output" == "error during connect:"* ]]; then
|
||||
echo "probe_error:${output//$'\n'/ }"
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ -z "$output" ]]; then
|
||||
echo "clean"
|
||||
else
|
||||
elif (( rc != 0 )); then
|
||||
# ffprobe could not parse the file — EBML header parsing failed, moov atom not found,
|
||||
# contradictionary STSC and STCO. This is the only class that may be remediated.
|
||||
echo "corrupt:${output//$'\n'/ }"
|
||||
else
|
||||
# Exit 0 with stderr output: a recoverable muxing complaint, most commonly
|
||||
# "Referenced QT chapter track not found", which many recent .mp4 releases emit and
|
||||
# which says nothing about playability. Equating any stderr with corruption is what
|
||||
# produced 103 "corrupt" files on 2026-08-23 — 28 of 43 newly scanned Radarr items.
|
||||
# Reported for visibility, never strike-tracked, never remediated.
|
||||
echo "suspect:${output//$'\n'/ }"
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -442,7 +483,7 @@ TOTAL_SCANNED=0
|
||||
TOTAL_CORRUPT=0
|
||||
TOTAL_REMEDIATED=0
|
||||
TOTAL_REMEDIATE_FAILED=0
|
||||
declare -A ARR_SCANNED ARR_SKIPPED_CACHED ARR_SKIPPED_UNMAPPED ARR_CORRUPT ARR_STRIKE_HELD ARR_REMEDIATED ARR_REMEDIATE_FAILED
|
||||
declare -A ARR_SCANNED ARR_SKIPPED_CACHED ARR_SKIPPED_UNMAPPED ARR_CORRUPT ARR_SUSPECT ARR_PROBE_ERRORS ARR_STRIKE_HELD ARR_REMEDIATED ARR_REMEDIATE_FAILED
|
||||
|
||||
for arr in sonarr radarr; do
|
||||
url_var="${arr^^}_URL"; key_var="${arr^^}_API_KEY"
|
||||
@@ -532,6 +573,12 @@ for arr in sonarr radarr; do
|
||||
STRIKE_HELD=0
|
||||
REMEDIATED=0
|
||||
REMEDIATE_FAILED=0
|
||||
PROBE_ERRORS=0
|
||||
SUSPECT_COUNT=0
|
||||
CONSECUTIVE_BAD=0
|
||||
QUEUE_PATH=()
|
||||
QUEUE_STRIKES=()
|
||||
QUEUE_ITEM=()
|
||||
|
||||
FRESH_CLEAN_TMP=$(mktemp)
|
||||
|
||||
@@ -566,23 +613,82 @@ for arr in sonarr radarr; do
|
||||
continue
|
||||
fi
|
||||
|
||||
# A docker-level failure is not evidence about the file. Count it, never queue it.
|
||||
if [[ "$result" == probe_error:* ]]; then
|
||||
(( PROBE_ERRORS++ ))
|
||||
(( CONSECUTIVE_BAD++ ))
|
||||
warn " ? $host_path — probe failed, NOT counted as corrupt: ${result#probe_error:}"
|
||||
if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then
|
||||
abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$result" == "clean" ]]; then
|
||||
CONSECUTIVE_BAD=0
|
||||
reset_scan_strikes "$host_path"
|
||||
echo -e "${host_path}\t${stamp}" >> "$FRESH_CLEAN_TMP"
|
||||
[[ "$ENABLE_LOGGING" == true ]] && echo " $ICON_SUCCESS $host_path"
|
||||
continue
|
||||
fi
|
||||
|
||||
# corrupt:<reason>
|
||||
# A successful probe that merely warned. Proves the container is alive, so it clears
|
||||
# the consecutive-failure tripwire, but it never becomes a strike.
|
||||
if [[ "$result" == suspect:* ]]; then
|
||||
CONSECUTIVE_BAD=0
|
||||
(( SUSPECT_COUNT++ ))
|
||||
[[ "$ENABLE_LOGGING" == true ]] && warn " ~ $host_path — ffprobe warning (exit 0), NOT corrupt: ${result#suspect:}"
|
||||
continue
|
||||
fi
|
||||
|
||||
# corrupt:<reason> — queued, NOT committed. Nothing reaches the strike file and nothing
|
||||
# is deleted until this arr has been fully probed and the guards below have passed. A
|
||||
# container that dies mid-scan makes every remaining file read as corrupt, and a delete
|
||||
# cannot be undone — so the destructive half has to wait until the corrupt rate for the
|
||||
# whole run is known. 2026-08-23: one Jellyfin restart produced 103 false positives.
|
||||
reason="${result#corrupt:}"
|
||||
(( CORRUPT_COUNT++ ))
|
||||
strikes=$(increment_scan_strikes "$host_path")
|
||||
(( CONSECUTIVE_BAD++ ))
|
||||
|
||||
prev_strikes=$(get_scan_strikes "$host_path")
|
||||
prev_strikes="${prev_strikes//[^0-9]/}"
|
||||
strikes=$(( ${prev_strikes:-0} + 1 ))
|
||||
|
||||
QUEUE_PATH+=("$host_path")
|
||||
QUEUE_STRIKES+=("$strikes")
|
||||
QUEUE_ITEM+=("$item")
|
||||
|
||||
echo " $ICON_ERROR CORRUPT: $host_path (strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT)"
|
||||
[[ "$ENABLE_LOGGING" == true ]] && echo " $reason"
|
||||
|
||||
if [[ "$REMEDIATE" != true ]]; then
|
||||
continue
|
||||
if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then
|
||||
abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly"
|
||||
fi
|
||||
done < <(echo "$ITEMS" | jq -c '.[]')
|
||||
|
||||
# ━━━ False-positive guards — run before anything is committed ━━━
|
||||
if (( CORRUPT_COUNT > 0 )); then
|
||||
# The pre-flight check only proves the container was up when the scan started.
|
||||
# Re-check now: a mid-scan death is exactly what this guard exists to catch.
|
||||
check_container_health "$FFPROBE_CONTAINER" "${DOCKER_TIMEOUT:-30}" "Arr Corruption Scan"
|
||||
|
||||
if (( SCANNED >= CORRUPTION_SCAN_GUARD_MIN_SCANNED )); then
|
||||
corrupt_pct=$(( CORRUPT_COUNT * 100 / SCANNED ))
|
||||
if (( corrupt_pct >= CORRUPTION_SCAN_MAX_CORRUPT_PCT )); then
|
||||
abort_scan "$CORRUPT_COUNT of $SCANNED probed files (${corrupt_pct}%) read as corrupt — at or above the ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% ceiling"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ━━━ Guards passed — commit strikes, then remediate whatever reached the limit ━━━
|
||||
for _q in "${!QUEUE_PATH[@]}"; do
|
||||
host_path="${QUEUE_PATH[$_q]}"
|
||||
strikes="${QUEUE_STRIKES[$_q]}"
|
||||
item="${QUEUE_ITEM[$_q]}"
|
||||
|
||||
set_scan_strikes "$host_path" "$strikes"
|
||||
|
||||
[[ "$REMEDIATE" != true ]] && continue
|
||||
|
||||
if (( strikes < CORRUPTION_SCAN_STRIKE_LIMIT )); then
|
||||
warn " $host_path — strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT, not yet remediating (needs repeat confirmation)"
|
||||
@@ -591,6 +697,8 @@ for arr in sonarr radarr; do
|
||||
fi
|
||||
reset_scan_strikes "$host_path"
|
||||
|
||||
file_id=$(echo "$item" | jq -r '.file_id')
|
||||
parent_id=$(echo "$item" | jq -r '.parent_id')
|
||||
title=$(echo "$item" | jq -r '.title')
|
||||
|
||||
http_code=$(curl -sf -o /dev/null -w "%{http_code}" -X DELETE \
|
||||
@@ -625,7 +733,7 @@ for arr in sonarr radarr; do
|
||||
warn " $title — deleted and verified, but search trigger returned HTTP $search_code"
|
||||
(( REMEDIATE_FAILED++ ))
|
||||
fi
|
||||
done < <(echo "$ITEMS" | jq -c '.[]')
|
||||
done
|
||||
|
||||
merge_clean_state "$FRESH_CLEAN_TMP"
|
||||
rm -f "$FRESH_CLEAN_TMP"
|
||||
@@ -634,6 +742,8 @@ for arr in sonarr radarr; do
|
||||
ARR_SKIPPED_CACHED[$arr]=$SKIPPED_CACHED
|
||||
ARR_SKIPPED_UNMAPPED[$arr]=$SKIPPED_UNMAPPED
|
||||
ARR_CORRUPT[$arr]=$CORRUPT_COUNT
|
||||
ARR_SUSPECT[$arr]=$SUSPECT_COUNT
|
||||
ARR_PROBE_ERRORS[$arr]=$PROBE_ERRORS
|
||||
ARR_STRIKE_HELD[$arr]=$STRIKE_HELD
|
||||
ARR_REMEDIATED[$arr]=$REMEDIATED
|
||||
ARR_REMEDIATE_FAILED[$arr]=$REMEDIATE_FAILED
|
||||
@@ -658,6 +768,8 @@ for arr in sonarr radarr; do
|
||||
echo " $ICON_SUCCESS Skipped (cached): ${ARR_SKIPPED_CACHED[$arr]}"
|
||||
echo " $ICON_WARN Skipped (unmapped): ${ARR_SKIPPED_UNMAPPED[$arr]}"
|
||||
echo " $ICON_ERROR Corrupt found: ${ARR_CORRUPT[$arr]}"
|
||||
echo " $ICON_WARN Warnings (exit 0): ${ARR_SUSPECT[$arr]} (reported, never remediated)"
|
||||
echo " $ICON_WARN Probe errors: ${ARR_PROBE_ERRORS[$arr]} (not counted as corrupt)"
|
||||
if [[ "$REMEDIATE" == true ]]; then
|
||||
echo " $ICON_WARN Held (strikes): ${ARR_STRIKE_HELD[$arr]}"
|
||||
echo " $ICON_SUCCESS Remediated: ${ARR_REMEDIATED[$arr]}"
|
||||
|
||||
Executable → Regular
+70
-8
@@ -457,6 +457,30 @@ NOW=$(date +%s)
|
||||
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
||||
> "$TO_DELETE_FILE"
|
||||
|
||||
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
|
||||
# Same contract as radarr_cleanup.sh: a file must classify for deletion on
|
||||
# LIDARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is removed. Covers the partial
|
||||
# classification failure that is too small to trip the tracked-count floor above. The file is
|
||||
# rebuilt from each run rather than edited, which is what prunes it.
|
||||
LIDARR_ORPHAN_STRIKE_LIMIT="${LIDARR_ORPHAN_STRIKE_LIMIT:-2}"
|
||||
STRIKES_FILE="${LIDARR_ORPHAN_STRIKES_FILE:-$DB_DIR/lidarr_orphan_strikes.tsv}"
|
||||
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
|
||||
touch "$STRIKES_FILE" 2>/dev/null || true
|
||||
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
|
||||
> "$STRIKES_NEW"
|
||||
HELD_COUNT=0
|
||||
HELD_BYTES=0
|
||||
|
||||
orphan_strike_ok() {
|
||||
local path="$1" prev strikes
|
||||
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
|
||||
strikes=$(( ${prev:-0} + 1 ))
|
||||
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
|
||||
(( strikes >= LIDARR_ORPHAN_STRIKE_LIMIT )) && return 0
|
||||
warn " strike $strikes/$LIDARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
|
||||
return 1
|
||||
}
|
||||
|
||||
while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
FILE_CTIME="${FILE_CTIME%%.*}"
|
||||
@@ -493,12 +517,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||
(( ORPHAN_COUNT++ ))
|
||||
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
else
|
||||
log "JUNK: $filepath"
|
||||
(( JUNK_COUNT++ ))
|
||||
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
fi
|
||||
|
||||
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
||||
@@ -506,13 +532,43 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
# 4.3ms), since find already has to stat() every entry anyway to know it's -type f.
|
||||
done < <(find "$LIDARR_MUSIC_ROOT" -type f -printf '%s %C@ %p\n' 2>/dev/null)
|
||||
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||
# Eligible, not classified: a file still serving its strikes is an orphan but is not queued this
|
||||
# run, so it must not appear in the denominator the budget reports against.
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
|
||||
|
||||
# Rebuilt, never edited. Skipped on a dry run: a preview that advanced real counters would make
|
||||
# the next real run delete a run early.
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
||||
# ==============================================================================================
|
||||
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$LIDARR_MAX_DELETE_GB" "Lidarr Cleanup"
|
||||
# A per-run budget, not a veto — see apply_delete_budget() in common.sh. The ceiling still caps
|
||||
# any single run; it just no longer deadlocks on a backlog larger than itself.
|
||||
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
|
||||
|
||||
if [[ "$I_KNOW" == true ]]; then
|
||||
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
|
||||
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
|
||||
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
|
||||
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
|
||||
else
|
||||
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$LIDARR_MAX_DELETE_GB"
|
||||
if [[ -n "$_BUDGET_STUCK" ]]; then
|
||||
error "Single file exceeds the ${LIDARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
|
||||
error " $_BUDGET_STUCK"
|
||||
error "Raise LIDARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
|
||||
notify "Lidarr cleanup stalled on $(hostname) — one file exceeds the ${LIDARR_MAX_DELETE_GB}GB budget" \
|
||||
"Lidarr Cleanup" "warning"
|
||||
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
|
||||
warn "Budget ${LIDARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
|
||||
notify "Lidarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname) — $_BUDGET_DEFERRED_COUNT file(s) deferred" \
|
||||
"Lidarr Cleanup" "normal"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||
# All safety layers passed — delete orphans and junk. Reuses TO_DELETE_FILE from the
|
||||
@@ -521,7 +577,7 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||
done < "$TO_DELETE_FILE"
|
||||
done < "$BUDGET_FILE"
|
||||
|
||||
info "Cleaning up empty folders..."
|
||||
find "$LIDARR_MUSIC_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null
|
||||
@@ -546,6 +602,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (cover art, metadata
|
||||
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${LIDARR_ORPHAN_AGE} days)"
|
||||
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${LIDARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
|
||||
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${LIDARR_MAX_DELETE_GB}GB run budget"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
@@ -554,8 +614,10 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Clean — nothing to remove"
|
||||
else
|
||||
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
||||
notify "Lidarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "warning"
|
||||
# What was actually removed, not what was classified. With strikes and a budget in force those
|
||||
# differ, and reporting the classification as the outcome is the oldest bug shape here.
|
||||
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
|
||||
notify "Lidarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "warning"
|
||||
# Notify Emby to clean missing files — removes ghost entries immediately
|
||||
notify_emby_scan
|
||||
fi
|
||||
|
||||
Executable → Regular
+134
-8
@@ -450,9 +450,46 @@ NOW=$(date +%s)
|
||||
# just delete them directly instead of re-walking and re-classifying every SCAN_ROOTS entry a
|
||||
# second time (2026-07-17) — the size-threshold check below needs to know the total before
|
||||
# deleting anything, not before knowing what to delete.
|
||||
# Carries size and ctime alongside the path now, because the budget pass below has to order by
|
||||
# age and stop at a byte ceiling — neither of which a bare path list can answer.
|
||||
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
||||
> "$TO_DELETE_FILE"
|
||||
|
||||
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
|
||||
# A file must classify for deletion on RADARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is
|
||||
# actually removed. Gate 6 already refuses a run whose tracked count collapsed; this covers the
|
||||
# partial failure underneath that threshold — one root folder failing to enumerate makes its
|
||||
# movies look orphaned while the overall percentage still looks fine, and a transient fault will
|
||||
# not reproduce on the next run.
|
||||
#
|
||||
# The file is REBUILT from this run's classifications rather than edited in place, which is what
|
||||
# prunes it: anything that stopped being an orphan simply is not written again, so a file that
|
||||
# Radarr re-adopts loses its strikes without needing a reset pass to find it.
|
||||
#
|
||||
# Keyed by host path, which is why this could not have worked before 2026-08-26 — wd_state_set
|
||||
# built a regex from the key, and a release tag like [Bluray-1080p] holds the reversed range 1-0,
|
||||
# so every write truncated the store to one line. See common.sh.
|
||||
RADARR_ORPHAN_STRIKE_LIMIT="${RADARR_ORPHAN_STRIKE_LIMIT:-2}"
|
||||
STRIKES_FILE="${RADARR_ORPHAN_STRIKES_FILE:-$DB_DIR/radarr_orphan_strikes.tsv}"
|
||||
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
|
||||
touch "$STRIKES_FILE" 2>/dev/null || true
|
||||
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
|
||||
> "$STRIKES_NEW"
|
||||
HELD_COUNT=0
|
||||
HELD_BYTES=0
|
||||
|
||||
# Records this run's strike for a file and says whether it has served enough of them.
|
||||
# Returns 0 when the file may be deleted, 1 when it is still accruing.
|
||||
orphan_strike_ok() {
|
||||
local path="$1" prev strikes
|
||||
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
|
||||
strikes=$(( ${prev:-0} + 1 ))
|
||||
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
|
||||
(( strikes >= RADARR_ORPHAN_STRIKE_LIMIT )) && return 0
|
||||
warn " strike $strikes/$RADARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
|
||||
return 1
|
||||
}
|
||||
|
||||
while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
FILE_CTIME="${FILE_CTIME%%.*}"
|
||||
@@ -487,12 +524,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||
(( ORPHAN_COUNT++ ))
|
||||
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
else
|
||||
log "JUNK: $filepath"
|
||||
(( JUNK_COUNT++ ))
|
||||
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
fi
|
||||
|
||||
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
||||
@@ -504,13 +543,94 @@ done < <(
|
||||
done | sort -u
|
||||
)
|
||||
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||
# Eligible, not classified. A file still serving its strikes was counted as an orphan above — it
|
||||
# is one — but it is not going to be deleted this run, so it must not appear in the denominator
|
||||
# the budget reports against or the run claims to have skipped work it never queued.
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
|
||||
|
||||
# Rebuilt, never edited: a path absent from this run is absent from the file, so a file Radarr
|
||||
# re-adopts drops its strikes with no reset pass needed. Skipped on a dry run — a preview that
|
||||
# advanced real strike counters would make the next real run delete a run early.
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
||||
# ==============================================================================================
|
||||
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$RADARR_MAX_DELETE_GB" "Radarr Cleanup"
|
||||
# The ceiling is a per-run budget, not a veto. It still means what it always meant — no single run
|
||||
# removes more than RADARR_MAX_DELETE_GB — but a backlog larger than the ceiling now drains over
|
||||
# consecutive nights instead of failing the orchestrator forever on a queue it cannot clear.
|
||||
# ── AI note (AI_ASSIST_CLEANUP) ───────────────────────────────────────────────────────────────
|
||||
# Describes the shape of what was classified. It decides nothing: the eligible set, the budget and
|
||||
# the strikes are all settled above and none of them read this. Switch AI_ASSIST_CLEANUP off and
|
||||
# the run removes exactly the same files — the log just loses a paragraph.
|
||||
#
|
||||
# ctime clustering is the signal worth surfacing. A normal upgrade cycle dribbles in over weeks; a
|
||||
# lump sharing one narrow ctime window with mtimes spread across months is a bulk write-back, which
|
||||
# is what a partnership merge against a partner holding older copies produces. That distinction
|
||||
# took a person an evening on 2026-08-26 and is the whole reason this note exists.
|
||||
if [[ "$ORPHAN_COUNT" -gt 0 ]] && [[ -s "$TO_DELETE_FILE" ]]; then
|
||||
_ai_ev=$(awk -F'\t' '
|
||||
{ n++; bytes += $1
|
||||
c = int($2)
|
||||
if (cmin == 0 || c < cmin) cmin = c
|
||||
if (c > cmax) cmax = c
|
||||
bucket[int(c / 21600)]++ }
|
||||
END {
|
||||
for (b in bucket) if (bucket[b] > top) { top = bucket[b] }
|
||||
printf "files=%d bytes_gb=%.1f ctime_span_hours=%.1f largest_6h_ctime_bucket=%d\n",
|
||||
n, bytes/1073741824, (cmax-cmin)/3600, top
|
||||
}' "$TO_DELETE_FILE")
|
||||
_ai_mt=$(cut -d"$(printf '\t')" -f3 "$TO_DELETE_FILE" | head -8 \
|
||||
| while IFS= read -r p; do [[ -f "$p" ]] && \
|
||||
printf '%s %s\n' "$(stat -c %y "$p" 2>/dev/null | cut -c1-7)" "$(basename "$p")"; done)
|
||||
|
||||
_ai_note=$(ai_assist_note AI_ASSIST_CLEANUP "You are looking at files an automated media-library cleanup has classified for deletion on an Unraid server. They are files on disk that the Radarr database no longer references.
|
||||
|
||||
EVIDENCE
|
||||
$_ai_ev
|
||||
sample (modification month, then path):
|
||||
$_ai_mt
|
||||
|
||||
A normal quality-upgrade cycle produces orphans whose ctimes are spread out over weeks, because each upgrade happens on its own day. A bulk event - a sync or restore writing files back onto this host - produces orphans sharing one narrow ctime window while their modification times stay spread across months, because the copy preserves modification time but resets ctime.
|
||||
|
||||
In no more than three sentences, say which of those two this looks like and name the numbers above that support it. Do not recommend an action. Do not speculate beyond the evidence given.") || _ai_note=""
|
||||
|
||||
if [[ -n "$_ai_note" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR AI note on this classification ━━━"
|
||||
printf '%s\n' "$_ai_note"
|
||||
fi
|
||||
unset _ai_ev _ai_mt
|
||||
fi
|
||||
|
||||
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
|
||||
|
||||
if [[ "$I_KNOW" == true ]]; then
|
||||
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
|
||||
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
|
||||
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
|
||||
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
|
||||
else
|
||||
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$RADARR_MAX_DELETE_GB"
|
||||
|
||||
if [[ -n "$_BUDGET_STUCK" ]]; then
|
||||
# One file larger than the whole budget can never fit, so it would be re-found and
|
||||
# re-deferred every night. Name it rather than loop on it silently.
|
||||
error "Single file exceeds the ${RADARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
|
||||
error " $_BUDGET_STUCK"
|
||||
error "Raise RADARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
|
||||
notify "Radarr cleanup stalled on $(hostname) — one file exceeds the ${RADARR_MAX_DELETE_GB}GB budget" \
|
||||
"Radarr Cleanup" "warning"
|
||||
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
|
||||
warn "Budget ${RADARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
|
||||
warn "Oldest first — the deferred files are the newest and are re-evaluated tomorrow"
|
||||
notify "Radarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname) — $_BUDGET_DEFERRED_COUNT file(s) deferred to the next run" \
|
||||
"Radarr Cleanup" "normal"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||
# Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and
|
||||
@@ -519,7 +639,7 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||
done < "$TO_DELETE_FILE"
|
||||
done < "$BUDGET_FILE"
|
||||
|
||||
info "Cleaning up empty folders..."
|
||||
for host_path in "${SCAN_ROOTS[@]}"; do
|
||||
@@ -545,6 +665,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles,
|
||||
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${RADARR_ORPHAN_AGE} days)"
|
||||
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${RADARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
|
||||
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${RADARR_MAX_DELETE_GB}GB run budget"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
@@ -553,8 +677,10 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Clean — nothing to remove"
|
||||
else
|
||||
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
||||
notify "Radarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
|
||||
# What was actually removed, not what was classified. With a budget in force those differ, and
|
||||
# reporting the classification as the outcome is the oldest bug shape in this codebase.
|
||||
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED classified files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
|
||||
notify "Radarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED classified files ($(format_bytes "$_BUDGET_KEPT_BYTES"))$([[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && echo ", $_BUDGET_DEFERRED_COUNT deferred")" \
|
||||
"Radarr Cleanup" "warning"
|
||||
# Notify Emby to clean missing files — removes ghost entries immediately
|
||||
notify_emby_scan
|
||||
|
||||
Executable → Regular
+70
-8
@@ -440,6 +440,30 @@ NOW=$(date +%s)
|
||||
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
||||
> "$TO_DELETE_FILE"
|
||||
|
||||
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
|
||||
# Same contract as radarr_cleanup.sh: a file must classify for deletion on
|
||||
# SONARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is removed. Covers the partial
|
||||
# classification failure that is too small to trip the tracked-count floor above. The file is
|
||||
# rebuilt from each run rather than edited, which is what prunes it.
|
||||
SONARR_ORPHAN_STRIKE_LIMIT="${SONARR_ORPHAN_STRIKE_LIMIT:-2}"
|
||||
STRIKES_FILE="${SONARR_ORPHAN_STRIKES_FILE:-$DB_DIR/sonarr_orphan_strikes.tsv}"
|
||||
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
|
||||
touch "$STRIKES_FILE" 2>/dev/null || true
|
||||
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
|
||||
> "$STRIKES_NEW"
|
||||
HELD_COUNT=0
|
||||
HELD_BYTES=0
|
||||
|
||||
orphan_strike_ok() {
|
||||
local path="$1" prev strikes
|
||||
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
|
||||
strikes=$(( ${prev:-0} + 1 ))
|
||||
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
|
||||
(( strikes >= SONARR_ORPHAN_STRIKE_LIMIT )) && return 0
|
||||
warn " strike $strikes/$SONARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
|
||||
return 1
|
||||
}
|
||||
|
||||
while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
FILE_CTIME="${FILE_CTIME%%.*}"
|
||||
@@ -474,12 +498,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||
(( ORPHAN_COUNT++ ))
|
||||
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
else
|
||||
log "JUNK: $filepath"
|
||||
(( JUNK_COUNT++ ))
|
||||
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
fi
|
||||
|
||||
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
||||
@@ -491,13 +517,43 @@ done < <(
|
||||
done | sort -u
|
||||
)
|
||||
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||
# Eligible, not classified: a file still serving its strikes is an orphan but is not queued this
|
||||
# run, so it must not appear in the denominator the budget reports against.
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
|
||||
|
||||
# Rebuilt, never edited. Skipped on a dry run: a preview that advanced real counters would make
|
||||
# the next real run delete a run early.
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
||||
# ==============================================================================================
|
||||
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$SONARR_MAX_DELETE_GB" "Sonarr Cleanup"
|
||||
# A per-run budget, not a veto — see apply_delete_budget() in common.sh. The ceiling still caps
|
||||
# any single run; it just no longer deadlocks on a backlog larger than itself.
|
||||
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
|
||||
|
||||
if [[ "$I_KNOW" == true ]]; then
|
||||
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
|
||||
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
|
||||
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
|
||||
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
|
||||
else
|
||||
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$SONARR_MAX_DELETE_GB"
|
||||
if [[ -n "$_BUDGET_STUCK" ]]; then
|
||||
error "Single file exceeds the ${SONARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
|
||||
error " $_BUDGET_STUCK"
|
||||
error "Raise SONARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
|
||||
notify "Sonarr cleanup stalled on $(hostname) — one file exceeds the ${SONARR_MAX_DELETE_GB}GB budget" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
|
||||
warn "Budget ${SONARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
|
||||
notify "Sonarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname) — $_BUDGET_DEFERRED_COUNT file(s) deferred" \
|
||||
"Sonarr Cleanup" "normal"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||
# Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and
|
||||
@@ -506,7 +562,7 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||
done < "$TO_DELETE_FILE"
|
||||
done < "$BUDGET_FILE"
|
||||
|
||||
info "Cleaning up empty folders..."
|
||||
for host_path in "${SCAN_ROOTS[@]}"; do
|
||||
@@ -532,6 +588,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles,
|
||||
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${SONARR_ORPHAN_AGE} days)"
|
||||
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${SONARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
|
||||
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${SONARR_MAX_DELETE_GB}GB run budget"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
@@ -540,8 +600,10 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Clean — nothing to remove"
|
||||
else
|
||||
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
||||
notify "Sonarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
|
||||
# What was actually removed, not what was classified. With strikes and a budget in force those
|
||||
# differ, and reporting the classification as the outcome is the oldest bug shape here.
|
||||
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
|
||||
notify "Sonarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
# Notify Emby to clean missing files — removes ghost entries immediately
|
||||
notify_emby_scan
|
||||
|
||||
@@ -1268,6 +1268,8 @@
|
||||
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||
# protects against API returning partial data on a bad day
|
||||
LIDARR_TRACKED_COUNT_FILE="${DB_DIR}/lidarr_tracked.count"
|
||||
LIDARR_ORPHAN_STRIKES_FILE="${DB_DIR}/lidarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
|
||||
LIDARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify before it is removed
|
||||
LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||
|
||||
# Lidarr tracked-data cache — shared by lidarr_cleanup.sh, lidarr_duplicate_artist_cleanup.sh,
|
||||
@@ -1343,6 +1345,8 @@
|
||||
SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||
# protects against API returning partial data on a bad day
|
||||
SONARR_TRACKED_COUNT_FILE="${DB_DIR}/sonarr_tracked.count"
|
||||
SONARR_ORPHAN_STRIKES_FILE="${DB_DIR}/sonarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
|
||||
SONARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify before it is removed
|
||||
SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||
SONARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveSeries command to
|
||||
# reach "completed" — generous because a large series can sit
|
||||
@@ -1355,6 +1359,15 @@
|
||||
# against a one-off ffprobe hiccup (mid-write file, NFS blip)
|
||||
# triggering an unnecessary delete. Resets to 0 the moment a
|
||||
# file probes clean again.
|
||||
CORRUPTION_SCAN_MAX_CORRUPT_PCT=10 # abort the run, committing nothing, if this share of
|
||||
# newly-scanned files reads as corrupt. A healthy library sits
|
||||
# near zero; a high rate means the detector is wrong, not the
|
||||
# library. Only counts ffprobe exit != 0.
|
||||
CORRUPTION_SCAN_MAX_CONSECUTIVE=15 # abort after this many files in a row fail to probe
|
||||
# cleanly — catches the ffprobe container dying mid-scan,
|
||||
# which the pre-flight health check cannot see.
|
||||
CORRUPTION_SCAN_GUARD_MIN_SCANNED=20 # below this many newly-scanned files the percentage
|
||||
# ceiling is not applied — too small a sample to judge.
|
||||
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
|
||||
SONARR_PROTECTED_PATTERNS=(
|
||||
# Subtitles
|
||||
@@ -1381,6 +1394,12 @@
|
||||
RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||
# protects against API returning partial data on a bad day
|
||||
RADARR_TRACKED_COUNT_FILE="${DB_DIR}/radarr_tracked.count"
|
||||
RADARR_ORPHAN_STRIKES_FILE="${DB_DIR}/radarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
|
||||
RADARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify for deletion before it is
|
||||
# removed. Gate 6 already catches an API returning far too few
|
||||
# tracked files; this catches the partial failure too small to trip
|
||||
# that percentage — one root folder failing to enumerate makes its
|
||||
# movies look orphaned, and a transient one will not repeat.
|
||||
RADARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||
RADARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveMovie command to
|
||||
# reach "completed" — mirrors SONARR_MOVE_POLL_TIMEOUT
|
||||
@@ -1647,6 +1666,10 @@
|
||||
# HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES
|
||||
BACKUP_VERIFY_SAMPLE=10 # random files to check per share
|
||||
BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample
|
||||
BACKUP_VERIFY_MD5_TIMEOUT_MAX=600 # ceiling for one remote checksum. The per-file budget
|
||||
# scales with size (~50MB/s); this caps it. A fixed
|
||||
# connect-sized timeout killed multi-GB checksums and
|
||||
# the empty result was then reported as MISSING.
|
||||
|
||||
# ━━━ SMART Health ━━━
|
||||
# Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*.
|
||||
@@ -2025,7 +2048,10 @@
|
||||
AI_ASSIST_REPORTS=false # tier 1 — digest / coffee report narration
|
||||
AI_ASSIST_WATCHDOG=false # tier 2 — file a finding when a watchdog counter passes its limit (needs AI_REPAIR_ENABLED)
|
||||
AI_ASSIST_DISCOVERY=false # tier 2 — discovery / classification judgement calls
|
||||
AI_ASSIST_CLEANUP=false # tier 2 — orphan and stuck-import triage
|
||||
AI_ASSIST_CLEANUP=false # tier 2 — orphan and stuck-import triage. Describes the shape of a
|
||||
# classification in the log; decides nothing. Off = identical deletions.
|
||||
AI_ASSIST_TIMEOUT=45 # seconds any single assist may take. An assist that can stall a
|
||||
# nightly cleanup is not an assist — it is silently skipped past this.
|
||||
AI_ASSIST_ONBOARD=false # tier 3 — onboarding / settings assistance
|
||||
|
||||
# ━━━ AI Repair ━━━
|
||||
|
||||
@@ -28,6 +28,42 @@
|
||||
# pointing at a directory the data is not in is worse than not having started.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Two halves, in order: move the files, then rewrite the conf keys that point at them. Doing it
|
||||
# the other way round would leave every path variable naming a location nothing had reached yet,
|
||||
# and any script that ran in between would create the old layout again underneath the new one.
|
||||
#
|
||||
# Idempotent. A path already under DATA_DIR is left alone, so a re-run after a partial migration
|
||||
# finishes the job rather than moving things twice or failing on what is already done.
|
||||
#
|
||||
# One-time by intent, not by a marker file. There is no "already migrated" flag — the check is
|
||||
# whether each individual path is already where it belongs, which is also what makes an
|
||||
# interrupted run safe to repeat.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Existing keys are rewritten, which is why conf_upgrade cannot do this.
|
||||
# conf_upgrade adds keys the template has and the installation does not, and never rewrites a
|
||||
# value the operator already holds — correct for it, and exactly why it is the wrong tool here.
|
||||
# STATE_DIR, BANDWIDTH_LOG, AI_INDEX_DB and two dozen more are existing keys whose values must
|
||||
# change, or they would go on naming the old layout forever while the new directory variables
|
||||
# sat beside them unused.
|
||||
#
|
||||
# Move, never copy-and-hope.
|
||||
# The data being relocated is the only copy — statistics, histories, the AI index, arr caches.
|
||||
# Everything is moved and the source is gone afterwards, so there is no second location that
|
||||
# might still be written to by something that missed the change.
|
||||
#
|
||||
# The conf rewrite is the last thing, and the riskiest thing.
|
||||
# Until it happens the installation still works from the old layout. That ordering means an
|
||||
# abort partway through leaves a system that runs, rather than one whose paths point at
|
||||
# nothing.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
@@ -49,6 +85,25 @@
|
||||
# check costs nothing and the failure is silent otherwise.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# This script reads conf to find the old locations and rewrites conf to record the new ones. It
|
||||
# is the one script here whose purpose is to change these values rather than obey them.
|
||||
#
|
||||
# Read to locate what moves
|
||||
# STATE_DIR, BANDWIDTH_LOG, AI_INDEX_DB, AI_MEMORY_FILE, AI_TOKEN_DB, ARR_CLEANUP_STATS,
|
||||
# ARR_SYNC_BLOCKLIST, CORRUPTION_SCAN_STATE_FILE, LIDARR_CACHE_FILE, ZFS_REPORT_LOG and the
|
||||
# rest of the per-script path keys — roughly two dozen in total.
|
||||
#
|
||||
# Written as the new roots
|
||||
# DATA_DIR and the directories beneath it: DB_DIR, STATE_DIR, AI_DATA_DIR,
|
||||
# CACHE_BACKUP_DIR, ARR_CACHE_BACKUP_DIR, CONF_CACHE_BACKUP_DIR, LOG_ARCHIVE_DIR.
|
||||
#
|
||||
# Every rewritten value is expressed as ${DB_DIR}/… rather than an absolute path, so a later
|
||||
# storage-mode migration moves them again by changing one variable.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
|
||||
@@ -42,7 +42,43 @@
|
||||
# my-Foo.xml routinely holds a container called something else. Matching on the filename
|
||||
# silently pushes the wrong template, or nothing at all.
|
||||
#
|
||||
# USAGE
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# Only in NORMAL state. FALLBACK_STATE_FILE is read before anything is pushed or removed, and
|
||||
# any other state refuses the action. A push during a live failover would deploy a second copy
|
||||
# of a container the partner is currently running on our behalf; a remove would delete the one
|
||||
# doing the covering.
|
||||
#
|
||||
# --status is exempt from that gate, because it only reports. Refusing to answer "what is
|
||||
# deployed over there" during a failover would withhold the information precisely when it is
|
||||
# most wanted.
|
||||
#
|
||||
# Every deploy is verified stopped, and a container that will not stay stopped is an error
|
||||
# rather than a warning — see DESIGN PRINCIPLES. A second live instance against the same data
|
||||
# is the failure this whole script exists inside.
|
||||
#
|
||||
# Push and remove are explicit modes with no default. Running the script with no flag does
|
||||
# nothing; neither action can be reached by accident, and neither is a side effect of editing
|
||||
# the tier list.
|
||||
#
|
||||
# --dry-run works in every mode and touches nothing on either host — no container is built,
|
||||
# started, stopped or removed, and no template is written or deleted.
|
||||
#
|
||||
# Remove deletes the container's appdata on the partner as well. That is deliberate and is the
|
||||
# most destructive thing here; the NORMAL-state gate above is what keeps it away from a
|
||||
# partner that is mid-handback.
|
||||
#
|
||||
# CONFIGURATION
|
||||
# master.conf
|
||||
# FALLBACK_<HOST>_TIER1..N the covered container names — what --push deploys and --status
|
||||
# reports on. This script reads that list; it never edits it.
|
||||
#
|
||||
# host*.conf
|
||||
# FALLBACK_STATE_FILE overrides where fallback.sh's state is read from. Defaults to
|
||||
# STATE_DIR/fallback_state.db. A missing file reads as NORMAL,
|
||||
# which is the correct default on a host where fallback has never
|
||||
# run.
|
||||
#
|
||||
# RUNTIME MODES
|
||||
# coverage_deploy.sh --push deploy every covered container onto the partner (stopped)
|
||||
# coverage_deploy.sh --remove stop, remove, and delete the pushed template on the partner
|
||||
# coverage_deploy.sh --status report, per covered container, whether it exists there
|
||||
|
||||
Executable → Regular
+46
-10
@@ -122,6 +122,7 @@ source "$SCRIPT_DIR/../load_config.sh"
|
||||
parse_args "$@"
|
||||
|
||||
SSH_TIMEOUT=15
|
||||
BACKUP_VERIFY_MD5_TIMEOUT_MAX="${BACKUP_VERIFY_MD5_TIMEOUT_MAX:-600}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
@@ -228,6 +229,7 @@ TOTAL_CHECKED=0
|
||||
TOTAL_MATCH=0
|
||||
TOTAL_MISMATCH=0
|
||||
TOTAL_MISSING=0
|
||||
TOTAL_UNVERIFIED=0
|
||||
SHARES_WITH_ISSUES=()
|
||||
|
||||
for share in "${VERIFY_SHARES[@]}"; do
|
||||
@@ -265,6 +267,7 @@ for share in "${VERIFY_SHARES[@]}"; do
|
||||
SHARE_MATCH=0
|
||||
SHARE_MISMATCH=0
|
||||
SHARE_MISSING=0
|
||||
SHARE_UNVERIFIED=0
|
||||
|
||||
for local_file in "${SAMPLE_FILES[@]}"; do
|
||||
[[ -z "$local_file" ]] && continue
|
||||
@@ -276,19 +279,49 @@ for share in "${VERIFY_SHARES[@]}"; do
|
||||
continue
|
||||
fi
|
||||
|
||||
# Remote checksum via SSH — timeout protected
|
||||
remote_md5=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
# The path is interpolated into a remote shell command, so it must be escaped for
|
||||
# reuse as one word. A bare '$local_file' inside single quotes breaks on the first
|
||||
# apostrophe — "Frieren - Beyond Journey's End" ended the quote early, md5sum fell
|
||||
# back to reading stdin, and the empty-input hash d41d8cd9... was reported as a
|
||||
# MISMATCH against a file that is byte-identical on the remote.
|
||||
printf -v remote_q '%q' "$local_file"
|
||||
|
||||
# Existence and content are separate questions. Asking them together means a slow
|
||||
# checksum is indistinguishable from an absent file.
|
||||
remote_exists=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o StrictHostKeyChecking=no \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"md5sum '$local_file' 2>/dev/null | awk '{print \$1}'" 2>/dev/null)
|
||||
"test -f $remote_q && echo yes" 2>/dev/null </dev/null)
|
||||
|
||||
(( TOTAL_CHECKED++ ))
|
||||
|
||||
if [[ -z "$remote_md5" ]]; then
|
||||
if [[ "$remote_exists" != "yes" ]]; then
|
||||
warn "$ICON_ERROR MISSING: $(basename "$local_file")"
|
||||
(( SHARE_MISSING++ ))
|
||||
(( TOTAL_MISSING++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# md5sum of a multi-GB file cannot finish inside a connect-sized timeout. Budget by
|
||||
# size — a 5.9GB file needs ~30s and was being killed at 15s, then counted MISSING
|
||||
# even though it was present and correct.
|
||||
local_size=$(stat -c%s "$local_file" 2>/dev/null || echo 0)
|
||||
md5_timeout=$(( local_size / 52428800 + SSH_TIMEOUT ))
|
||||
(( md5_timeout > BACKUP_VERIFY_MD5_TIMEOUT_MAX )) && md5_timeout=$BACKUP_VERIFY_MD5_TIMEOUT_MAX
|
||||
|
||||
remote_md5=$(timeout "$md5_timeout" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o StrictHostKeyChecking=no \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"md5sum $remote_q 2>/dev/null | awk '{print \$1}'" 2>/dev/null </dev/null)
|
||||
|
||||
if [[ -z "$remote_md5" ]]; then
|
||||
# Present but unreadable within budget. Reporting this as a mismatch or a miss
|
||||
# would be a claim the run did not earn.
|
||||
warn "$ICON_WARN UNVERIFIED (checksum timed out after ${md5_timeout}s): $(basename "$local_file")"
|
||||
(( SHARE_UNVERIFIED++ ))
|
||||
(( TOTAL_UNVERIFIED++ ))
|
||||
elif [[ "$local_md5" == "$remote_md5" ]]; then
|
||||
log "MATCH: $(basename "$local_file")"
|
||||
(( SHARE_MATCH++ ))
|
||||
@@ -303,8 +336,8 @@ for share in "${VERIFY_SHARES[@]}"; do
|
||||
done
|
||||
|
||||
# Per-share result — only visible if issues found
|
||||
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 ]]; then
|
||||
warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH"
|
||||
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 || "$SHARE_UNVERIFIED" -gt 0 ]]; then
|
||||
warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH unverified: $SHARE_UNVERIFIED"
|
||||
SHARES_WITH_ISSUES+=("$SHARE_NAME")
|
||||
else
|
||||
echo "$SHARE_NAME — all $SHARE_MATCH files match ✅"
|
||||
@@ -325,10 +358,11 @@ echo "$ICON_VERIFY Checked: $TOTAL_CHECKED files"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
||||
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
|
||||
warn "Missing: $TOTAL_MISSING"
|
||||
[[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
|
||||
if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 || "$TOTAL_UNVERIFIED" -gt 0 ]]; then
|
||||
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
|
||||
warn "Missing: $TOTAL_MISSING"
|
||||
[[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
|
||||
[[ "$TOTAL_UNVERIFIED" -gt 0 ]] && warn "Unverified: $TOTAL_UNVERIFIED (present, checksum timed out)"
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
@@ -337,6 +371,8 @@ elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention: ${SHARES_WITH_ISSUES[*]}"
|
||||
notify "Backup verify FAILED on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" \
|
||||
"Backup Verify" "warning"
|
||||
elif [[ "$TOTAL_UNVERIFIED" -gt 0 ]]; then
|
||||
warn "Status: $TOTAL_MATCH verified, $TOTAL_UNVERIFIED could not be checksummed in time — NOT a clean run"
|
||||
else
|
||||
echo "$ICON_DONE Status: all $TOTAL_CHECKED files match across ${#VERIFY_SHARES[@]} shares ✅"
|
||||
fi
|
||||
|
||||
@@ -20,6 +20,42 @@
|
||||
# decision to notify is the exit code rather than this script parsing the text it just printed.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Silence is the normal output.
|
||||
# A report that always says something is a report nobody reads. A perfect week prints nothing
|
||||
# and notifies nothing, so anything that does appear in the Sunday report is worth the glance.
|
||||
#
|
||||
# The exit code is the decision, not the text.
|
||||
# uptime_probe.php --report exits 1 when it has something to say and 0 when it does not. This
|
||||
# script never parses the output it just printed to work out whether to notify — a report whose
|
||||
# wording changed would otherwise silently stop notifying.
|
||||
#
|
||||
# It reads; it never probes.
|
||||
# The measurements are already taken, once a minute, by Tools/uptime_probe.sh. Re-probing at
|
||||
# report time would describe Sunday morning rather than the week being reported on.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Read-only. Reads the stored history and prints; records nothing, and cannot alter the data it
|
||||
# is reporting on.
|
||||
#
|
||||
# UPTIME_PROBE_ENABLED gates the whole run — with the probe off there is no history worth
|
||||
# reporting, and this says nothing rather than reporting an empty week as a perfect one.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# uptime_report.sh the weekly read. Silent when every domain was 100%.
|
||||
#
|
||||
# Called from COFFEE_REPORT_SCRIPTS; takes no arguments and has no other mode. For live figures
|
||||
# or a per-domain table, use Tools/uptime_probe.sh --status.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
|
||||
@@ -110,10 +110,11 @@ naming every card on the Monitor grid, its width and its order, and generates th
|
||||
from it — the column ladder, the span clamps at each width, the row-height cap, and the
|
||||
compensation when a conditional card is absent. Column counts are 8/4/2/1 and spans are 1/2/4/8,
|
||||
which is what lets the board re-cut itself at any width with no holes and no hand-placed card.
|
||||
Breakpoints are arithmetic over `VV_MON_CARD_FLOOR`, never chosen by eye. `pages/monitor.php`
|
||||
carries the card bodies and nothing about where they go; moving a card is moving a line in that
|
||||
array. The page cross-checks the declaration against the cards that actually rendered and says so
|
||||
in the browser if they disagree.
|
||||
Breakpoints are arithmetic over `VV_MON_CARD_FLOOR`, never chosen by eye, and the row-height cap
|
||||
is conditional on the same arithmetic — it applies only where a rung fits one screen.
|
||||
`pages/monitor.php` carries the card bodies and nothing about where they go; moving a card is
|
||||
moving a line in that array. The page cross-checks the declaration against the cards that
|
||||
actually rendered and says so in the browser if they disagree.
|
||||
|
||||
**Caching.** Several endpoints serve from `$VV_CACHE_DIR` (`/tmp/varaverk/api`, tmpfs) rather than hitting live
|
||||
APIs on every page view, refreshed by `Tools/api_cache_writer.sh`. `?live=1` bypasses the
|
||||
|
||||
@@ -8,10 +8,34 @@
|
||||
// Reads ai_log_fixtures.txt and this host's /var/log/syslog*. Files nothing, writes nothing,
|
||||
// and calls no part of the sweep beyond vv_ai_syslog_findings() on lines it supplies itself.
|
||||
//
|
||||
// EXIT
|
||||
// 0 when every fixture is recognised as written. Precision findings are reported but never
|
||||
// fail the run: what a real syslog contains is a fact about the machine, not about the
|
||||
// patterns, and a genuinely failing disk should not turn this into a red test.
|
||||
// DESIGN PRINCIPLES
|
||||
// Only recall can fail the run.
|
||||
// A missed fixture is a fact about the patterns and is always a defect. A precision hit is
|
||||
// a fact about this machine — a genuinely failing disk should not turn this red, and if it
|
||||
// did, the honest fix would be to stop having a failing disk rather than to edit a pattern.
|
||||
//
|
||||
// Precision is replayed against real history, not a sample.
|
||||
// The patterns that cause damage are the ones matching ordinary operation, and ordinary
|
||||
// operation is exactly what a hand-written fixture file never contains. Only the machine's
|
||||
// own syslog can show what a pattern fires on when nothing is wrong.
|
||||
//
|
||||
// The sweep is never invoked, only its matcher.
|
||||
// vv_ai_syslog_findings() is called on lines this file supplies. Running the real sweep
|
||||
// would file findings, and a test that has to be cleaned up afterwards stops being run.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Read-only. Reads ai_log_fixtures.txt and this host's /var/log/syslog*; files no finding,
|
||||
// writes no store, and touches no conf beyond the enable flag.
|
||||
//
|
||||
// Exit 0 when every fixture is recognised as written. Precision findings are reported but
|
||||
// never fail the run — see DESIGN PRINCIPLES.
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// php ai_log_check.php both checks
|
||||
// php ai_log_check.php --recall fixtures only
|
||||
// php ai_log_check.php --precision replay this host's syslog history only
|
||||
//
|
||||
// Not scheduled, and deliberately so. Run it after touching VV_AI_SYSLOG_PATTERNS.
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
require_once dirname(__DIR__) . '/include/ai_repair.php';
|
||||
|
||||
|
||||
@@ -15,6 +15,47 @@
|
||||
#
|
||||
# Run it after touching VV_AI_SYSLOG_PATTERNS. Nothing here writes: no findings are filed, no
|
||||
# conf is read for anything but the enable flag, and the sweep is never invoked.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# A wrapper. The work is in ai_log_check.php, next to the sweep's own matcher — the patterns and
|
||||
# vv_ai_syslog_findings() live in include/ai_repair.php, and a bash reimplementation of the
|
||||
# matching would be a second set of regexes to keep in step with the first.
|
||||
#
|
||||
# Flags are forwarded verbatim; nothing is interpreted here.
|
||||
#
|
||||
# Not scheduled and in no orchestrator. This is a development check that runs when the patterns
|
||||
# change, not on a timer — nothing on the running system depends on it.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Two checks, because the failure modes are opposite.
|
||||
# Recall catches a pattern that stopped matching; precision catches one that matches too much.
|
||||
# A single test would catch one and silently permit the other, and the second is the one that
|
||||
# fills the findings store with noise until the operator stops reading it.
|
||||
#
|
||||
# Precision is measured against this machine's real history.
|
||||
# A hand-written fixture file can show that a pattern matches what it should. Only a real
|
||||
# syslog can show what it also matches when nothing is wrong.
|
||||
#
|
||||
# Only recall fails the run.
|
||||
# What a real syslog contains is a fact about the machine, not about the patterns. A genuinely
|
||||
# failing disk should not turn this red.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Read-only. No finding is filed, no store is written, and the repair sweep itself is never run
|
||||
# — only its matcher, on lines this check supplies.
|
||||
#
|
||||
# Safe to run on a live host at any time, including one that is currently faulting. It observes
|
||||
# the syslog it replays and changes nothing about it.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
|
||||
@@ -13,16 +13,39 @@
|
||||
// php Tools/conf_widget_check.php assertions, then the live summary
|
||||
// php Tools/conf_widget_check.php --list every live field and its inferred control
|
||||
//
|
||||
// WHY IT ASSERTS AGAINST SNIPPETS AND NOT THE LIVE CONF
|
||||
// The live conf is the thing being described, so it cannot also be the thing that proves the
|
||||
// description right — an inference rule that silently stopped matching would keep passing as
|
||||
// the conf drifted to suit it. The snippets are frozen copies of each convention as written,
|
||||
// so a rule change that breaks one shows up here rather than as a wrong control on a page.
|
||||
// DESIGN PRINCIPLES
|
||||
// Assertions run against snippets, never against the live conf.
|
||||
// The live conf is the thing being described, so it cannot also be the thing that proves
|
||||
// the description right — an inference rule that silently stopped matching would keep
|
||||
// passing as the conf drifted to suit it. The snippets are frozen copies of each
|
||||
// convention as written, so a rule change that breaks one shows up here rather than as a
|
||||
// wrong control on a page.
|
||||
//
|
||||
// WHAT AN INFERENCE IS NOT
|
||||
// Consistent with confform.php, none of this validates. A number field carrying min and max is
|
||||
// a courtesy to whoever is typing, not a promise the value is sensible — the consuming script
|
||||
// still owns that question.
|
||||
// An inference is a drawing decision, not a validation.
|
||||
// Consistent with confform.php, none of this validates. A number field carrying min and
|
||||
// max is a courtesy to whoever is typing, not a promise the value is sensible — the
|
||||
// consuming script still owns that question.
|
||||
//
|
||||
// The live pass reports, it does not assert.
|
||||
// What this host's master.conf infers to is a description of that file, not a verdict on
|
||||
// it. Turning the live summary into pass/fail would make an unusual but legitimate
|
||||
// setting look like a defect.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Read-only. Parses conf and reports; writes no conf, no store and no page.
|
||||
//
|
||||
// Exits non-zero only when a snippet assertion fails, so it can gate a commit without a real
|
||||
// conf's contents ever being able to break the build.
|
||||
//
|
||||
// Never renders. It reports which control would be drawn; the drawing stays in confform.php,
|
||||
// so this cannot disagree with the page by construction.
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// php Tools/conf_widget_check.php assertions, then the live summary
|
||||
// php Tools/conf_widget_check.php --list every live field and its inferred control
|
||||
//
|
||||
// Hand-run. Not scheduled and in no orchestrator — run it after touching _vv_conf_widget(),
|
||||
// after adding a conf convention, or when a setting draws as the wrong control.
|
||||
//
|
||||
// DEPENDS ON
|
||||
// include/confform.php _vv_conf_parse_field_range(), vv_conf_key_is_secret()
|
||||
|
||||
@@ -5,31 +5,49 @@
|
||||
// to reach it. Generated so the assistant can answer "how do I change X" with a path through
|
||||
// the pages instead of an instruction to open master.conf.
|
||||
//
|
||||
// WHY THE ASSISTANT NEEDS THIS AT ALL
|
||||
// The retrieval index reads git-tracked files. PHP body markup is not indexed and would be
|
||||
// useless if it were — a page is a pile of divs, not a description of itself — so the assistant
|
||||
// has never had any way to know the UI exists. It could name a conf key and nothing more.
|
||||
// pages/readme/*.md is the one directory the chunker classifies as kind='ui', which is why the
|
||||
// output lands there and not in docs/.
|
||||
//
|
||||
// WHY IT IS GENERATED
|
||||
// A hand-written map is a second description of the pages, and the moment a card moves it
|
||||
// starts lying with total confidence — which is worse than saying nothing, because the
|
||||
// assistant will repeat it. Everything here is derived from the same registries the pages
|
||||
// themselves are built from: VV_SCRIPT_CONF_SECTIONS for what the Scheduler shows per script,
|
||||
// VV_UI_SECTION_SURFACES for the pages that show sections by subject, and the conf files for
|
||||
// the settings and their controls.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// Hand-run, and re-run after adding a conf section, a script mapping or a settings surface.
|
||||
// Writes exactly one file and nothing else.
|
||||
// Reads the section registries and the conf files, resolves each setting to the page and card
|
||||
// that renders it, and writes the whole map in one pass. Nothing is merged with what is
|
||||
// already there — the output is derived entirely from the registries, so a stale entry cannot
|
||||
// survive a rebuild.
|
||||
//
|
||||
// php Tools/ui_map_build.php write the map
|
||||
// php Tools/ui_map_build.php --check report what it would change, write nothing
|
||||
// DESIGN PRINCIPLES
|
||||
// The assistant cannot see the UI any other way.
|
||||
// The retrieval index reads git-tracked files. PHP body markup is not indexed and would be
|
||||
// useless if it were — a page is a pile of divs, not a description of itself — so the
|
||||
// assistant has never had any way to know the UI exists. It could name a conf key and
|
||||
// nothing more. pages/readme/*.md is the one directory the chunker classifies as
|
||||
// kind='ui', which is why the output lands there and not in docs/.
|
||||
//
|
||||
// Only sections that are genuinely reachable are listed. A section no page renders is reported
|
||||
// at the end as unreachable rather than silently omitted — a setting with no route is a real
|
||||
// finding, and the map is the only place that would notice.
|
||||
// Generated, because a hand-written map lies with confidence.
|
||||
// A second description of the pages starts being wrong the moment a card moves, and that
|
||||
// is worse than saying nothing, because the assistant will repeat it. Everything here is
|
||||
// derived from the same registries the pages themselves are built from:
|
||||
// VV_SCRIPT_CONF_SECTIONS for what the Scheduler shows per script, VV_UI_SECTION_SURFACES
|
||||
// for the pages that show sections by subject, and the conf files for the settings and
|
||||
// their controls.
|
||||
//
|
||||
// An unreachable section is reported, never dropped.
|
||||
// A section no page renders is listed at the end rather than silently omitted. A setting
|
||||
// with no route through the UI is a real finding, and this map is the only thing that
|
||||
// would ever notice.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Writes exactly one file, pages/readme/ui-map.md, and nothing else. No conf is modified, no
|
||||
// page is touched, and the registries it reads are only read.
|
||||
//
|
||||
// --check reports what would change and writes nothing, so the map can be verified current in
|
||||
// a commit without regenerating it.
|
||||
//
|
||||
// Generated output only. Nothing hand-edited belongs in ui-map.md — an edit there is lost on
|
||||
// the next run, which is the correct behaviour for a derived file and the reason the header
|
||||
// says so.
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// php Tools/ui_map_build.php write the map
|
||||
// php Tools/ui_map_build.php --check report what it would change, write nothing
|
||||
//
|
||||
// Hand-run. Re-run after adding a conf section, a script mapping or a settings surface.
|
||||
//
|
||||
// DEPENDS ON
|
||||
// include/confform.php the section registries, the parser, and the inferred controls
|
||||
|
||||
@@ -235,6 +235,27 @@ body.vv-fullscreen #displaybox { padding-left: 1rem !important; padding-top: .5r
|
||||
/* Hide scrollbars on any nested scrollable div inside monitor cards */
|
||||
#vv-monitor .vv-card div::-webkit-scrollbar { display: none; }
|
||||
|
||||
/* System card, held to the Network card's height.
|
||||
At eight columns these two share a row (system 1 + ups 1 + cpu 2 + memory 2 + network 2), so
|
||||
the grid already stretches them to the same height and nothing here is needed. Below eight the
|
||||
board re-cuts into rows of four: System lands in row 1 with UPS and CPU, Network in row 2 with
|
||||
Memory, and once rows are sized to content (vv_mon_rung_overflows) the two are free to differ.
|
||||
System is the taller — it carries a host block and a Varaverk block — so it is the one capped.
|
||||
|
||||
222px is Network measured rather than guessed: 2 border + 24 padding + 25 h3 + 61 header block
|
||||
+ 110 canvas. The canvas is a fixed 110px, which is what makes the number worth writing down —
|
||||
most of Network's height cannot drift. What can is the IP list (LAN/EXT/TS), roughly 18px a row,
|
||||
so a host resolving fewer of the three leaves System slightly tall against it.
|
||||
|
||||
Two ids on the selectors, to outrank #vv-monitor .vv-card from the auto-rows hatch above. That
|
||||
hatch lifts overflow off every card so content-sized rows can work; this card is the one place
|
||||
the clamps have to go back on, or a max-height with overflow:visible would draw straight
|
||||
through the card's own border. */
|
||||
@media (max-width: 1383px), (max-height: 700px) {
|
||||
#vv-monitor #vv-system { max-height: 222px; overflow: hidden; }
|
||||
#vv-monitor #vv-system > div { overflow-y: auto; min-height: 0; }
|
||||
}
|
||||
|
||||
/* Dynamic row heights capped per screen tier — rows size to content, never exceed the cap.
|
||||
minmax(0, Xpx): track is content-driven but capped; align-items:stretch makes all cards
|
||||
in a row fill the track, so short cards (Pools, Watchdog) match tall ones (Array).
|
||||
@@ -264,10 +285,13 @@ body.vv-fullscreen #displaybox { padding-left: 1rem !important; padding-top: .5r
|
||||
worked, while landscape (851x393) missed it and inherited the four-row cap — 58px cards with
|
||||
overflow:hidden and scrollbars disabled. Nothing was visible and nothing said so.
|
||||
|
||||
The width half is the two-column rung from monitor_board.php (below 696px). It is written as a
|
||||
literal here because a media query cannot read a custom property; if VV_MON_CARD_FLOOR changes,
|
||||
this number and the AI row's two below are the only three places that have to follow it. */
|
||||
@media (max-width: 695px), (max-height: 700px) {
|
||||
Height only now. The width half was a literal 695 that had to be kept in step with
|
||||
VV_MON_CARD_FLOOR by hand; monitor_board.php emits these same three rules for every rung whose
|
||||
row count exceeds VV_MON_ROWS_PER_SCREEN — see vv_mon_rung_overflows() — which covers that band
|
||||
and the four-column one above it from the arithmetic instead. What is left here is the case the
|
||||
ladder cannot see: a window wide enough for a rung that does fit a screen, on a screen too short
|
||||
to give those rows a usable height. */
|
||||
@media (max-height: 700px) {
|
||||
#vv-monitor { grid-auto-rows: auto; }
|
||||
#vv-monitor .vv-card { overflow: visible; }
|
||||
#vv-monitor .vv-card > div { overflow-y: visible; min-height: auto; }
|
||||
|
||||
@@ -1363,6 +1363,15 @@ vv_ai_profiles_script();
|
||||
profile: profile,
|
||||
question: q,
|
||||
history: JSON.stringify(sendable()),
|
||||
// Where the caller is standing, so a question can say "this setting" and mean it. The
|
||||
// backend whitelists it and passes it to the worker, which turns it into a location
|
||||
// line; a page that sets no scope sends '' and the worker omits that line.
|
||||
//
|
||||
// Keep this here. The scope-aware dock in 185abdb sent it from the page's own ask call;
|
||||
// 8eeb4c3 folded that dock into this component and the line did not come with it, so
|
||||
// for the whole of that window Scheduler computed a scope, re-read it at send time, and
|
||||
// the model never saw it. Nothing errored — the answer just arrived ungrounded.
|
||||
scope: (typeof o.scope === 'function' ? o.scope() : (o.scope || '')),
|
||||
kind: (PROFILES[profile].kind && kindEl) ? kindEl.value : '',
|
||||
// A checkbox where the page offers one, otherwise whatever the page decides from the
|
||||
// profile, otherwise on. The Scheduler reasons only when diagnosing: working out what a
|
||||
|
||||
@@ -13,9 +13,14 @@
|
||||
// Powers of two, all the way down.
|
||||
// Column counts are 8, 4, 2, 1 and every span is 1, 2, 4 or 8. That pairing is what makes
|
||||
// the board tile with no holes at every width without a single hand-placed card: 32 span
|
||||
// units divide into 8 columns as 4 rows, into 4 as 8 rows, into 2 as 16, into 1 as 32.
|
||||
// A rung of 6 or 10 columns does not divide a span of 4, which is why the previous ladder
|
||||
// needed a per-breakpoint override for every wide card and still left holes.
|
||||
// units divide into 8 columns as 4 rows and into 4 as 8. A rung of 6 or 10 columns does
|
||||
// not divide a span of 4, which is why the previous ladder needed a per-breakpoint
|
||||
// override for every wide card and still left holes.
|
||||
//
|
||||
// Below four columns the arithmetic stops being a division. Spans clamp to the column
|
||||
// count, so a span-4 card contributes 2 at two columns and 1 at one, and the board is 13
|
||||
// rows and then 18 rather than the 16 and 32 a naive division predicts. It still tiles
|
||||
// without holes, for a different reason: a clamped card fills its row outright.
|
||||
//
|
||||
// Breakpoints are arithmetic, not taste.
|
||||
// A column count is viable exactly when the cards still fit:
|
||||
@@ -50,9 +55,15 @@
|
||||
// — the one array every other function here reads. Change the layout by editing
|
||||
// that declaration, never by editing what follows.
|
||||
// Geometry vv_mon_rung_min(), vv_mon_rung_rows(), vv_mon_rung_rowdiv(),
|
||||
// vv_mon_absent_class()
|
||||
// vv_mon_rung_overflows(), vv_mon_absent_class()
|
||||
// — derive the column ladder and each card's span from the declaration.
|
||||
// Emitters vv_mon_board_css(), vv_mon_absorb_css(), vv_mon_board_js()
|
||||
// vv_mon_rung_overflows() is the one that asks whether a rung is already taller
|
||||
// than one screen, which is when the row-height cap stops helping and starts
|
||||
// clipping.
|
||||
// Emitters vv_mon_board_css(), vv_mon_absorb_css(), vv_mon_autorows_css(),
|
||||
// vv_mon_board_js()
|
||||
// — vv_mon_autorows_css() is the hatch for an overflowing rung: rows sized to
|
||||
// content, and the overflow clamps lifted off the cards so they can use it.
|
||||
// Validation vv_mon_board_check()
|
||||
// — the only one that reports rather than renders; see OPERATIONAL SAFEGUARDS.
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
@@ -74,9 +85,13 @@ const VV_MON_PAD = 20; // horizontal padding the Unraid page wrapper takes off
|
||||
// The column ladder, widest first.
|
||||
const VV_MON_RUNGS = [8, 4, 2, 1];
|
||||
|
||||
// Cap on how many rows may share one screen height. The row-height cap divides the viewport by
|
||||
// this, so a card can never grow past a quarter of the screen and push the rest off it. Rungs
|
||||
// with fewer rows than this divide by their own row count instead and fill the screen.
|
||||
// Cap on how many rows may share one screen height — and the line between two behaviours, not one
|
||||
// rule. A rung with no more rows than this fits a screen: the row-height cap divides the viewport
|
||||
// by the rung's own row count, or by this when the rung has more, so the board fills the screen
|
||||
// and no card grows past its share. A rung with MORE rows than this cannot fit however its rows
|
||||
// are sized, so the cap is dropped there and rows size to content instead — see
|
||||
// vv_mon_rung_overflows(). Capping a board that scrolls regardless buys nothing and clips every
|
||||
// card to pay for it. This board is 4 rows at eight columns and 8 at four, so both cases are live.
|
||||
const VV_MON_ROWS_PER_SCREEN = 4;
|
||||
|
||||
// ── The board ─────────────────────────────────────────────────────────────────────────────────
|
||||
@@ -164,6 +179,25 @@ function vv_mon_rung_rowdiv(int $cols): int {
|
||||
return min(vv_mon_rung_rows($cols), VV_MON_ROWS_PER_SCREEN);
|
||||
}
|
||||
|
||||
// Whether this rung is already taller than one screen. The row-height cap means "no card taller
|
||||
// than 1/VV_MON_ROWS_PER_SCREEN of the screen", which is only worth paying for on a rung that
|
||||
// fits a screen — there it is what keeps the board from scrolling. On a rung with more rows than
|
||||
// the cap the page scrolls no matter what, so the cap buys nothing and only cuts each card's
|
||||
// content off. At four columns this board is eight rows: an 11" tablet in landscape was being
|
||||
// given 126px cards on a board two screens tall.
|
||||
function vv_mon_rung_overflows(int $cols): bool {
|
||||
return vv_mon_rung_rows($cols) > VV_MON_ROWS_PER_SCREEN;
|
||||
}
|
||||
|
||||
// Rows sized by content instead of by the cap, for a rung that overflows the screen anyway.
|
||||
// Mirrors the phone hatch in css/varaverk.css: the cards have to give up overflow:hidden too, or
|
||||
// they keep clipping at a height nothing is constraining any more.
|
||||
function vv_mon_autorows_css(): string {
|
||||
return '#vv-monitor{grid-auto-rows:auto;}'
|
||||
. '#vv-monitor .vv-card{overflow:visible;}'
|
||||
. '#vv-monitor .vv-card>div{overflow-y:visible;min-height:auto;}';
|
||||
}
|
||||
|
||||
// The class the page puts on #vv-monitor when a card is not rendered, so the generated absorb
|
||||
// rules can fire. Derived from the id on both sides — PHP writes the rule, JS writes the class —
|
||||
// so the two can never drift apart by a typo.
|
||||
@@ -216,6 +250,7 @@ function vv_mon_board_css(): string {
|
||||
$out = '';
|
||||
|
||||
$out .= "#vv-monitor{--vv-cols:$top;--vv-rowdiv:" . vv_mon_rung_rowdiv($top) . ";}\n";
|
||||
if (vv_mon_rung_overflows($top)) $out .= vv_mon_autorows_css() . "\n";
|
||||
|
||||
$order = 0;
|
||||
foreach ($cards as $card) {
|
||||
@@ -245,6 +280,7 @@ function vv_mon_board_css(): string {
|
||||
if ($wide) $out .= implode(',', $wide) . "{--vv-sp:$cols;}";
|
||||
}
|
||||
|
||||
if (vv_mon_rung_overflows($cols)) $out .= vv_mon_autorows_css();
|
||||
$out .= vv_mon_absorb_css($cols);
|
||||
$out .= "}\n";
|
||||
}
|
||||
|
||||
@@ -1079,6 +1079,7 @@ function vvPollMonitor(live) {
|
||||
<span style="color:#444;">Running</span> <span style="color:#888;">${_runningCtrs} ctr${_runningCtrs !== 1 ? 's' : ''}${_runningVMs > 0 ? ` · ${_runningVMs} VM` : ''}</span>
|
||||
<span style="color:#444;">Version</span> <span style="color:#3a3a3a;">${ver}</span>
|
||||
</div>
|
||||
|
||||
<!-- Varaverk's own figures, ruled off from the host's. Same grid so the labels line up,
|
||||
separate block so it reads as a different subject rather than more of the same. -->
|
||||
<div style="border-top:1px solid #1e1e1e;margin-top:8px;padding-top:7px;">
|
||||
|
||||
@@ -10,6 +10,22 @@ a minute, so polling faster could not make it newer — it only decides how soon
|
||||
**Cards you do not have do not appear.** No GPU, no UPS, no VMs, no partner — the card is absent
|
||||
rather than showing zeros. An empty card would be a permanent reminder of nothing.
|
||||
|
||||
**The board re-flows on a narrow screen.** On a tablet or a phone the cards regroup into fewer,
|
||||
wider columns rather than shrinking in place. Nothing is hidden and nothing is dropped — the same
|
||||
cards are there in the same order, cut into different rows.
|
||||
|
||||
Below the full-width layout the page scrolls as a whole instead of each card scrolling inside
|
||||
itself. A board that is taller than the screen has to scroll somewhere, and scrolling the page
|
||||
once is better than being handed a screen of cards that are each too short to read.
|
||||
|
||||
**Cards in the same row share a height.** They line up along the bottom, so a card with little to
|
||||
say carries empty space under it. That space means its neighbour is the taller card — not that a
|
||||
reading is missing.
|
||||
|
||||
The System card is the one exception. On a narrow screen it stops after Version and the Varaverk
|
||||
figures below scroll, so that the host itself — hostname, clock, array state, uptime — is what
|
||||
you get without scrolling anything.
|
||||
|
||||
---
|
||||
|
||||
## Reference — System, power, CPU, memory, network
|
||||
|
||||
@@ -37,6 +37,12 @@
|
||||
// Step 2: auto-populate + guide + checklist.
|
||||
// master.conf pull (for partner servers) lives in the checklist, not here.
|
||||
|
||||
// The assistant is a bonus on this page, never a dependency: vv_ai_ui_on() is false on a node
|
||||
// with AI off or no model reachable, which is the normal state of the fresh install this wizard
|
||||
// exists to serve. The card simply is not rendered and setup proceeds exactly as before.
|
||||
require_once dirname(__DIR__) . '/include/ai_chat.php';
|
||||
if (vv_ai_ui_on()) vv_ai_chat_assets();
|
||||
|
||||
$detectedHostname = vv_get_hostname();
|
||||
|
||||
// ── Identity already on disk? ────────────────────────────────────────────────────────────────
|
||||
@@ -313,6 +319,21 @@ hr.vv-hr { border: none; border-top: 1px solid #1e1e1e; margin: 22px 0; }
|
||||
<div id="vv-onboard-panel"></div>
|
||||
<div id="vv-done-banner"></div>
|
||||
|
||||
<?php if (vv_ai_ui_on()): ?>
|
||||
<hr class="vv-hr">
|
||||
<div id="vv-su-ai-card">
|
||||
<?php vv_ai_chat_markup('vv-su-ai', [
|
||||
'profile' => 'varaverk',
|
||||
'compact' => true,
|
||||
'title' => 'Setup assistant',
|
||||
'height' => '260px',
|
||||
'tall' => '460px',
|
||||
'empty' => 'Stuck on a step? Ask what it wants and why — "what is the API key for", '
|
||||
. '"why does SSH need the other host", "what does populate actually do".',
|
||||
]); ?>
|
||||
</div>
|
||||
<?php endif; ?>
|
||||
|
||||
<div style="margin-top:18px;text-align:right;">
|
||||
<a href="#" id="vv-exit-link" onclick="vvGoNext(event)"
|
||||
style="font-size:11px;color:#3a3a3a;text-decoration:none;">Skip →</a>
|
||||
@@ -443,6 +464,7 @@ function vvShowStep2(redirect, apiKey) {
|
||||
_vvRedirect = redirect || '?tab=scheduler';
|
||||
document.getElementById('vv-step1').style.display = 'none';
|
||||
document.getElementById('vv-step2').style.display = 'block';
|
||||
vvSetupAiInit(); // card is visible now, so it can measure itself correctly
|
||||
if (apiKey && apiKey.ok) {
|
||||
const btn = document.getElementById('vv-key-btn');
|
||||
const status = document.getElementById('vv-key-status');
|
||||
@@ -676,6 +698,47 @@ function vvStartPhase2Watch(sinceMs) {
|
||||
_vvPartnerPoll = setInterval(check, 6000);
|
||||
}
|
||||
|
||||
// ── Setup assistant ───────────────────────────────────────────────────────────
|
||||
// Scoped to the step the operator is actually stuck on, so "what does this want" resolves without
|
||||
// them naming it. The worker turns a non-empty scope into a line saying what is open in the
|
||||
// WebGUI; the varaverk profile holds no scoped_log capability, so a checklist id never triggers a
|
||||
// log lookup and cannot produce a "log missing" note for something that was never a script.
|
||||
//
|
||||
// Re-read at send time, not captured: the checklist re-polls while the page is open and the step
|
||||
// they are on can change between opening the composer and pressing Ask.
|
||||
//
|
||||
// First not-ok, not-deferred item — deferred means "I have decided to skip this", which is not
|
||||
// where they are stuck. Falls back to the bare page when the list is clean or has not loaded.
|
||||
function vvSetupScope() {
|
||||
const d = _vvLastChecklist;
|
||||
if (!d || !Array.isArray(d.items)) return 'Setup';
|
||||
const stuck = d.items.find(i => !i.ok && !i.deferred);
|
||||
return stuck && stuck.id ? 'Setup/' + stuck.id : 'Setup';
|
||||
}
|
||||
|
||||
// Mounted when Step 2 is revealed, not at parse time. This is the only page where the card starts
|
||||
// inside a display:none block, and a chat that measured itself while hidden would come up wrong
|
||||
// with nothing to correct it — the component carries no resize observer. Idempotent because
|
||||
// vvShowStep2 is reachable more than once.
|
||||
let _vvSetupAiUp = false;
|
||||
function vvSetupAiInit() {
|
||||
if (_vvSetupAiUp) return;
|
||||
if (typeof VvAiChat !== 'function') return;
|
||||
if (!document.getElementById('vv-su-ai-chat')) return; // AI off — card was never rendered
|
||||
_vvSetupAiUp = true;
|
||||
VvAiChat({
|
||||
prefix: 'vv-su-ai',
|
||||
profile: 'varaverk',
|
||||
// Pinned, for the same reason the Partnership card pins it: resuming whatever thread was last
|
||||
// touched anywhere in the UI could land a first-run operator mid-way through someone else's
|
||||
// Scheduler conversation, on a card with no picker to get back from.
|
||||
resumeProfile: 'varaverk',
|
||||
scope: vvSetupScope,
|
||||
scopeLabel: 'Setup',
|
||||
empty: 'Stuck on a step? Ask what it wants and why.',
|
||||
});
|
||||
}
|
||||
|
||||
function vvLoadChecklist() {
|
||||
fetch('/plugins/varaverk/api/checklist.php?_=' + Date.now())
|
||||
.then(r => r.json()).then(d => {
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
# rsync would then stop nothing before copying a live database. They are excluded by name and
|
||||
# must stay excluded.
|
||||
#
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# A docker that cannot be read is not a docker with nothing in it.
|
||||
# If `docker ps -a` fails, times out, or returns nothing on a host that is meant to run
|
||||
# containers, EVERY name looks missing and one run would strike the entire configuration.
|
||||
@@ -34,7 +35,24 @@
|
||||
# verified by sourcing the result in a subshell and confirming the key still parses as an
|
||||
# array with exactly one fewer element.
|
||||
#
|
||||
# USAGE
|
||||
# Each removal is its own verified rewrite.
|
||||
# Several names can reach the limit in one run, and each is removed and re-verified
|
||||
# independently rather than batched into a single edit. A rewrite that fails verification
|
||||
# therefore costs that one entry, not every entry the run intended to prune.
|
||||
#
|
||||
# --dry-run records no strike. A dry run that advanced the counter would eventually prune
|
||||
# through repetition alone, which is the opposite of what it is for.
|
||||
#
|
||||
# CONFIGURATION
|
||||
# master.conf
|
||||
# CONF_PRUNE_STRIKE_LIMIT consecutive runs a name must be missing before it is removed.
|
||||
# Seeing the container again resets its strike to zero immediately,
|
||||
# so a rebuild costs one strike at most.
|
||||
#
|
||||
# The keys this may prune are an explicit allow list in the script, deliberately not a conf
|
||||
# value — see DESIGN PRINCIPLES for why a pattern is the wrong shape here.
|
||||
#
|
||||
# RUNTIME MODES
|
||||
# conf_container_prune.sh strike, and prune anything at the limit
|
||||
# conf_container_prune.sh --dry-run report what would be struck and pruned, write nothing
|
||||
# conf_container_prune.sh --status show current strikes and stop
|
||||
|
||||
@@ -270,7 +270,17 @@ log "$ICON_SUCCESS Internet reachable"
|
||||
# ==============================================================================================
|
||||
if [[ -n "$DDNS_DOMAIN" ]] && [[ -n "$DDNS_CONTAINER" ]]; then
|
||||
PUBLIC_IP=$(curl -sf --max-time 5 https://ifconfig.me 2>/dev/null | tr -d '[:space:]')
|
||||
DNS_IP=$(dig +short "$DDNS_DOMAIN" @1.1.1.1 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1)
|
||||
# dig's failure text names the resolver it could not reach (";; communications error to
|
||||
# 1.1.1.1#53: timed out"), so scraping its output for an address yields the server, not the
|
||||
# answer — a guaranteed mismatch that restarts DDNS over what is only a DNS timeout. Trust
|
||||
# the exit status, and anchor the match so only a bare answer line counts. NXDOMAIN exits 0
|
||||
# with no output and correctly falls through to the "could not resolve" branch below.
|
||||
if DNS_ANSWER=$(dig +short "$DDNS_DOMAIN" @1.1.1.1 2>/dev/null); then
|
||||
DNS_IP=$(printf '%s\n' "$DNS_ANSWER" \
|
||||
| grep -Eox '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1)
|
||||
else
|
||||
DNS_IP=""
|
||||
fi
|
||||
|
||||
if [[ -z "$PUBLIC_IP" ]]; then
|
||||
warn "Could not determine public IP — skipping DDNS check"
|
||||
|
||||
@@ -2273,8 +2273,10 @@ acquire_rsync_lock() {
|
||||
# ==============================================================================================
|
||||
# Generic get/set for flat "key<sep>value" state files (one entry per line) — the pattern
|
||||
# every watchdog's strike-tracking and state-file logic was independently reimplementing.
|
||||
# grep -v + tempfile-swap on write, not sed -i in place — avoids sed treating a key containing
|
||||
# regex metacharacters (container names, etc.) as part of the substitution pattern.
|
||||
# Keys are matched as literal prefixes via awk substr(), never as regexes. A key is often a
|
||||
# media file path, and a release tag like [Bluray-1080p] is a valid-looking bracket expression
|
||||
# holding the reversed range 1-0 — grep -E rejects it, exits 2, and the tempfile-swap then
|
||||
# commits an empty file, silently wiping every other entry.
|
||||
#
|
||||
# Usage: wd_state_get "$key" "$file" [sep=:]
|
||||
# wd_state_set "$key" "$value" "$file" [sep=:]
|
||||
@@ -2284,14 +2286,24 @@ acquire_rsync_lock() {
|
||||
# keep their own thin same-named wrapper around these rather than changing call sites.
|
||||
wd_state_get() {
|
||||
local key="$1" file="$2" sep="${3:-:}"
|
||||
grep -E "^${key}${sep}" "$file" 2>/dev/null | cut -d"$sep" -f2-
|
||||
[[ -f "$file" ]] || return 0
|
||||
awk -v pfx="${key}${sep}" \
|
||||
'substr($0, 1, length(pfx)) == pfx { print substr($0, length(pfx) + 1); exit }' \
|
||||
"$file" 2>/dev/null
|
||||
}
|
||||
|
||||
# Returns 1 without touching the state file if the rewrite fails, so a read error costs the
|
||||
# caller one update rather than the whole file.
|
||||
wd_state_set() {
|
||||
local key="$1" value="$2" file="$3" sep="${4:-:}"
|
||||
grep -vE "^${key}${sep}" "$file" 2>/dev/null > "${file}.tmp"
|
||||
echo "${key}${sep}${value}" >> "${file}.tmp"
|
||||
mv "${file}.tmp" "$file"
|
||||
local tmp="${file}.tmp"
|
||||
: > "$tmp" || return 1
|
||||
if [[ -f "$file" ]]; then
|
||||
awk -v pfx="${key}${sep}" \
|
||||
'substr($0, 1, length(pfx)) != pfx' "$file" > "$tmp" || { rm -f "$tmp"; return 1; }
|
||||
fi
|
||||
echo "${key}${sep}${value}" >> "$tmp"
|
||||
mv "$tmp" "$file"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
@@ -2722,6 +2734,100 @@ check_delete_size_threshold() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Applies a per-run delete budget instead of refusing the whole run. Reads "<size>\t<ctime>\t<path>"
|
||||
# from $1, writes the paths that fit inside $3 GB to $2 (oldest ctime first), and reports what was
|
||||
# held back in _BUDGET_*.
|
||||
#
|
||||
# A cap that aborts cannot drain a backlog larger than itself. Every run re-finds the same lump,
|
||||
# exceeds the same ceiling and stops, so the queue is stuck permanently and only a human with
|
||||
# --i-know-what-im-doing can clear it. Observed 2026-08-24..26: a partnership merge restored ten
|
||||
# months of superseded Radarr files in one night, 126GB against a 100GB cap, and the daily
|
||||
# orchestrator then failed three nights running on a queue it could never drain. Budgeting keeps
|
||||
# the ceiling meaning exactly what it meant — no single run removes more than max_gb — while the
|
||||
# backlog clears over consecutive runs with nobody touching it.
|
||||
#
|
||||
# Strict oldest-first, stopping at the first file that does not fit rather than skipping it for a
|
||||
# smaller one further down. Skipping would drain more per run but lets a large file be passed over
|
||||
# indefinitely; stopping guarantees every deferred file is newer than everything already removed,
|
||||
# so nothing can starve.
|
||||
#
|
||||
# One case is deliberately left for a human: a single file larger than the entire budget can never
|
||||
# fit, so _BUDGET_STUCK is set and the caller says so rather than looping forever on it.
|
||||
apply_delete_budget() {
|
||||
local src="$1" dst="$2" max_gb="$3"
|
||||
local max_bytes tab
|
||||
tab=$(printf '\t')
|
||||
max_bytes=$(awk "BEGIN {printf \"%d\", $max_gb * 1073741824}")
|
||||
|
||||
_BUDGET_KEPT_BYTES=0; _BUDGET_KEPT_COUNT=0
|
||||
_BUDGET_DEFERRED_BYTES=0; _BUDGET_DEFERRED_COUNT=0
|
||||
_BUDGET_STUCK=""
|
||||
|
||||
: > "$dst"
|
||||
[[ -s "$src" ]] || return 0
|
||||
|
||||
local out
|
||||
out=$(sort -t"$tab" -k2,2n "$src" | awk -F'\t' -v max="$max_bytes" -v dst="$dst" '
|
||||
{
|
||||
if (stop) { defb += $1; defc++; next }
|
||||
if (used + $1 > max) { stop = 1
|
||||
if (kept == 0) stuck = $3
|
||||
defb += $1; defc++; next }
|
||||
used += $1; kept++
|
||||
print $3 > dst
|
||||
}
|
||||
END { printf "%d\t%d\t%d\t%d\t%s", used+0, kept+0, defb+0, defc+0, stuck }
|
||||
')
|
||||
IFS=$'\t' read -r _BUDGET_KEPT_BYTES _BUDGET_KEPT_COUNT \
|
||||
_BUDGET_DEFERRED_BYTES _BUDGET_DEFERRED_COUNT _BUDGET_STUCK <<< "$out"
|
||||
}
|
||||
|
||||
# Asks the local generation model to characterise a block of evidence, for the one line a human
|
||||
# would otherwise have to derive by hand. Prints the note on stdout and returns 0; returns 1 and
|
||||
# prints nothing whenever anything at all is missing, off, slow or malformed.
|
||||
#
|
||||
# The contract is that failure is indistinguishable from the feature being switched off, because
|
||||
# every caller must behave identically either way. An assist that can block a nightly cleanup is
|
||||
# not an assist — so this never retries, never blocks longer than its timeout, and never returns
|
||||
# a partial answer for a caller to interpret.
|
||||
#
|
||||
# Usage: note=$(ai_assist_note AI_ASSIST_CLEANUP "$prompt") && echo "$note"
|
||||
# $1 name of the AI_ASSIST_* flag governing this caller — passed by name, checked here, so a
|
||||
# caller cannot accidentally run an assist its own toggle says is off
|
||||
# $2 the prompt, evidence included
|
||||
# $3 timeout in seconds (default AI_ASSIST_TIMEOUT, else 45)
|
||||
ai_assist_note() {
|
||||
local flag="$1" prompt="$2" ai_timeout="${3:-${AI_ASSIST_TIMEOUT:-45}}"
|
||||
|
||||
[[ "${AI_ENABLED:-false}" == "true" ]] || return 1
|
||||
[[ "${!flag:-false}" == "true" ]] || return 1
|
||||
[[ -n "${MY_ID:-}" ]] || return 1
|
||||
command -v jq >/dev/null 2>&1 || return 1
|
||||
|
||||
local u_var="${MY_ID}_OLLAMA_URL" m_var="${MY_ID}_OLLAMA_MODEL"
|
||||
local url="${!u_var:-}" model="${!m_var:-}"
|
||||
[[ -n "$url" && -n "$model" ]] || return 1
|
||||
|
||||
local body out
|
||||
# temperature 0: this is a description of evidence, and the same evidence should not produce a
|
||||
# different characterisation on a rerun.
|
||||
body=$(jq -nc --arg m "$model" --arg p "$prompt" \
|
||||
'{model:$m, prompt:$p, stream:false, options:{temperature:0}}' 2>/dev/null) || return 1
|
||||
|
||||
out=$(timeout "$ai_timeout" curl -sf --max-time "$ai_timeout" \
|
||||
-H 'Content-Type: application/json' -d "$body" \
|
||||
"${url%/}/api/generate" 2>/dev/null | jq -r '.response // empty' 2>/dev/null)
|
||||
|
||||
# Reasoning models emit a <think> block before the answer. It is not the note — a log line a
|
||||
# human is meant to skim cannot open with several hundred words of the model talking itself
|
||||
# through the arithmetic. Strip to the last close tag; a response with no block is unchanged.
|
||||
[[ "$out" == *"</think>"* ]] && out="${out##*</think>}"
|
||||
out="${out#"${out%%[![:space:]]*}"}"
|
||||
|
||||
[[ -n "$out" ]] || return 1
|
||||
printf '%s\n' "$out"
|
||||
}
|
||||
|
||||
# True if filepath's extension (case-insensitive) matches one of the given extensions.
|
||||
# Usage: has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"
|
||||
has_extension() {
|
||||
|
||||
Reference in New Issue
Block a user