FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ffprobe exit 0 with a chapter-track warning is not corruption; guard the delete path against a detector that is wrong at scale

Browse Source
This commit is contained in:
Gmer4Lfe
2026-08-25 18:29:43 -04:00
parent 9492dc4c39
commit d42b1e2dda
2 changed files with 129 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Arrs_Stack/arr_corruption_scan.sh
Executable → Regular
+120 -8
View File
@@ -273,6 +273,9 @@ touch "$CORRUPTION_SCAN_STATE_FILE"
CORRUPTION_SCAN_STRIKES_FILE="${CORRUPTION_SCAN_STRIKES_FILE:-$DATA_DIR/corruption_scan_strikes.tsv}"
CORRUPTION_SCAN_STRIKE_LIMIT="${CORRUPTION_SCAN_STRIKE_LIMIT:-2}"
CORRUPTION_SCAN_MAX_CORRUPT_PCT="${CORRUPTION_SCAN_MAX_CORRUPT_PCT:-10}"
CORRUPTION_SCAN_MAX_CONSECUTIVE="${CORRUPTION_SCAN_MAX_CONSECUTIVE:-15}"
CORRUPTION_SCAN_GUARD_MIN_SCANNED="${CORRUPTION_SCAN_GUARD_MIN_SCANNED:-20}"
mkdir -p "$(dirname "$CORRUPTION_SCAN_STRIKES_FILE")"
touch "$CORRUPTION_SCAN_STRIKES_FILE"
@@ -301,6 +304,19 @@ reset_scan_strikes() {
[[ -n "$current" && "$current" != "0" ]] && set_scan_strikes "$1" 0
}
# Bails out of the whole run without committing anything. Safe to call at any point before
# the commit phase: strikes are queued in memory until then, so an abort leaves the strike
# file exactly as the previous run left it and deletes nothing.
abort_scan() {
local why="$1"
error "Corruption scan ABORTED — $why"
error "No strikes recorded and nothing remediated this run — the library was not trusted."
[[ -n "${FRESH_CLEAN_TMP:-}" ]] && rm -f "$FRESH_CLEAN_TMP"
notify "Corruption scan aborted on $(hostname) ($MY_ID) — $why. Nothing deleted." \
"Arr Corruption Scan" "warning"
exit 1
}
# Per-arr API shape differences — everything else in the scan/strike/remediate loop below is
# identical between Sonarr and Radarr.
declare -A ARR_FILE_ENDPOINT=( [sonarr]="episodefile" [radarr]="moviefile" )
@@ -322,6 +338,8 @@ if [[ "$SHOW_STATUS" == true ]]; then
echo "$ICON_GEAR State file: $CORRUPTION_SCAN_STATE_FILE"
echo "$ICON_GEAR Strike limit: $CORRUPTION_SCAN_STRIKE_LIMIT"
echo "$ICON_GEAR Remediate: $REMEDIATE"
echo "$ICON_GEAR Corrupt ceiling: ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% of scanned (min ${CORRUPTION_SCAN_GUARD_MIN_SCANNED} scanned)"
echo "$ICON_GEAR Consecutive trip: $CORRUPTION_SCAN_MAX_CONSECUTIVE"
echo "$ICON_GEAR Scan limit: ${SCAN_LIMIT:-unlimited} (per arr)"
echo "$ICON_GEAR Path filter: ${PATH_FILTER:-none}"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
@@ -357,13 +375,36 @@ ffprobe_translate_path() {
# Probes one file. Echoes "clean" or "corrupt:<reason>". Never trusts a truncated/garbled
# stderr as automatically corrupt — only a real non-empty ffprobe stderr counts.
probe_file() {
local host_path="$1" container_path output
local host_path="$1" container_path output rc
container_path=$(ffprobe_translate_path "$host_path") || { echo "unmapped"; return; }
output=$(docker exec "$FFPROBE_CONTAINER" "$FFPROBE_BIN" -v error "$container_path" 2>&1)
rc=$?
# docker exec writes its own failures to the same stream ffprobe uses, so a stopped
# container or an unreachable daemon is otherwise indistinguishable from a corrupt
# header. A stopped container exits 1 with a daemon message; a missing binary exits
# 127 — neither is evidence about the file, so both must be caught.
if (( rc >= 125 )) \
|| [[ "$output" == "Error response from daemon:"* \
|| "$output" == "Cannot connect to the Docker daemon"* \
|| "$output" == "error during connect:"* ]]; then
echo "probe_error:${output//$'\n'/ }"
return
fi
if [[ -z "$output" ]]; then
echo "clean"
else
elif (( rc != 0 )); then
# ffprobe could not parse the file — EBML header parsing failed, moov atom not found,
# contradictionary STSC and STCO. This is the only class that may be remediated.
echo "corrupt:${output//$'\n'/ }"
else
# Exit 0 with stderr output: a recoverable muxing complaint, most commonly
# "Referenced QT chapter track not found", which many recent .mp4 releases emit and
# which says nothing about playability. Equating any stderr with corruption is what
# produced 103 "corrupt" files on 2026-08-23 — 28 of 43 newly scanned Radarr items.
# Reported for visibility, never strike-tracked, never remediated.
echo "suspect:${output//$'\n'/ }"
fi
}
@@ -442,7 +483,7 @@ TOTAL_SCANNED=0
TOTAL_CORRUPT=0
TOTAL_REMEDIATED=0
TOTAL_REMEDIATE_FAILED=0
declare -A ARR_SCANNED ARR_SKIPPED_CACHED ARR_SKIPPED_UNMAPPED ARR_CORRUPT ARR_STRIKE_HELD ARR_REMEDIATED ARR_REMEDIATE_FAILED
declare -A ARR_SCANNED ARR_SKIPPED_CACHED ARR_SKIPPED_UNMAPPED ARR_CORRUPT ARR_SUSPECT ARR_PROBE_ERRORS ARR_STRIKE_HELD ARR_REMEDIATED ARR_REMEDIATE_FAILED
for arr in sonarr radarr; do
url_var="${arr^^}_URL"; key_var="${arr^^}_API_KEY"
@@ -532,6 +573,12 @@ for arr in sonarr radarr; do
STRIKE_HELD=0
REMEDIATED=0
REMEDIATE_FAILED=0
PROBE_ERRORS=0
SUSPECT_COUNT=0
CONSECUTIVE_BAD=0
QUEUE_PATH=()
QUEUE_STRIKES=()
QUEUE_ITEM=()
FRESH_CLEAN_TMP=$(mktemp)
@@ -566,23 +613,82 @@ for arr in sonarr radarr; do
continue
fi
# A docker-level failure is not evidence about the file. Count it, never queue it.
if [[ "$result" == probe_error:* ]]; then
(( PROBE_ERRORS++ ))
(( CONSECUTIVE_BAD++ ))
warn " ? $host_path — probe failed, NOT counted as corrupt: ${result#probe_error:}"
if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then
abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly"
fi
continue
fi
if [[ "$result" == "clean" ]]; then
CONSECUTIVE_BAD=0
reset_scan_strikes "$host_path"
echo -e "${host_path}\t${stamp}" >> "$FRESH_CLEAN_TMP"
[[ "$ENABLE_LOGGING" == true ]] && echo " $ICON_SUCCESS $host_path"
continue
fi
# corrupt:<reason>
# A successful probe that merely warned. Proves the container is alive, so it clears
# the consecutive-failure tripwire, but it never becomes a strike.
if [[ "$result" == suspect:* ]]; then
CONSECUTIVE_BAD=0
(( SUSPECT_COUNT++ ))
[[ "$ENABLE_LOGGING" == true ]] && warn " ~ $host_path — ffprobe warning (exit 0), NOT corrupt: ${result#suspect:}"
continue
fi
# corrupt:<reason> — queued, NOT committed. Nothing reaches the strike file and nothing
# is deleted until this arr has been fully probed and the guards below have passed. A
# container that dies mid-scan makes every remaining file read as corrupt, and a delete
# cannot be undone — so the destructive half has to wait until the corrupt rate for the
# whole run is known. 2026-08-23: one Jellyfin restart produced 103 false positives.
reason="${result#corrupt:}"
(( CORRUPT_COUNT++ ))
strikes=$(increment_scan_strikes "$host_path")
(( CONSECUTIVE_BAD++ ))
prev_strikes=$(get_scan_strikes "$host_path")
prev_strikes="${prev_strikes//[^0-9]/}"
strikes=$(( ${prev_strikes:-0} + 1 ))
QUEUE_PATH+=("$host_path")
QUEUE_STRIKES+=("$strikes")
QUEUE_ITEM+=("$item")
echo " $ICON_ERROR CORRUPT: $host_path (strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT)"
[[ "$ENABLE_LOGGING" == true ]] && echo " $reason"
if [[ "$REMEDIATE" != true ]]; then
continue
if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then
abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly"
fi
done < <(echo "$ITEMS" | jq -c '.[]')
# ━━━ False-positive guards — run before anything is committed ━━━
if (( CORRUPT_COUNT > 0 )); then
# The pre-flight check only proves the container was up when the scan started.
# Re-check now: a mid-scan death is exactly what this guard exists to catch.
check_container_health "$FFPROBE_CONTAINER" "${DOCKER_TIMEOUT:-30}" "Arr Corruption Scan"
if (( SCANNED >= CORRUPTION_SCAN_GUARD_MIN_SCANNED )); then
corrupt_pct=$(( CORRUPT_COUNT * 100 / SCANNED ))
if (( corrupt_pct >= CORRUPTION_SCAN_MAX_CORRUPT_PCT )); then
abort_scan "$CORRUPT_COUNT of $SCANNED probed files (${corrupt_pct}%) read as corrupt — at or above the ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% ceiling"
fi
fi
fi
# ━━━ Guards passed — commit strikes, then remediate whatever reached the limit ━━━
for _q in "${!QUEUE_PATH[@]}"; do
host_path="${QUEUE_PATH[$_q]}"
strikes="${QUEUE_STRIKES[$_q]}"
item="${QUEUE_ITEM[$_q]}"
set_scan_strikes "$host_path" "$strikes"
[[ "$REMEDIATE" != true ]] && continue
if (( strikes < CORRUPTION_SCAN_STRIKE_LIMIT )); then
warn " $host_path — strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT, not yet remediating (needs repeat confirmation)"
@@ -591,6 +697,8 @@ for arr in sonarr radarr; do
fi
reset_scan_strikes "$host_path"
file_id=$(echo "$item" | jq -r '.file_id')
parent_id=$(echo "$item" | jq -r '.parent_id')
title=$(echo "$item" | jq -r '.title')
http_code=$(curl -sf -o /dev/null -w "%{http_code}" -X DELETE \
@@ -625,7 +733,7 @@ for arr in sonarr radarr; do
warn " $title — deleted and verified, but search trigger returned HTTP $search_code"
(( REMEDIATE_FAILED++ ))
fi
done < <(echo "$ITEMS" | jq -c '.[]')
done
merge_clean_state "$FRESH_CLEAN_TMP"
rm -f "$FRESH_CLEAN_TMP"
@@ -634,6 +742,8 @@ for arr in sonarr radarr; do
ARR_SKIPPED_CACHED[$arr]=$SKIPPED_CACHED
ARR_SKIPPED_UNMAPPED[$arr]=$SKIPPED_UNMAPPED
ARR_CORRUPT[$arr]=$CORRUPT_COUNT
ARR_SUSPECT[$arr]=$SUSPECT_COUNT
ARR_PROBE_ERRORS[$arr]=$PROBE_ERRORS
ARR_STRIKE_HELD[$arr]=$STRIKE_HELD
ARR_REMEDIATED[$arr]=$REMEDIATED
ARR_REMEDIATE_FAILED[$arr]=$REMEDIATE_FAILED
@@ -658,6 +768,8 @@ for arr in sonarr radarr; do
echo " $ICON_SUCCESS Skipped (cached): ${ARR_SKIPPED_CACHED[$arr]}"
echo " $ICON_WARN Skipped (unmapped): ${ARR_SKIPPED_UNMAPPED[$arr]}"
echo " $ICON_ERROR Corrupt found: ${ARR_CORRUPT[$arr]}"
echo " $ICON_WARN Warnings (exit 0): ${ARR_SUSPECT[$arr]} (reported, never remediated)"
echo " $ICON_WARN Probe errors: ${ARR_PROBE_ERRORS[$arr]} (not counted as corrupt)"
if [[ "$REMEDIATE" == true ]]; then
echo " $ICON_WARN Held (strikes): ${ARR_STRIKE_HELD[$arr]}"
echo " $ICON_SUCCESS Remediated: ${ARR_REMEDIATED[$arr]}"
Deployment/master.conf.template
+9
View File
@@ -1355,6 +1355,15 @@
# against a one-off ffprobe hiccup (mid-write file, NFS blip)
# triggering an unnecessary delete. Resets to 0 the moment a
# file probes clean again.
CORRUPTION_SCAN_MAX_CORRUPT_PCT=10 # abort the run, committing nothing, if this share of
# newly-scanned files reads as corrupt. A healthy library sits
# near zero; a high rate means the detector is wrong, not the
# library. Only counts ffprobe exit != 0.
CORRUPTION_SCAN_MAX_CONSECUTIVE=15 # abort after this many files in a row fail to probe
# cleanly — catches the ffprobe container dying mid-scan,
# which the pre-flight health check cannot see.
CORRUPTION_SCAN_GUARD_MIN_SCANNED=20 # below this many newly-scanned files the percentage
# ceiling is not applied — too small a sample to judge.
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
SONARR_PROTECTED_PATTERNS=(
# Subtitles