From d42b1e2dda87313b48efe24d58e8279694d34b2e Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Tue, 25 Aug 2026 18:29:43 -0400 Subject: [PATCH] ffprobe exit 0 with a chapter-track warning is not corruption; guard the delete path against a detector that is wrong at scale --- Arrs_Stack/arr_corruption_scan.sh | 128 ++++++++++++++++++++++++++++-- Deployment/master.conf.template | 9 +++ 2 files changed, 129 insertions(+), 8 deletions(-) mode change 100755 => 100644 Arrs_Stack/arr_corruption_scan.sh diff --git a/Arrs_Stack/arr_corruption_scan.sh b/Arrs_Stack/arr_corruption_scan.sh old mode 100755 new mode 100644 index ef3b79a..2bbe6c8 --- a/Arrs_Stack/arr_corruption_scan.sh +++ b/Arrs_Stack/arr_corruption_scan.sh @@ -273,6 +273,9 @@ touch "$CORRUPTION_SCAN_STATE_FILE" CORRUPTION_SCAN_STRIKES_FILE="${CORRUPTION_SCAN_STRIKES_FILE:-$DATA_DIR/corruption_scan_strikes.tsv}" CORRUPTION_SCAN_STRIKE_LIMIT="${CORRUPTION_SCAN_STRIKE_LIMIT:-2}" +CORRUPTION_SCAN_MAX_CORRUPT_PCT="${CORRUPTION_SCAN_MAX_CORRUPT_PCT:-10}" +CORRUPTION_SCAN_MAX_CONSECUTIVE="${CORRUPTION_SCAN_MAX_CONSECUTIVE:-15}" +CORRUPTION_SCAN_GUARD_MIN_SCANNED="${CORRUPTION_SCAN_GUARD_MIN_SCANNED:-20}" mkdir -p "$(dirname "$CORRUPTION_SCAN_STRIKES_FILE")" touch "$CORRUPTION_SCAN_STRIKES_FILE" @@ -301,6 +304,19 @@ reset_scan_strikes() { [[ -n "$current" && "$current" != "0" ]] && set_scan_strikes "$1" 0 } +# Bails out of the whole run without committing anything. Safe to call at any point before +# the commit phase: strikes are queued in memory until then, so an abort leaves the strike +# file exactly as the previous run left it and deletes nothing. +abort_scan() { + local why="$1" + error "Corruption scan ABORTED — $why" + error "No strikes recorded and nothing remediated this run — the library was not trusted." + [[ -n "${FRESH_CLEAN_TMP:-}" ]] && rm -f "$FRESH_CLEAN_TMP" + notify "Corruption scan aborted on $(hostname) ($MY_ID) — $why. Nothing deleted." \ + "Arr Corruption Scan" "warning" + exit 1 +} + # Per-arr API shape differences — everything else in the scan/strike/remediate loop below is # identical between Sonarr and Radarr. declare -A ARR_FILE_ENDPOINT=( [sonarr]="episodefile" [radarr]="moviefile" ) @@ -322,6 +338,8 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "$ICON_GEAR State file: $CORRUPTION_SCAN_STATE_FILE" echo "$ICON_GEAR Strike limit: $CORRUPTION_SCAN_STRIKE_LIMIT" echo "$ICON_GEAR Remediate: $REMEDIATE" + echo "$ICON_GEAR Corrupt ceiling: ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% of scanned (min ${CORRUPTION_SCAN_GUARD_MIN_SCANNED} scanned)" + echo "$ICON_GEAR Consecutive trip: $CORRUPTION_SCAN_MAX_CONSECUTIVE" echo "$ICON_GEAR Scan limit: ${SCAN_LIMIT:-unlimited} (per arr)" echo "$ICON_GEAR Path filter: ${PATH_FILTER:-none}" echo "━━━━━━━━━━━━━━━━━━━━━━━" @@ -357,13 +375,36 @@ ffprobe_translate_path() { # Probes one file. Echoes "clean" or "corrupt:". Never trusts a truncated/garbled # stderr as automatically corrupt — only a real non-empty ffprobe stderr counts. probe_file() { - local host_path="$1" container_path output + local host_path="$1" container_path output rc container_path=$(ffprobe_translate_path "$host_path") || { echo "unmapped"; return; } output=$(docker exec "$FFPROBE_CONTAINER" "$FFPROBE_BIN" -v error "$container_path" 2>&1) + rc=$? + + # docker exec writes its own failures to the same stream ffprobe uses, so a stopped + # container or an unreachable daemon is otherwise indistinguishable from a corrupt + # header. A stopped container exits 1 with a daemon message; a missing binary exits + # 127 — neither is evidence about the file, so both must be caught. + if (( rc >= 125 )) \ + || [[ "$output" == "Error response from daemon:"* \ + || "$output" == "Cannot connect to the Docker daemon"* \ + || "$output" == "error during connect:"* ]]; then + echo "probe_error:${output//$'\n'/ }" + return + fi + if [[ -z "$output" ]]; then echo "clean" - else + elif (( rc != 0 )); then + # ffprobe could not parse the file — EBML header parsing failed, moov atom not found, + # contradictionary STSC and STCO. This is the only class that may be remediated. echo "corrupt:${output//$'\n'/ }" + else + # Exit 0 with stderr output: a recoverable muxing complaint, most commonly + # "Referenced QT chapter track not found", which many recent .mp4 releases emit and + # which says nothing about playability. Equating any stderr with corruption is what + # produced 103 "corrupt" files on 2026-08-23 — 28 of 43 newly scanned Radarr items. + # Reported for visibility, never strike-tracked, never remediated. + echo "suspect:${output//$'\n'/ }" fi } @@ -442,7 +483,7 @@ TOTAL_SCANNED=0 TOTAL_CORRUPT=0 TOTAL_REMEDIATED=0 TOTAL_REMEDIATE_FAILED=0 -declare -A ARR_SCANNED ARR_SKIPPED_CACHED ARR_SKIPPED_UNMAPPED ARR_CORRUPT ARR_STRIKE_HELD ARR_REMEDIATED ARR_REMEDIATE_FAILED +declare -A ARR_SCANNED ARR_SKIPPED_CACHED ARR_SKIPPED_UNMAPPED ARR_CORRUPT ARR_SUSPECT ARR_PROBE_ERRORS ARR_STRIKE_HELD ARR_REMEDIATED ARR_REMEDIATE_FAILED for arr in sonarr radarr; do url_var="${arr^^}_URL"; key_var="${arr^^}_API_KEY" @@ -532,6 +573,12 @@ for arr in sonarr radarr; do STRIKE_HELD=0 REMEDIATED=0 REMEDIATE_FAILED=0 + PROBE_ERRORS=0 + SUSPECT_COUNT=0 + CONSECUTIVE_BAD=0 + QUEUE_PATH=() + QUEUE_STRIKES=() + QUEUE_ITEM=() FRESH_CLEAN_TMP=$(mktemp) @@ -566,23 +613,82 @@ for arr in sonarr radarr; do continue fi + # A docker-level failure is not evidence about the file. Count it, never queue it. + if [[ "$result" == probe_error:* ]]; then + (( PROBE_ERRORS++ )) + (( CONSECUTIVE_BAD++ )) + warn " ? $host_path — probe failed, NOT counted as corrupt: ${result#probe_error:}" + if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then + abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly" + fi + continue + fi + if [[ "$result" == "clean" ]]; then + CONSECUTIVE_BAD=0 reset_scan_strikes "$host_path" echo -e "${host_path}\t${stamp}" >> "$FRESH_CLEAN_TMP" [[ "$ENABLE_LOGGING" == true ]] && echo " $ICON_SUCCESS $host_path" continue fi - # corrupt: + # A successful probe that merely warned. Proves the container is alive, so it clears + # the consecutive-failure tripwire, but it never becomes a strike. + if [[ "$result" == suspect:* ]]; then + CONSECUTIVE_BAD=0 + (( SUSPECT_COUNT++ )) + [[ "$ENABLE_LOGGING" == true ]] && warn " ~ $host_path — ffprobe warning (exit 0), NOT corrupt: ${result#suspect:}" + continue + fi + + # corrupt: — queued, NOT committed. Nothing reaches the strike file and nothing + # is deleted until this arr has been fully probed and the guards below have passed. A + # container that dies mid-scan makes every remaining file read as corrupt, and a delete + # cannot be undone — so the destructive half has to wait until the corrupt rate for the + # whole run is known. 2026-08-23: one Jellyfin restart produced 103 false positives. reason="${result#corrupt:}" (( CORRUPT_COUNT++ )) - strikes=$(increment_scan_strikes "$host_path") + (( CONSECUTIVE_BAD++ )) + + prev_strikes=$(get_scan_strikes "$host_path") + prev_strikes="${prev_strikes//[^0-9]/}" + strikes=$(( ${prev_strikes:-0} + 1 )) + + QUEUE_PATH+=("$host_path") + QUEUE_STRIKES+=("$strikes") + QUEUE_ITEM+=("$item") + echo " $ICON_ERROR CORRUPT: $host_path (strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT)" [[ "$ENABLE_LOGGING" == true ]] && echo " $reason" - if [[ "$REMEDIATE" != true ]]; then - continue + if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then + abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly" fi + done < <(echo "$ITEMS" | jq -c '.[]') + + # ━━━ False-positive guards — run before anything is committed ━━━ + if (( CORRUPT_COUNT > 0 )); then + # The pre-flight check only proves the container was up when the scan started. + # Re-check now: a mid-scan death is exactly what this guard exists to catch. + check_container_health "$FFPROBE_CONTAINER" "${DOCKER_TIMEOUT:-30}" "Arr Corruption Scan" + + if (( SCANNED >= CORRUPTION_SCAN_GUARD_MIN_SCANNED )); then + corrupt_pct=$(( CORRUPT_COUNT * 100 / SCANNED )) + if (( corrupt_pct >= CORRUPTION_SCAN_MAX_CORRUPT_PCT )); then + abort_scan "$CORRUPT_COUNT of $SCANNED probed files (${corrupt_pct}%) read as corrupt — at or above the ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% ceiling" + fi + fi + fi + + # ━━━ Guards passed — commit strikes, then remediate whatever reached the limit ━━━ + for _q in "${!QUEUE_PATH[@]}"; do + host_path="${QUEUE_PATH[$_q]}" + strikes="${QUEUE_STRIKES[$_q]}" + item="${QUEUE_ITEM[$_q]}" + + set_scan_strikes "$host_path" "$strikes" + + [[ "$REMEDIATE" != true ]] && continue if (( strikes < CORRUPTION_SCAN_STRIKE_LIMIT )); then warn " $host_path — strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT, not yet remediating (needs repeat confirmation)" @@ -591,6 +697,8 @@ for arr in sonarr radarr; do fi reset_scan_strikes "$host_path" + file_id=$(echo "$item" | jq -r '.file_id') + parent_id=$(echo "$item" | jq -r '.parent_id') title=$(echo "$item" | jq -r '.title') http_code=$(curl -sf -o /dev/null -w "%{http_code}" -X DELETE \ @@ -625,7 +733,7 @@ for arr in sonarr radarr; do warn " $title — deleted and verified, but search trigger returned HTTP $search_code" (( REMEDIATE_FAILED++ )) fi - done < <(echo "$ITEMS" | jq -c '.[]') + done merge_clean_state "$FRESH_CLEAN_TMP" rm -f "$FRESH_CLEAN_TMP" @@ -634,6 +742,8 @@ for arr in sonarr radarr; do ARR_SKIPPED_CACHED[$arr]=$SKIPPED_CACHED ARR_SKIPPED_UNMAPPED[$arr]=$SKIPPED_UNMAPPED ARR_CORRUPT[$arr]=$CORRUPT_COUNT + ARR_SUSPECT[$arr]=$SUSPECT_COUNT + ARR_PROBE_ERRORS[$arr]=$PROBE_ERRORS ARR_STRIKE_HELD[$arr]=$STRIKE_HELD ARR_REMEDIATED[$arr]=$REMEDIATED ARR_REMEDIATE_FAILED[$arr]=$REMEDIATE_FAILED @@ -658,6 +768,8 @@ for arr in sonarr radarr; do echo " $ICON_SUCCESS Skipped (cached): ${ARR_SKIPPED_CACHED[$arr]}" echo " $ICON_WARN Skipped (unmapped): ${ARR_SKIPPED_UNMAPPED[$arr]}" echo " $ICON_ERROR Corrupt found: ${ARR_CORRUPT[$arr]}" + echo " $ICON_WARN Warnings (exit 0): ${ARR_SUSPECT[$arr]} (reported, never remediated)" + echo " $ICON_WARN Probe errors: ${ARR_PROBE_ERRORS[$arr]} (not counted as corrupt)" if [[ "$REMEDIATE" == true ]]; then echo " $ICON_WARN Held (strikes): ${ARR_STRIKE_HELD[$arr]}" echo " $ICON_SUCCESS Remediated: ${ARR_REMEDIATED[$arr]}" diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index b4dc5e6..07a2b76 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -1355,6 +1355,15 @@ # against a one-off ffprobe hiccup (mid-write file, NFS blip) # triggering an unnecessary delete. Resets to 0 the moment a # file probes clean again. + CORRUPTION_SCAN_MAX_CORRUPT_PCT=10 # abort the run, committing nothing, if this share of + # newly-scanned files reads as corrupt. A healthy library sits + # near zero; a high rate means the detector is wrong, not the + # library. Only counts ffprobe exit != 0. + CORRUPTION_SCAN_MAX_CONSECUTIVE=15 # abort after this many files in a row fail to probe + # cleanly — catches the ffprobe container dying mid-scan, + # which the pre-flight health check cannot see. + CORRUPTION_SCAN_GUARD_MIN_SCANNED=20 # below this many newly-scanned files the percentage + # ceiling is not applied — too small a sample to judge. SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") SONARR_PROTECTED_PATTERNS=( # Subtitles