ffprobe exit 0 with a chapter-track warning is not corruption; guard the delete path against a detector that is wrong at scale

Arrs_Stack/arr_corruption_scan.sh

@@ -273,6 +273,9 @@ touch "$CORRUPTION_SCAN_STATE_FILE"
 
 CORRUPTION_SCAN_STRIKES_FILE="${CORRUPTION_SCAN_STRIKES_FILE:-$DATA_DIR/corruption_scan_strikes.tsv}"
 CORRUPTION_SCAN_STRIKE_LIMIT="${CORRUPTION_SCAN_STRIKE_LIMIT:-2}"
+CORRUPTION_SCAN_MAX_CORRUPT_PCT="${CORRUPTION_SCAN_MAX_CORRUPT_PCT:-10}"
+CORRUPTION_SCAN_MAX_CONSECUTIVE="${CORRUPTION_SCAN_MAX_CONSECUTIVE:-15}"
+CORRUPTION_SCAN_GUARD_MIN_SCANNED="${CORRUPTION_SCAN_GUARD_MIN_SCANNED:-20}"
 mkdir -p "$(dirname "$CORRUPTION_SCAN_STRIKES_FILE")"
 touch "$CORRUPTION_SCAN_STRIKES_FILE"
 
@@ -301,6 +304,19 @@ reset_scan_strikes() {
     [[ -n "$current" && "$current" != "0" ]] && set_scan_strikes "$1" 0
 }
 
+# Bails out of the whole run without committing anything. Safe to call at any point before
+# the commit phase: strikes are queued in memory until then, so an abort leaves the strike
+# file exactly as the previous run left it and deletes nothing.
+abort_scan() {
+    local why="$1"
+    error "Corruption scan ABORTED — $why"
+    error "No strikes recorded and nothing remediated this run — the library was not trusted."
+    [[ -n "${FRESH_CLEAN_TMP:-}" ]] && rm -f "$FRESH_CLEAN_TMP"
+    notify "Corruption scan aborted on $(hostname) ($MY_ID) — $why. Nothing deleted." \
+        "Arr Corruption Scan" "warning"
+    exit 1
+}
+
 # Per-arr API shape differences — everything else in the scan/strike/remediate loop below is
 # identical between Sonarr and Radarr.
 declare -A ARR_FILE_ENDPOINT=(   [sonarr]="episodefile" [radarr]="moviefile" )
@@ -322,6 +338,8 @@ if [[ "$SHOW_STATUS" == true ]]; then
     echo "$ICON_GEAR    State file:        $CORRUPTION_SCAN_STATE_FILE"
     echo "$ICON_GEAR    Strike limit:      $CORRUPTION_SCAN_STRIKE_LIMIT"
     echo "$ICON_GEAR    Remediate:         $REMEDIATE"
+    echo "$ICON_GEAR    Corrupt ceiling:   ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% of scanned (min ${CORRUPTION_SCAN_GUARD_MIN_SCANNED} scanned)"
+    echo "$ICON_GEAR    Consecutive trip:  $CORRUPTION_SCAN_MAX_CONSECUTIVE"
     echo "$ICON_GEAR    Scan limit:        ${SCAN_LIMIT:-unlimited} (per arr)"
     echo "$ICON_GEAR    Path filter:       ${PATH_FILTER:-none}"
     echo "━━━━━━━━━━━━━━━━━━━━━━━"
@@ -357,13 +375,36 @@ ffprobe_translate_path() {
 # Probes one file. Echoes "clean" or "corrupt:<reason>". Never trusts a truncated/garbled
 # stderr as automatically corrupt — only a real non-empty ffprobe stderr counts.
 probe_file() {
-    local host_path="$1" container_path output
+    local host_path="$1" container_path output rc
     container_path=$(ffprobe_translate_path "$host_path") || { echo "unmapped"; return; }
     output=$(docker exec "$FFPROBE_CONTAINER" "$FFPROBE_BIN" -v error "$container_path" 2>&1)
+    rc=$?
+
+    # docker exec writes its own failures to the same stream ffprobe uses, so a stopped
+    # container or an unreachable daemon is otherwise indistinguishable from a corrupt
+    # header. A stopped container exits 1 with a daemon message; a missing binary exits
+    # 127 — neither is evidence about the file, so both must be caught.
+    if (( rc >= 125 )) \
+        || [[ "$output" == "Error response from daemon:"* \
+        || "$output" == "Cannot connect to the Docker daemon"* \
+        || "$output" == "error during connect:"* ]]; then
+        echo "probe_error:${output//$'\n'/ }"
+        return
+    fi
+
     if [[ -z "$output" ]]; then
         echo "clean"
-    else
+    elif (( rc != 0 )); then
+        # ffprobe could not parse the file — EBML header parsing failed, moov atom not found,
+        # contradictionary STSC and STCO. This is the only class that may be remediated.
         echo "corrupt:${output//$'\n'/ }"
+    else
+        # Exit 0 with stderr output: a recoverable muxing complaint, most commonly
+        # "Referenced QT chapter track not found", which many recent .mp4 releases emit and
+        # which says nothing about playability. Equating any stderr with corruption is what
+        # produced 103 "corrupt" files on 2026-08-23 — 28 of 43 newly scanned Radarr items.
+        # Reported for visibility, never strike-tracked, never remediated.
+        echo "suspect:${output//$'\n'/ }"
     fi
 }
 
@@ -442,7 +483,7 @@ TOTAL_SCANNED=0
 TOTAL_CORRUPT=0
 TOTAL_REMEDIATED=0
 TOTAL_REMEDIATE_FAILED=0
-declare -A ARR_SCANNED ARR_SKIPPED_CACHED ARR_SKIPPED_UNMAPPED ARR_CORRUPT ARR_STRIKE_HELD ARR_REMEDIATED ARR_REMEDIATE_FAILED
+declare -A ARR_SCANNED ARR_SKIPPED_CACHED ARR_SKIPPED_UNMAPPED ARR_CORRUPT ARR_SUSPECT ARR_PROBE_ERRORS ARR_STRIKE_HELD ARR_REMEDIATED ARR_REMEDIATE_FAILED
 
 for arr in sonarr radarr; do
     url_var="${arr^^}_URL"; key_var="${arr^^}_API_KEY"
@@ -532,6 +573,12 @@ for arr in sonarr radarr; do
     STRIKE_HELD=0
     REMEDIATED=0
     REMEDIATE_FAILED=0
+    PROBE_ERRORS=0
+    SUSPECT_COUNT=0
+    CONSECUTIVE_BAD=0
+    QUEUE_PATH=()
+    QUEUE_STRIKES=()
+    QUEUE_ITEM=()
 
     FRESH_CLEAN_TMP=$(mktemp)
 
@@ -566,23 +613,82 @@ for arr in sonarr radarr; do
             continue
         fi
 
+        # A docker-level failure is not evidence about the file. Count it, never queue it.
+        if [[ "$result" == probe_error:* ]]; then
+            (( PROBE_ERRORS++ ))
+            (( CONSECUTIVE_BAD++ ))
+            warn "  ? $host_path — probe failed, NOT counted as corrupt: ${result#probe_error:}"
+            if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then
+                abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly"
+            fi
+            continue
+        fi
+
         if [[ "$result" == "clean" ]]; then
+            CONSECUTIVE_BAD=0
             reset_scan_strikes "$host_path"
             echo -e "${host_path}\t${stamp}" >> "$FRESH_CLEAN_TMP"
             [[ "$ENABLE_LOGGING" == true ]] && echo "  $ICON_SUCCESS $host_path"
             continue
         fi
 
-        # corrupt:<reason>
+        # A successful probe that merely warned. Proves the container is alive, so it clears
+        # the consecutive-failure tripwire, but it never becomes a strike.
+        if [[ "$result" == suspect:* ]]; then
+            CONSECUTIVE_BAD=0
+            (( SUSPECT_COUNT++ ))
+            [[ "$ENABLE_LOGGING" == true ]] && warn "  ~ $host_path — ffprobe warning (exit 0), NOT corrupt: ${result#suspect:}"
+            continue
+        fi
+
+        # corrupt:<reason> — queued, NOT committed. Nothing reaches the strike file and nothing
+        # is deleted until this arr has been fully probed and the guards below have passed. A
+        # container that dies mid-scan makes every remaining file read as corrupt, and a delete
+        # cannot be undone — so the destructive half has to wait until the corrupt rate for the
+        # whole run is known. 2026-08-23: one Jellyfin restart produced 103 false positives.
         reason="${result#corrupt:}"
         (( CORRUPT_COUNT++ ))
-        strikes=$(increment_scan_strikes "$host_path")
+        (( CONSECUTIVE_BAD++ ))
+
+        prev_strikes=$(get_scan_strikes "$host_path")
+        prev_strikes="${prev_strikes//[^0-9]/}"
+        strikes=$(( ${prev_strikes:-0} + 1 ))
+
+        QUEUE_PATH+=("$host_path")
+        QUEUE_STRIKES+=("$strikes")
+        QUEUE_ITEM+=("$item")
+
         echo "  $ICON_ERROR CORRUPT: $host_path (strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT)"
         [[ "$ENABLE_LOGGING" == true ]] && echo "      $reason"
 
-        if [[ "$REMEDIATE" != true ]]; then
-            continue
+        if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then
+            abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly"
         fi
+    done < <(echo "$ITEMS" | jq -c '.[]')
+
+    # ━━━ False-positive guards — run before anything is committed ━━━
+    if (( CORRUPT_COUNT > 0 )); then
+        # The pre-flight check only proves the container was up when the scan started.
+        # Re-check now: a mid-scan death is exactly what this guard exists to catch.
+        check_container_health "$FFPROBE_CONTAINER" "${DOCKER_TIMEOUT:-30}" "Arr Corruption Scan"
+
+        if (( SCANNED >= CORRUPTION_SCAN_GUARD_MIN_SCANNED )); then
+            corrupt_pct=$(( CORRUPT_COUNT * 100 / SCANNED ))
+            if (( corrupt_pct >= CORRUPTION_SCAN_MAX_CORRUPT_PCT )); then
+                abort_scan "$CORRUPT_COUNT of $SCANNED probed files (${corrupt_pct}%) read as corrupt — at or above the ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% ceiling"
+            fi
+        fi
+    fi
+
+    # ━━━ Guards passed — commit strikes, then remediate whatever reached the limit ━━━
+    for _q in "${!QUEUE_PATH[@]}"; do
+        host_path="${QUEUE_PATH[$_q]}"
+        strikes="${QUEUE_STRIKES[$_q]}"
+        item="${QUEUE_ITEM[$_q]}"
+
+        set_scan_strikes "$host_path" "$strikes"
+
+        [[ "$REMEDIATE" != true ]] && continue
 
         if (( strikes < CORRUPTION_SCAN_STRIKE_LIMIT )); then
             warn "  $host_path — strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT, not yet remediating (needs repeat confirmation)"
@@ -591,6 +697,8 @@ for arr in sonarr radarr; do
         fi
         reset_scan_strikes "$host_path"
 
+        file_id=$(echo "$item" | jq -r '.file_id')
+        parent_id=$(echo "$item" | jq -r '.parent_id')
         title=$(echo "$item" | jq -r '.title')
 
         http_code=$(curl -sf -o /dev/null -w "%{http_code}" -X DELETE \
@@ -625,7 +733,7 @@ for arr in sonarr radarr; do
             warn "  $title — deleted and verified, but search trigger returned HTTP $search_code"
             (( REMEDIATE_FAILED++ ))
         fi
-    done < <(echo "$ITEMS" | jq -c '.[]')
+    done
 
     merge_clean_state "$FRESH_CLEAN_TMP"
     rm -f "$FRESH_CLEAN_TMP"
@@ -634,6 +742,8 @@ for arr in sonarr radarr; do
     ARR_SKIPPED_CACHED[$arr]=$SKIPPED_CACHED
     ARR_SKIPPED_UNMAPPED[$arr]=$SKIPPED_UNMAPPED
     ARR_CORRUPT[$arr]=$CORRUPT_COUNT
+    ARR_SUSPECT[$arr]=$SUSPECT_COUNT
+    ARR_PROBE_ERRORS[$arr]=$PROBE_ERRORS
     ARR_STRIKE_HELD[$arr]=$STRIKE_HELD
     ARR_REMEDIATED[$arr]=$REMEDIATED
     ARR_REMEDIATE_FAILED[$arr]=$REMEDIATE_FAILED
@@ -658,6 +768,8 @@ for arr in sonarr radarr; do
     echo "    $ICON_SUCCESS Skipped (cached):    ${ARR_SKIPPED_CACHED[$arr]}"
     echo "    $ICON_WARN    Skipped (unmapped):  ${ARR_SKIPPED_UNMAPPED[$arr]}"
     echo "    $ICON_ERROR   Corrupt found:       ${ARR_CORRUPT[$arr]}"
+    echo "    $ICON_WARN    Warnings (exit 0):   ${ARR_SUSPECT[$arr]} (reported, never remediated)"
+    echo "    $ICON_WARN    Probe errors:        ${ARR_PROBE_ERRORS[$arr]} (not counted as corrupt)"
     if [[ "$REMEDIATE" == true ]]; then
         echo "    $ICON_WARN    Held (strikes):      ${ARR_STRIKE_HELD[$arr]}"
         echo "    $ICON_SUCCESS Remediated:          ${ARR_REMEDIATED[$arr]}"

Deployment/master.conf.template

@@ -1355,6 +1355,15 @@
                                     # against a one-off ffprobe hiccup (mid-write file, NFS blip)
                                     # triggering an unnecessary delete. Resets to 0 the moment a
                                     # file probes clean again.
+    CORRUPTION_SCAN_MAX_CORRUPT_PCT=10   # abort the run, committing nothing, if this share of
+                                    # newly-scanned files reads as corrupt. A healthy library sits
+                                    # near zero; a high rate means the detector is wrong, not the
+                                    # library. Only counts ffprobe exit != 0.
+    CORRUPTION_SCAN_MAX_CONSECUTIVE=15   # abort after this many files in a row fail to probe
+                                    # cleanly — catches the ffprobe container dying mid-scan,
+                                    # which the pre-flight health check cannot see.
+    CORRUPTION_SCAN_GUARD_MIN_SCANNED=20  # below this many newly-scanned files the percentage
+                                    # ceiling is not applied — too small a sample to judge.
     SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
     SONARR_PROTECTED_PATTERNS=(
         # Subtitles

Watchdogs/System/network_watchdog.sh

@@ -270,7 +270,17 @@ log "$ICON_SUCCESS Internet reachable"
 # ==============================================================================================
 if [[ -n "$DDNS_DOMAIN" ]] && [[ -n "$DDNS_CONTAINER" ]]; then
     PUBLIC_IP=$(curl -sf --max-time 5 https://ifconfig.me 2>/dev/null | tr -d '[:space:]')
-    DNS_IP=$(dig +short "$DDNS_DOMAIN" @1.1.1.1 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1)
+    # dig's failure text names the resolver it could not reach (";; communications error to
+    # 1.1.1.1#53: timed out"), so scraping its output for an address yields the server, not the
+    # answer — a guaranteed mismatch that restarts DDNS over what is only a DNS timeout. Trust
+    # the exit status, and anchor the match so only a bare answer line counts. NXDOMAIN exits 0
+    # with no output and correctly falls through to the "could not resolve" branch below.
+    if DNS_ANSWER=$(dig +short "$DDNS_DOMAIN" @1.1.1.1 2>/dev/null); then
+        DNS_IP=$(printf '%s\n' "$DNS_ANSWER" \
+            | grep -Eox '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1)
+    else
+        DNS_IP=""
+    fi
 
     if [[ -z "$PUBLIC_IP" ]]; then
         warn "Could not determine public IP — skipping DDNS check"

common.sh

@@ -2273,8 +2273,10 @@ acquire_rsync_lock() {
 # ==============================================================================================
 # Generic get/set for flat "key<sep>value" state files (one entry per line) — the pattern
 # every watchdog's strike-tracking and state-file logic was independently reimplementing.
-# grep -v + tempfile-swap on write, not sed -i in place — avoids sed treating a key containing
-# regex metacharacters (container names, etc.) as part of the substitution pattern.
+# Keys are matched as literal prefixes via awk substr(), never as regexes. A key is often a
+# media file path, and a release tag like [Bluray-1080p] is a valid-looking bracket expression
+# holding the reversed range 1-0 — grep -E rejects it, exits 2, and the tempfile-swap then
+# commits an empty file, silently wiping every other entry.
 #
 # Usage: wd_state_get "$key" "$file" [sep=:]
 #        wd_state_set "$key" "$value" "$file" [sep=:]
@@ -2284,14 +2286,24 @@ acquire_rsync_lock() {
 # keep their own thin same-named wrapper around these rather than changing call sites.
 wd_state_get() {
     local key="$1" file="$2" sep="${3:-:}"
-    grep -E "^${key}${sep}" "$file" 2>/dev/null | cut -d"$sep" -f2-
+    [[ -f "$file" ]] || return 0
+    awk -v pfx="${key}${sep}" \
+        'substr($0, 1, length(pfx)) == pfx { print substr($0, length(pfx) + 1); exit }' \
+        "$file" 2>/dev/null
 }
 
+# Returns 1 without touching the state file if the rewrite fails, so a read error costs the
+# caller one update rather than the whole file.
 wd_state_set() {
     local key="$1" value="$2" file="$3" sep="${4:-:}"
-    grep -vE "^${key}${sep}" "$file" 2>/dev/null > "${file}.tmp"
-    echo "${key}${sep}${value}" >> "${file}.tmp"
-    mv "${file}.tmp" "$file"
+    local tmp="${file}.tmp"
+    : > "$tmp" || return 1
+    if [[ -f "$file" ]]; then
+        awk -v pfx="${key}${sep}" \
+            'substr($0, 1, length(pfx)) != pfx' "$file" > "$tmp" || { rm -f "$tmp"; return 1; }
+    fi
+    echo "${key}${sep}${value}" >> "$tmp"
+    mv "$tmp" "$file"
 }
 
 # ==============================================================================================