From ab35fcd84d1a470058bfb0c7ba8016ba27e775cd Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sat, 18 Jul 2026 16:48:32 -0400 Subject: [PATCH] =?UTF-8?q?Add=20arr=5Fcorruption=5Fscan.sh=20=E2=80=94=20?= =?UTF-8?q?sequential=20ffprobe-based=20corruption=20scan=20for=20Sonarr?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Healarr does the same job but crashes on a Go concurrency bug (unsynchronized map access) whenever multiple corruption events land close together — confirmed via its own crash log, not fixable from our side. Processing one file at a time here sidesteps the whole bug class instead of trying to work around it. Delete + explicit EpisodeSearch rather than relying on Sonarr's own background missing-search cycle, since that skips unmonitored episodes and this shouldn't. --- Arrs_Stack/arr_corruption_scan.sh | 390 ++++++++++++++++++++++++++++++ Deployment/host.conf.template | 11 + Deployment/master.conf.template | 1 + 3 files changed, 402 insertions(+) create mode 100755 Arrs_Stack/arr_corruption_scan.sh diff --git a/Arrs_Stack/arr_corruption_scan.sh b/Arrs_Stack/arr_corruption_scan.sh new file mode 100755 index 0000000..6a2bc13 --- /dev/null +++ b/Arrs_Stack/arr_corruption_scan.sh @@ -0,0 +1,390 @@ +#!/bin/bash +# ============================================================================================== +# ============================ Arr Corruption Scan ============================================== +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Scans Sonarr's tracked video files for corrupt headers (ffprobe-based, same detection +# method as the third-party Healarr tool) and, in --remediate mode, deletes the bad file +# from Sonarr and explicitly triggers an EpisodeSearch to replace it. +# +# Built after Healarr crashed mid-scan on a genuine Go concurrency bug (unsynchronized +# map access when multiple corruption events land at once — confirmed via its own crash +# log, not fixable from our side). The core idea (scan → delete → re-search) isn't hard to +# replicate; the fix here is architectural: this script processes one file at a time, +# strictly sequential, so the race condition that killed Healarr can't happen — there's +# nothing running concurrently to race. +# +# ============================================================================================== +# WHY A SEPARATE CONTAINER FOR FFPROBE +# ============================================================================================== +# +# Neither Sonarr nor Radarr bundle ffprobe. ffprobe runs via `docker exec` into a +# different container that does — confirmed live 2026-07-18: +# Jellyfin — working ffprobe, but only mounts Tv_Shows + Movies +# Emby — mounts everything (kids/anime shares too), but its bundled +# ffprobe binary is broken (2017-dated, fails to exec — likely a +# missing dynamic linker dependency, not something to fix here) +# Jellyfin is what's configured (HOST*_FFPROBE_CONTAINER) — matches the current TV-share +# testing scope. Kids/anime coverage needs a working ffprobe source with those mounts +# before this script can cover those shares; not solved yet. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# One file at a time, in this order per file: +# 1. Skip if unchanged (mtime+size) since the last time it verified clean — state file +# avoids re-probing the entire library every run, which would take far too long at +# this library size (90k+ tracked files). +# 2. ffprobe via `docker exec` into FFPROBE_CONTAINER. Empty stderr + exit 0 = clean. +# Anything else = corrupt (same signature as Healarr: "Invalid data found when +# processing input", EBML header errors, etc.) +# 3. Report-only by default. --remediate additionally: +# a. DELETE the specific episodefile record via Sonarr's API +# b. Verify hasFile flipped false (never trust the DELETE response alone) +# c. Explicitly trigger EpisodeSearch for that episode — this is deliberate, not +# left to Sonarr's own background missing-search cycle, because that cycle +# skips unmonitored episodes entirely. An explicit EpisodeSearch call does not +# have that restriction (confirmed live: two unmonitored episodes Healarr +# healed both still got successfully re-grabbed via this exact same kind of +# search call, logged in Sonarr's history as "UserInvokedSearch"). +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# host*.conf +# SONARR_URL / SONARR_API_KEY / SONARR_PATH_MAP — existing, aliased by detect_hosts() +# HOST*_FFPROBE_CONTAINER — container name with a working ffprobe binary +# HOST*_FFPROBE_BIN — full path to that binary inside the container +# HOST*_FFPROBE_PATH_MAP — host path prefix → that container's internal path prefix +# (separate from the arr's own path map — the ffprobe +# container almost certainly mounts shares differently) +# +# master.conf +# CORRUPTION_SCAN_STATE_FILE — path to the clean-file skip-cache (default in DATA_DIR) +# SONARR_VERSION_MAJOR — reused from sonarr_cleanup.sh for the API version check +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# arr_corruption_scan.sh — report-only, scans everything not yet +# verified clean +# arr_corruption_scan.sh --remediate — delete + re-search on every corrupt file found +# arr_corruption_scan.sh --limit=50 — cap this run to 50 newly-probed files (state +# file makes repeat runs cheap regardless, but +# useful for a bounded first test) +# arr_corruption_scan.sh --log — verbose (prints every clean file too) +# arr_corruption_scan.sh --status — show config and exit +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../load_config.sh" + +# --remediate / --limit are script-local, not recognized by parse_args — check raw args +# before they get filtered. +REMEDIATE=false +SCAN_LIMIT=0 +for _arg in "$@"; do + case "$_arg" in + --remediate) REMEDIATE=true ;; + --limit=*) SCAN_LIMIT="${_arg#*=}" ;; + esac +done +unset _arg + +parse_args "$@" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +if ! command -v curl >/dev/null 2>&1; then + error "curl not found — required for Sonarr API calls" + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + error "jq not found — required for JSON parsing" + exit 1 +fi + +if ! command -v docker &>/dev/null; then + error "Docker command not found" + exit 1 +fi + +acquire_lock "wait" +TMP_DIR="/tmp/arr_corruption_scan_$$" +mkdir -p "$TMP_DIR" +trap "_release_all_locks; rm -rf $TMP_DIR" EXIT + +detect_hosts + +if [[ -z "${SONARR_URL:-}" ]] || [[ -z "${SONARR_API_KEY:-}" ]]; then + info "Sonarr not configured on $MY_ID ($LOCAL_SERVER_NAME) — skipping" + exit 0 +fi + +build_arr_path_map "SONARR" + +# FFPROBE_* aren't part of the shared arr alias set in detect_hosts() — resolve them here, +# same eval-based pattern build_arr_path_map() uses for the associative array. +FFPROBE_CONTAINER_VAR="${MY_ID}_FFPROBE_CONTAINER" +FFPROBE_CONTAINER="${!FFPROBE_CONTAINER_VAR:-}" +FFPROBE_BIN_VAR="${MY_ID}_FFPROBE_BIN" +FFPROBE_BIN="${!FFPROBE_BIN_VAR:-}" + +declare -A FFPROBE_PATH_MAP=() +_fp_map_var="${MY_ID}_FFPROBE_PATH_MAP" +eval "for key in \"\${!${_fp_map_var}[@]}\"; do + FFPROBE_PATH_MAP[\"\$key\"]=\"\${${_fp_map_var}[\$key]}\" +done" +unset _fp_map_var + +if [[ -z "$FFPROBE_CONTAINER" || -z "$FFPROBE_BIN" ]]; then + error "FFPROBE_CONTAINER/FFPROBE_BIN not configured on $MY_ID — skipping" + exit 0 +fi + +CORRUPTION_SCAN_STATE_FILE="${CORRUPTION_SCAN_STATE_FILE:-$DATA_DIR/corruption_scan_state.tsv}" +mkdir -p "$(dirname "$CORRUPTION_SCAN_STATE_FILE")" +touch "$CORRUPTION_SCAN_STATE_FILE" + +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_GEAR Sonarr URL: $SONARR_URL" + echo "$ICON_GEAR FFprobe container: $FFPROBE_CONTAINER" + echo "$ICON_GEAR FFprobe binary: $FFPROBE_BIN" + echo "$ICON_GEAR FFprobe path map: ${#FFPROBE_PATH_MAP[@]} entries" + echo "$ICON_GEAR State file: $CORRUPTION_SCAN_STATE_FILE" + echo "$ICON_GEAR Remediate: $REMEDIATE" + echo "$ICON_GEAR Scan limit: ${SCAN_LIMIT:-unlimited}" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +echo "" +echo " $MY_ID ($LOCAL_SERVER_NAME) — $SONARR_URL" +[[ "$REMEDIATE" == true ]] && warn "REMEDIATE MODE — corrupt files will be deleted and re-searched" \ + || info "Report-only — pass --remediate to act" + +echo "" +echo "━━━ $ICON_SHIELD Safety Checks ━━━" +check_container_health "$FFPROBE_CONTAINER" 15 "Corruption Scan" +check_container_health "Sonarr" 15 "Corruption Scan" + +check_arr_version "$SONARR_URL" "$SONARR_API_KEY" "v3" "$SONARR_VERSION_MAJOR" "Sonarr" || exit 1 + +# ============================================================================================== +# ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# Translates a host filesystem path to FFPROBE_CONTAINER's internal path via prefix match +# against FFPROBE_PATH_MAP. Empty output (return 1) means this file's share isn't covered +# by the ffprobe container yet — caller must skip, not guess. +ffprobe_translate_path() { + local host_path="$1" prefix + for prefix in "${!FFPROBE_PATH_MAP[@]}"; do + if [[ "$host_path" == "$prefix"/* ]]; then + echo "${FFPROBE_PATH_MAP[$prefix]}${host_path#$prefix}" + return 0 + fi + done + return 1 +} + +# Probes one file. Echoes "clean" or "corrupt:". Never trusts a truncated/garbled +# stderr as automatically corrupt — only a real non-empty ffprobe stderr counts. +probe_file() { + local host_path="$1" container_path output + container_path=$(ffprobe_translate_path "$host_path") || { echo "unmapped"; return; } + output=$(docker exec "$FFPROBE_CONTAINER" "$FFPROBE_BIN" -v error "$container_path" 2>&1) + if [[ -z "$output" ]]; then + echo "clean" + else + echo "corrupt:${output//$'\n'/ }" + fi +} + +# ============================================================================================== +# ━━━ Load clean-file state (skip cache) ━━━ +# ============================================================================================== +declare -A CLEAN_STATE +while IFS=$'\t' read -r _s_path _s_stamp; do + [[ -n "$_s_path" ]] && CLEAN_STATE["$_s_path"]="$_s_stamp" +done < "$CORRUPTION_SCAN_STATE_FILE" +unset _s_path _s_stamp +info "Loaded ${#CLEAN_STATE[@]} previously-verified-clean entries" + +# ============================================================================================== +# ━━━ Fetch Sonarr's tracked episode files ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_SYNC Fetching Sonarr Tracked Files ━━━" + +# Prefer the shared per-episode-file cache written by sonarr_cleanup.sh (has id/episodeId/ +# seriesId/path already) — falls back to a live per-series walk only on a genuine miss. +EPISODE_FILES=$(arr_get_cached_items "sonarr" 14400) +if [[ -z "$EPISODE_FILES" || "$EPISODE_FILES" == "null" ]]; then + info "No fresh cached episode-file data — fetching live (this is the slow path)" + SERIES_RESPONSE=$(arr_get_tracked_data "sonarr" "$SONARR_URL" "$SONARR_API_KEY" "v3") || { + error "Failed to fetch series from Sonarr" + exit 1 + } + SERIES_IDS=$(echo "$SERIES_RESPONSE" | jq -r '.[].id') + all_tmp=$(mktemp) + while IFS= read -r sid; do + [[ -z "$sid" ]] && continue + arr_api "$SONARR_URL" "$SONARR_API_KEY" "v3" "episodefile?seriesId=${sid}" "Sonarr" 2>/dev/null \ + >> "$all_tmp" + done <<< "$SERIES_IDS" + EPISODE_FILES=$(jq -s 'add // []' "$all_tmp" 2>/dev/null) + rm -f "$all_tmp" + arr_item_cache_write "sonarr" "$EPISODE_FILES" +fi + +FILE_COUNT=$(echo "$EPISODE_FILES" | jq 'length' 2>/dev/null) +if [[ -z "$FILE_COUNT" || "$FILE_COUNT" -eq 0 ]]; then + error "0 tracked episode files — aborting" + exit 1 +fi +info "$FILE_COUNT tracked episode files" + +# ============================================================================================== +# ━━━ Scan ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_CLEAN Scanning ━━━" + +SCANNED=0 +SKIPPED_CACHED=0 +SKIPPED_UNMAPPED=0 +CORRUPT_COUNT=0 +REMEDIATED=0 +REMEDIATE_FAILED=0 + +# Fresh clean-file stamps are appended here as they're found (O(1) per file) rather than +# rewritten into the growing state file on every hit — at this library's scale (90k+ files) +# a per-file rewrite-the-whole-file approach would be O(n²) and far too slow. +FRESH_CLEAN_TMP=$(mktemp) + +while IFS= read -r item; do + api_path=$(echo "$item" | jq -r '.path') + episode_file_id=$(echo "$item" | jq -r '.id') + series_id=$(echo "$item" | jq -r '.seriesId') + + host_path=$(translate_path "$api_path") + [[ -f "$host_path" ]] || continue + + stamp="$(stat -c '%Y:%s' "$host_path" 2>/dev/null)" + [[ -z "$stamp" ]] && continue + + if [[ "${CLEAN_STATE[$host_path]:-}" == "$stamp" ]]; then + (( SKIPPED_CACHED++ )) + continue + fi + + (( SCANNED++ )) + if [[ "$SCAN_LIMIT" -gt 0 && "$SCANNED" -gt "$SCAN_LIMIT" ]]; then + (( SCANNED-- )) + break + fi + + result=$(probe_file "$host_path") + + if [[ "$result" == "unmapped" ]]; then + (( SKIPPED_UNMAPPED++ )) + [[ "$ENABLE_LOGGING" == true ]] && warn " ? $host_path — no FFPROBE_PATH_MAP entry covers this share" + continue + fi + + if [[ "$result" == "clean" ]]; then + echo -e "${host_path}\t${stamp}" >> "$FRESH_CLEAN_TMP" + [[ "$ENABLE_LOGGING" == true ]] && echo " $ICON_SUCCESS $host_path" + continue + fi + + # corrupt: + reason="${result#corrupt:}" + (( CORRUPT_COUNT++ )) + echo " $ICON_ERROR CORRUPT: $host_path" + [[ "$ENABLE_LOGGING" == true ]] && echo " $reason" + + if [[ "$REMEDIATE" != true ]]; then + continue + fi + + episode_id=$(echo "$item" | jq -r '.episodeId // empty') + title=$(echo "$item" | jq -r '.sceneName // .relativePath // .path') + + http_code=$(curl -sf -o /dev/null -w "%{http_code}" -X DELETE \ + --max-time 15 -H "X-Api-Key: $SONARR_API_KEY" \ + "${SONARR_URL}/api/v3/episodefile/${episode_file_id}" 2>/dev/null) + + if [[ "$http_code" != "200" ]]; then + error " ✗ $title — delete failed (HTTP $http_code)" + (( REMEDIATE_FAILED++ )) + continue + fi + + sleep 2 + verify_hasfile=$(arr_api "$SONARR_URL" "$SONARR_API_KEY" "v3" "episode/${episode_id}" "Sonarr" 2>/dev/null \ + | jq -r '.hasFile // "unknown"') + + if [[ "$verify_hasfile" != "false" ]]; then + error " ✗ $title — deleted but hasFile still '$verify_hasfile' — not searching, needs review" + (( REMEDIATE_FAILED++ )) + continue + fi + + search_code=$(curl -sf -o /dev/null -w "%{http_code}" -X POST \ + --max-time 30 -H "X-Api-Key: $SONARR_API_KEY" -H "Content-Type: application/json" \ + -d "{\"name\":\"EpisodeSearch\",\"episodeIds\":[${episode_id}]}" \ + "${SONARR_URL}/api/v3/command" 2>/dev/null) + + if [[ "$search_code" == "200" || "$search_code" == "201" ]]; then + echo " $ICON_SUCCESS $title — deleted, verified, search triggered" + (( REMEDIATED++ )) + else + warn " $title — deleted and verified, but search trigger returned HTTP $search_code" + (( REMEDIATE_FAILED++ )) + fi +done < <(echo "$EPISODE_FILES" | jq -c '.[]') + +# Merge fresh clean-file stamps with the existing state, newest wins per path — reading +# fresh entries first (before the old base file) means the first occurrence tac/awk keeps +# is always the newest one for any path re-verified this run. +STATE_TMP=$(mktemp) +cat "$FRESH_CLEAN_TMP" "$CORRUPTION_SCAN_STATE_FILE" | awk -F'\t' '!seen[$1]++' | sort > "$STATE_TMP" +mv "$STATE_TMP" "$CORRUPTION_SCAN_STATE_FILE" +rm -f "$FRESH_CLEAN_TMP" + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== +echo "" +echo "━━━━━ $ICON_SUMMARY CORRUPTION SCAN SUMMARY ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_SYNC Newly scanned: $SCANNED" +echo "$ICON_SUCCESS Skipped (cached): $SKIPPED_CACHED" +echo "$ICON_WARN Skipped (unmapped): $SKIPPED_UNMAPPED" +echo "$ICON_ERROR Corrupt found: $CORRUPT_COUNT" +if [[ "$REMEDIATE" == true ]]; then + echo "$ICON_SUCCESS Remediated: $REMEDIATED" + echo "$ICON_ERROR Remediation failed: $REMEDIATE_FAILED" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +exit 0 diff --git a/Deployment/host.conf.template b/Deployment/host.conf.template index 38cac4a..4ab6719 100644 --- a/Deployment/host.conf.template +++ b/Deployment/host.conf.template @@ -457,6 +457,17 @@ # ["/movies"]="/mnt/user/Movies" ) +# ━━━ Corruption Scan ━━━ + # Container that has a working ffprobe binary AND mounts the same shares as the arrs' + # roots — check `docker inspect ` for its mounts before filling this in. + HOSTN_FFPROBE_CONTAINER="" + HOSTN_FFPROBE_BIN="" + + declare -A HOSTN_FFPROBE_PATH_MAP=( + # ["/mnt/user/Tv_Shows"]="/ext-tv-shows" + # ["/mnt/user/Movies"]="/ext-movies" + ) + # ━━━ Arr Recovery Toggles ━━━ HOSTN_LIDARR_RECOVERY=false HOSTN_SONARR_RECOVERY=true diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index fb47f96..2c31d5c 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -1184,6 +1184,7 @@ # reach "completed" — generous because a large series can sit # queued behind other moves already in progress, not just its # own copy time + CORRUPTION_SCAN_STATE_FILE="${DATA_DIR}/corruption_scan_state.tsv" # clean-file skip-cache SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") SONARR_PROTECTED_PATTERNS=( # Subtitles