#!/bin/bash # ============================================================================================== # ============================ Arr Corruption Scan ============================================== # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Scans Sonarr's tracked video files for corrupt headers (ffprobe-based, same detection # method as the third-party Healarr tool) and, in --remediate mode, deletes the bad file # from Sonarr and explicitly triggers an EpisodeSearch to replace it. # # Built after Healarr crashed mid-scan on a genuine Go concurrency bug (unsynchronized # map access when multiple corruption events land at once — confirmed via its own crash # log, not fixable from our side). The core idea (scan → delete → re-search) isn't hard to # replicate; the fix here is architectural: this script processes one file at a time, # strictly sequential, so the race condition that killed Healarr can't happen — there's # nothing running concurrently to race. # # ============================================================================================== # WHY A SEPARATE CONTAINER FOR FFPROBE # ============================================================================================== # # Neither Sonarr nor Radarr bundle ffprobe. ffprobe runs via `docker exec` into a # different container that does — confirmed live 2026-07-18: # Jellyfin — working ffprobe, but only mounts Tv_Shows + Movies # Emby — mounts everything (kids/anime shares too), but its bundled # ffprobe binary is broken (2017-dated, fails to exec — likely a # missing dynamic linker dependency, not something to fix here) # Jellyfin is what's configured (HOST*_FFPROBE_CONTAINER) — matches the current TV-share # testing scope. Kids/anime coverage needs a working ffprobe source with those mounts # before this script can cover those shares; not solved yet. # # ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # # One file at a time, in this order per file: # 1. Skip if unchanged (mtime+size) since the last time it verified clean — state file # avoids re-probing the entire library every run, which would take far too long at # this library size (90k+ tracked files). # 2. ffprobe via `docker exec` into FFPROBE_CONTAINER. Empty stderr + exit 0 = clean. # Anything else = corrupt (same signature as Healarr: "Invalid data found when # processing input", EBML header errors, etc.) # 3. Report-only by default. --remediate additionally: # a. DELETE the specific episodefile record via Sonarr's API # b. Verify hasFile flipped false (never trust the DELETE response alone) # c. Explicitly trigger EpisodeSearch for that episode — this is deliberate, not # left to Sonarr's own background missing-search cycle, because that cycle # skips unmonitored episodes entirely. An explicit EpisodeSearch call does not # have that restriction (confirmed live: two unmonitored episodes Healarr # healed both still got successfully re-grabbed via this exact same kind of # search call, logged in Sonarr's history as "UserInvokedSearch"). # # ============================================================================================== # CONFIGURATION # ============================================================================================== # # host*.conf # SONARR_URL / SONARR_API_KEY / SONARR_PATH_MAP — existing, aliased by detect_hosts() # HOST*_FFPROBE_CONTAINER — container name with a working ffprobe binary # HOST*_FFPROBE_BIN — full path to that binary inside the container # HOST*_FFPROBE_PATH_MAP — host path prefix → that container's internal path prefix # (separate from the arr's own path map — the ffprobe # container almost certainly mounts shares differently) # # master.conf # CORRUPTION_SCAN_STATE_FILE — path to the clean-file skip-cache (default in DATA_DIR) # SONARR_VERSION_MAJOR — reused from sonarr_cleanup.sh for the API version check # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # arr_corruption_scan.sh — report-only, scans everything not yet # verified clean # arr_corruption_scan.sh --remediate — delete + re-search on every corrupt file found # arr_corruption_scan.sh --limit=50 — cap this run to 50 newly-probed files (state # file makes repeat runs cheap regardless, but # useful for a bounded first test) # arr_corruption_scan.sh --log — verbose (prints every clean file too) # arr_corruption_scan.sh --status — show config and exit # arr_corruption_scan.sh --filter=Becker — only consider paths containing this substring # (testing/targeting a specific show; state # file and everything else behaves normally) # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" # --remediate / --limit are script-local, not recognized by parse_args — check raw args # before they get filtered. REMEDIATE=false SCAN_LIMIT=0 PATH_FILTER="" for _arg in "$@"; do case "$_arg" in --remediate) REMEDIATE=true ;; --limit=*) SCAN_LIMIT="${_arg#*=}" ;; --filter=*) PATH_FILTER="${_arg#*=}" ;; esac done unset _arg parse_args "$@" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi if ! command -v curl >/dev/null 2>&1; then error "curl not found — required for Sonarr API calls" exit 1 fi if ! command -v jq >/dev/null 2>&1; then error "jq not found — required for JSON parsing" exit 1 fi if ! command -v docker &>/dev/null; then error "Docker command not found" exit 1 fi acquire_lock "wait" TMP_DIR="/tmp/arr_corruption_scan_$$" mkdir -p "$TMP_DIR" trap "_release_all_locks; rm -rf $TMP_DIR" EXIT detect_hosts if [[ -z "${SONARR_URL:-}" ]] || [[ -z "${SONARR_API_KEY:-}" ]]; then info "Sonarr not configured on $MY_ID ($LOCAL_SERVER_NAME) — skipping" exit 0 fi build_arr_path_map "SONARR" # FFPROBE_* aren't part of the shared arr alias set in detect_hosts() — resolve them here, # same eval-based pattern build_arr_path_map() uses for the associative array. FFPROBE_CONTAINER_VAR="${MY_ID}_FFPROBE_CONTAINER" FFPROBE_CONTAINER="${!FFPROBE_CONTAINER_VAR:-}" FFPROBE_BIN_VAR="${MY_ID}_FFPROBE_BIN" FFPROBE_BIN="${!FFPROBE_BIN_VAR:-}" declare -A FFPROBE_PATH_MAP=() _fp_map_var="${MY_ID}_FFPROBE_PATH_MAP" eval "for key in \"\${!${_fp_map_var}[@]}\"; do FFPROBE_PATH_MAP[\"\$key\"]=\"\${${_fp_map_var}[\$key]}\" done" unset _fp_map_var if [[ -z "$FFPROBE_CONTAINER" || -z "$FFPROBE_BIN" ]]; then error "FFPROBE_CONTAINER/FFPROBE_BIN not configured on $MY_ID — skipping" exit 0 fi CORRUPTION_SCAN_STATE_FILE="${CORRUPTION_SCAN_STATE_FILE:-$DATA_DIR/corruption_scan_state.tsv}" mkdir -p "$(dirname "$CORRUPTION_SCAN_STATE_FILE")" touch "$CORRUPTION_SCAN_STATE_FILE" if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_GEAR Sonarr URL: $SONARR_URL" echo "$ICON_GEAR FFprobe container: $FFPROBE_CONTAINER" echo "$ICON_GEAR FFprobe binary: $FFPROBE_BIN" echo "$ICON_GEAR FFprobe path map: ${#FFPROBE_PATH_MAP[@]} entries" echo "$ICON_GEAR State file: $CORRUPTION_SCAN_STATE_FILE" echo "$ICON_GEAR Remediate: $REMEDIATE" echo "$ICON_GEAR Scan limit: ${SCAN_LIMIT:-unlimited}" echo "$ICON_GEAR Path filter: ${PATH_FILTER:-none}" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi echo "" echo " $MY_ID ($LOCAL_SERVER_NAME) — $SONARR_URL" [[ "$REMEDIATE" == true ]] && warn "REMEDIATE MODE — corrupt files will be deleted and re-searched" \ || info "Report-only — pass --remediate to act" echo "" echo "━━━ $ICON_SHIELD Safety Checks ━━━" check_container_health "$FFPROBE_CONTAINER" 15 "Corruption Scan" check_container_health "Sonarr" 15 "Corruption Scan" check_arr_version "$SONARR_URL" "$SONARR_API_KEY" "v3" "$SONARR_VERSION_MAJOR" "Sonarr" || exit 1 # ============================================================================================== # ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Translates a host filesystem path to FFPROBE_CONTAINER's internal path via prefix match # against FFPROBE_PATH_MAP. Empty output (return 1) means this file's share isn't covered # by the ffprobe container yet — caller must skip, not guess. ffprobe_translate_path() { local host_path="$1" prefix for prefix in "${!FFPROBE_PATH_MAP[@]}"; do if [[ "$host_path" == "$prefix"/* ]]; then echo "${FFPROBE_PATH_MAP[$prefix]}${host_path#$prefix}" return 0 fi done return 1 } # Probes one file. Echoes "clean" or "corrupt:". Never trusts a truncated/garbled # stderr as automatically corrupt — only a real non-empty ffprobe stderr counts. probe_file() { local host_path="$1" container_path output container_path=$(ffprobe_translate_path "$host_path") || { echo "unmapped"; return; } output=$(docker exec "$FFPROBE_CONTAINER" "$FFPROBE_BIN" -v error "$container_path" 2>&1) if [[ -z "$output" ]]; then echo "clean" else echo "corrupt:${output//$'\n'/ }" fi } # ============================================================================================== # ━━━ Load clean-file state (skip cache) ━━━ # ============================================================================================== declare -A CLEAN_STATE while IFS=$'\t' read -r _s_path _s_stamp; do [[ -n "$_s_path" ]] && CLEAN_STATE["$_s_path"]="$_s_stamp" done < "$CORRUPTION_SCAN_STATE_FILE" unset _s_path _s_stamp info "Loaded ${#CLEAN_STATE[@]} previously-verified-clean entries" # ============================================================================================== # ━━━ Fetch Sonarr's tracked episode files ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_SYNC Fetching Sonarr Tracked Files ━━━" # Prefer the shared per-episode-file cache written by sonarr_cleanup.sh (has id/episodeId/ # seriesId/path already) — falls back to a live per-series walk only on a genuine miss. EPISODE_FILES=$(arr_get_cached_items "sonarr" 14400) if [[ -z "$EPISODE_FILES" || "$EPISODE_FILES" == "null" ]]; then info "No fresh cached episode-file data — fetching live (this is the slow path)" SERIES_RESPONSE=$(arr_get_tracked_data "sonarr" "$SONARR_URL" "$SONARR_API_KEY" "v3") || { error "Failed to fetch series from Sonarr" exit 1 } SERIES_IDS=$(echo "$SERIES_RESPONSE" | jq -r '.[].id') all_tmp=$(mktemp) while IFS= read -r sid; do [[ -z "$sid" ]] && continue arr_api "$SONARR_URL" "$SONARR_API_KEY" "v3" "episodefile?seriesId=${sid}" "Sonarr" 2>/dev/null \ >> "$all_tmp" done <<< "$SERIES_IDS" EPISODE_FILES=$(jq -s 'add // []' "$all_tmp" 2>/dev/null) rm -f "$all_tmp" arr_item_cache_write "sonarr" "$EPISODE_FILES" fi FILE_COUNT=$(echo "$EPISODE_FILES" | jq 'length' 2>/dev/null) if [[ -z "$FILE_COUNT" || "$FILE_COUNT" -eq 0 ]]; then error "0 tracked episode files — aborting" exit 1 fi info "$FILE_COUNT tracked episode files" # ============================================================================================== # ━━━ Scan ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_CLEAN Scanning ━━━" SCANNED=0 SKIPPED_CACHED=0 SKIPPED_UNMAPPED=0 CORRUPT_COUNT=0 REMEDIATED=0 REMEDIATE_FAILED=0 # Fresh clean-file stamps are appended here as they're found (O(1) per file) rather than # rewritten into the growing state file on every hit — at this library's scale (90k+ files) # a per-file rewrite-the-whole-file approach would be O(n²) and far too slow. FRESH_CLEAN_TMP=$(mktemp) while IFS= read -r item; do api_path=$(echo "$item" | jq -r '.path') episode_file_id=$(echo "$item" | jq -r '.id') series_id=$(echo "$item" | jq -r '.seriesId') host_path=$(translate_path "$api_path") [[ -f "$host_path" ]] || continue [[ -n "$PATH_FILTER" && "$host_path" != *"$PATH_FILTER"* ]] && continue stamp="$(stat -c '%Y:%s' "$host_path" 2>/dev/null)" [[ -z "$stamp" ]] && continue if [[ "${CLEAN_STATE[$host_path]:-}" == "$stamp" ]]; then (( SKIPPED_CACHED++ )) continue fi (( SCANNED++ )) if [[ "$SCAN_LIMIT" -gt 0 && "$SCANNED" -gt "$SCAN_LIMIT" ]]; then (( SCANNED-- )) break fi result=$(probe_file "$host_path") if [[ "$result" == "unmapped" ]]; then (( SKIPPED_UNMAPPED++ )) [[ "$ENABLE_LOGGING" == true ]] && warn " ? $host_path — no FFPROBE_PATH_MAP entry covers this share" continue fi if [[ "$result" == "clean" ]]; then echo -e "${host_path}\t${stamp}" >> "$FRESH_CLEAN_TMP" [[ "$ENABLE_LOGGING" == true ]] && echo " $ICON_SUCCESS $host_path" continue fi # corrupt: reason="${result#corrupt:}" (( CORRUPT_COUNT++ )) echo " $ICON_ERROR CORRUPT: $host_path" [[ "$ENABLE_LOGGING" == true ]] && echo " $reason" if [[ "$REMEDIATE" != true ]]; then continue fi episode_id=$(echo "$item" | jq -r '.episodeId // empty') title=$(echo "$item" | jq -r '.sceneName // .relativePath // .path') http_code=$(curl -sf -o /dev/null -w "%{http_code}" -X DELETE \ --max-time 15 -H "X-Api-Key: $SONARR_API_KEY" \ "${SONARR_URL}/api/v3/episodefile/${episode_file_id}" 2>/dev/null) if [[ "$http_code" != "200" ]]; then error " ✗ $title — delete failed (HTTP $http_code)" (( REMEDIATE_FAILED++ )) continue fi sleep 2 verify_hasfile=$(arr_api "$SONARR_URL" "$SONARR_API_KEY" "v3" "episode/${episode_id}" "Sonarr" 2>/dev/null \ | jq -r '.hasFile // "unknown"') if [[ "$verify_hasfile" != "false" ]]; then error " ✗ $title — deleted but hasFile still '$verify_hasfile' — not searching, needs review" (( REMEDIATE_FAILED++ )) continue fi search_code=$(curl -sf -o /dev/null -w "%{http_code}" -X POST \ --max-time 30 -H "X-Api-Key: $SONARR_API_KEY" -H "Content-Type: application/json" \ -d "{\"name\":\"EpisodeSearch\",\"episodeIds\":[${episode_id}]}" \ "${SONARR_URL}/api/v3/command" 2>/dev/null) if [[ "$search_code" == "200" || "$search_code" == "201" ]]; then echo " $ICON_SUCCESS $title — deleted, verified, search triggered" (( REMEDIATED++ )) else warn " $title — deleted and verified, but search trigger returned HTTP $search_code" (( REMEDIATE_FAILED++ )) fi done < <(echo "$EPISODE_FILES" | jq -c '.[]') # Merge fresh clean-file stamps with the existing state, newest wins per path — reading # fresh entries first (before the old base file) means the first occurrence tac/awk keeps # is always the newest one for any path re-verified this run. STATE_TMP=$(mktemp) cat "$FRESH_CLEAN_TMP" "$CORRUPTION_SCAN_STATE_FILE" | awk -F'\t' '!seen[$1]++' | sort > "$STATE_TMP" mv "$STATE_TMP" "$CORRUPTION_SCAN_STATE_FILE" rm -f "$FRESH_CLEAN_TMP" # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY CORRUPTION SCAN SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_SYNC Newly scanned: $SCANNED" echo "$ICON_SUCCESS Skipped (cached): $SKIPPED_CACHED" echo "$ICON_WARN Skipped (unmapped): $SKIPPED_UNMAPPED" echo "$ICON_ERROR Corrupt found: $CORRUPT_COUNT" if [[ "$REMEDIATE" == true ]]; then echo "$ICON_SUCCESS Remediated: $REMEDIATED" echo "$ICON_ERROR Remediation failed: $REMEDIATE_FAILED" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" exit 0