790 lines
38 KiB
Bash
790 lines
38 KiB
Bash
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================ Arr Corruption Scan ==============================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Scans Sonarr's and Radarr's tracked video files for corrupt headers (ffprobe-based, same
|
|
# detection method as the third-party Healarr tool) and, in --remediate mode, deletes the bad
|
|
# file from the owning arr and explicitly triggers a search to replace it.
|
|
#
|
|
# Built after Healarr crashed mid-scan on a genuine Go concurrency bug (unsynchronized
|
|
# map access when multiple corruption events land at once — confirmed via its own crash
|
|
# log, not fixable from our side). The core idea (scan → delete → re-search) isn't hard to
|
|
# replicate; the fix here is architectural: this script processes one file at a time,
|
|
# strictly sequential, so the race condition that killed Healarr can't happen — there's
|
|
# nothing running concurrently to race.
|
|
#
|
|
# Sonarr-only originally (2026-07-18/19); Radarr/Movies coverage added 2026-07-21 as a second
|
|
# arr in the same per-file scan/strike/remediate loop, not a separate script — the detection,
|
|
# strike, and state-file logic is identical, only the API shape (episodefile vs moviefile,
|
|
# EpisodeSearch vs MoviesSearch) differs. Radarr's moviefile list is fetched batched
|
|
# (movieId=... query params, BATCH_SIZE at a time) rather than off the movie list's embedded
|
|
# .movieFile alone — Radarr supports a second tracked file per movie (alternate editions/
|
|
# extras) that never shows up there, same gap radarr_cleanup.sh hit and fixed 2026-07-19;
|
|
# reusing that batched-fetch shape here instead of the simpler single-file read so a
|
|
# corruption scan doesn't silently skip every alternate edition in the library.
|
|
#
|
|
# ==============================================================================================
|
|
# WHY A SEPARATE CONTAINER FOR FFPROBE
|
|
# ==============================================================================================
|
|
#
|
|
# Neither Sonarr nor Radarr bundle ffprobe. ffprobe runs via `docker exec` into a
|
|
# different container that does — confirmed live 2026-07-18:
|
|
# Jellyfin — working ffprobe, mounts every share Emby does (Tv_Shows, Movies, kids/
|
|
# anime shares, standup) as of 2026-07-18
|
|
# Emby — mounts everything too, but its bundled ffprobe binary is broken
|
|
# (2017-dated, fails to exec — likely a missing dynamic linker
|
|
# dependency, not something to fix here)
|
|
# Jellyfin is what's configured (HOST*_FFPROBE_CONTAINER).
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# Runs Sonarr then Radarr, sequentially, never parallel — one arr failing/unconfigured never
|
|
# blocks the other. Within each arr, one file at a time, in this order per file:
|
|
# 1. Skip if unchanged (mtime+size) since the last time it verified clean — state file
|
|
# avoids re-probing the entire library every run, which would take far too long at
|
|
# this library size (90k+ tracked files).
|
|
# 2. ffprobe via `docker exec` into FFPROBE_CONTAINER. Empty stderr + exit 0 = clean.
|
|
# Anything else = corrupt (same signature as Healarr: "Invalid data found when
|
|
# processing input", EBML header errors, etc.)
|
|
# 3. Report-only by default. --remediate additionally:
|
|
# a. DELETE the specific episodefile/moviefile record via the arr's API
|
|
# b. Verify hasFile flipped false (never trust the DELETE response alone)
|
|
# c. Explicitly trigger EpisodeSearch/MoviesSearch for that episode/movie — this is
|
|
# deliberate, not left to the arr's own background missing-search cycle, because
|
|
# that cycle skips unmonitored items entirely. An explicit search call does not
|
|
# have that restriction (confirmed live: two unmonitored episodes Healarr healed
|
|
# both still got successfully re-grabbed via this exact same kind of search call,
|
|
# logged in Sonarr's history as "UserInvokedSearch").
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Detect Always, Act Only on Request
|
|
# A bare run probes and reports. Deleting a file the arr believes it has is a
|
|
# destructive act, so it requires --remediate explicitly. The scan can be scheduled
|
|
# weekly and read without any risk of it removing media on its own.
|
|
#
|
|
# One Probe Result Is Not Evidence
|
|
# ffprobe can fail for reasons that have nothing to do with the file — a mid-write
|
|
# import, an NFS blip, a container restart. Corruption must be observed
|
|
# CORRUPTION_SCAN_STRIKE_LIMIT times consecutively before remediation acts, and a
|
|
# single clean re-probe resets the counter.
|
|
#
|
|
# Skip-Cache Over Re-Probing
|
|
# At 90k+ tracked files a full re-probe every run is not viable. Files unchanged by
|
|
# mtime and size since they last verified clean are skipped, so each run spends its
|
|
# time on what actually changed rather than re-proving the library from scratch.
|
|
#
|
|
# Delete the Record, Let the Arr Re-Acquire
|
|
# Remediation removes the file record and explicitly triggers a search. The arr is
|
|
# left to obtain a good copy through its normal path — this script never tries to
|
|
# repair a file in place.
|
|
#
|
|
# Explicit Search, Not the Background Cycle
|
|
# The re-search is triggered directly rather than left to the arr's own missing-search
|
|
# cycle, because that cycle skips unmonitored items entirely and would silently leave
|
|
# an unmonitored corrupt file deleted and never replaced.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root Enforcement
|
|
# docker exec into the ffprobe container requires root.
|
|
#
|
|
# Lock Acquisition
|
|
# acquire_lock prevents overlapping runs. Two instances would both probe and could
|
|
# both count a strike against the same file, reaching the limit in half the intended
|
|
# number of observations.
|
|
#
|
|
# Host Detection
|
|
# detect_hosts() aliases the arr URLs, API keys and HOST*_FFPROBE_CONTAINER.
|
|
#
|
|
# jq Dependency Check
|
|
# Fails fast if jq is missing — the tracked-file lists and every hasFile verification
|
|
# are parsed with it.
|
|
#
|
|
# Report-Only Default
|
|
# Nothing is deleted without --remediate.
|
|
#
|
|
# ffprobe Configuration Check
|
|
# Exits cleanly if FFPROBE_CONTAINER / FFPROBE_BIN are unconfigured for this host.
|
|
#
|
|
# ffprobe Container Health Check
|
|
# check_container_health() verifies the container is running and healthy before any
|
|
# probing. Every probe is a docker exec into it — if it is stopped or unhealthy every
|
|
# exec fails, every file reads as corrupt, and two consecutive runs would clear the
|
|
# strike limit and hand --remediate the whole library to delete.
|
|
#
|
|
# API Reachability + Version Gate
|
|
# check_api then check_arr_version per arr. A version mismatch skips that arr rather
|
|
# than issuing deletes against an API whose file-record endpoints may have moved.
|
|
#
|
|
# Per-Arr Isolation
|
|
# Sonarr and Radarr run sequentially, and one failing, unconfigured or version-
|
|
# mismatched arr never blocks the other.
|
|
#
|
|
# Unmapped Path Skip
|
|
# Files whose arr-side path cannot be mapped into the ffprobe container's mount
|
|
# namespace are skipped and counted, never probed through a wrong path and never
|
|
# treated as corrupt because the probe could not see them.
|
|
#
|
|
# Strike Threshold
|
|
# CORRUPTION_SCAN_STRIKE_LIMIT consecutive corrupt detections are required before
|
|
# --remediate deletes anything. A transient ffprobe failure cannot trigger a delete,
|
|
# and a clean re-probe clears the counter.
|
|
#
|
|
# Post-Delete Verification
|
|
# The DELETE response is never trusted. hasFile is re-checked and must have flipped
|
|
# false before the re-search is issued, so a failed delete never leaves the arr
|
|
# searching for something it still believes it has.
|
|
#
|
|
# Targeted Deletion
|
|
# Only the specific episodefile/moviefile record for the corrupt file is removed —
|
|
# never the series, movie, or any sibling file.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# host*.conf
|
|
# SONARR_URL / SONARR_API_KEY, RADARR_URL / RADARR_API_KEY — existing, aliased by
|
|
# detect_hosts(). Either arr missing its URL/key is skipped, not fatal.
|
|
# HOST*_FFPROBE_CONTAINER — container name with a working ffprobe binary
|
|
# HOST*_FFPROBE_BIN — full path to that binary inside the container
|
|
# HOST*_FFPROBE_PATH_MAP — host path prefix → that container's internal path prefix
|
|
# (separate from the arrs' own path maps — the ffprobe
|
|
# container almost certainly mounts shares differently)
|
|
#
|
|
# master.conf
|
|
# CORRUPTION_SCAN_STATE_FILE — path to the clean-file skip-cache (default in DATA_DIR),
|
|
# shared across both arrs — keyed by host path, which never
|
|
# collides between a Sonarr and a Radarr share
|
|
# CORRUPTION_SCAN_STRIKES_FILE — path to the consecutive-corrupt-detection counter (default
|
|
# in DATA_DIR), keyed by host path
|
|
# CORRUPTION_SCAN_STRIKE_LIMIT — consecutive corrupt detections required before --remediate
|
|
# acts on a file (default 2) — guards against a one-off
|
|
# ffprobe hiccup (mid-write file, NFS blip) triggering an
|
|
# unnecessary delete+re-search. Resets on a clean re-probe.
|
|
# SONARR_VERSION_MAJOR / RADARR_VERSION_MAJOR — reused from sonarr_cleanup.sh/
|
|
# radarr_cleanup.sh for the API version check
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# arr_corruption_scan.sh — report-only, scans everything not yet
|
|
# verified clean (Sonarr then Radarr)
|
|
# arr_corruption_scan.sh --remediate — delete + re-search on every corrupt file found
|
|
# arr_corruption_scan.sh --limit=50 — cap EACH arr to 50 newly-probed files this run
|
|
# (state file makes repeat runs cheap regardless,
|
|
# but useful for a bounded first test)
|
|
# arr_corruption_scan.sh --log — verbose (prints every clean file too)
|
|
# arr_corruption_scan.sh --status — show config and exit
|
|
# arr_corruption_scan.sh --filter=Becker — only consider paths containing this substring
|
|
# (testing/targeting a specific show/movie; state
|
|
# file and everything else behaves normally)
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
# --remediate / --limit are script-local, not recognized by parse_args — check raw args
|
|
# before they get filtered.
|
|
REMEDIATE=false
|
|
SCAN_LIMIT=0
|
|
PATH_FILTER=""
|
|
for _arg in "$@"; do
|
|
case "$_arg" in
|
|
--remediate) REMEDIATE=true ;;
|
|
--limit=*) SCAN_LIMIT="${_arg#*=}" ;;
|
|
--filter=*) PATH_FILTER="${_arg#*=}" ;;
|
|
esac
|
|
done
|
|
unset _arg
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v curl >/dev/null 2>&1; then
|
|
error "curl not found — required for arr API calls"
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v jq >/dev/null 2>&1; then
|
|
error "jq not found — required for JSON parsing"
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker command not found"
|
|
exit 1
|
|
fi
|
|
|
|
acquire_lock "wait"
|
|
TMP_DIR="/tmp/arr_corruption_scan_$$"
|
|
mkdir -p "$TMP_DIR"
|
|
trap "_release_all_locks; rm -rf $TMP_DIR" EXIT
|
|
|
|
detect_hosts
|
|
|
|
# FFPROBE_* aren't part of the shared arr alias set in detect_hosts() — resolve them here,
|
|
# same eval-based pattern build_arr_path_map() uses for the associative array.
|
|
FFPROBE_CONTAINER_VAR="${MY_ID}_FFPROBE_CONTAINER"
|
|
FFPROBE_CONTAINER="${!FFPROBE_CONTAINER_VAR:-}"
|
|
FFPROBE_BIN_VAR="${MY_ID}_FFPROBE_BIN"
|
|
FFPROBE_BIN="${!FFPROBE_BIN_VAR:-}"
|
|
|
|
declare -A FFPROBE_PATH_MAP=()
|
|
_fp_map_var="${MY_ID}_FFPROBE_PATH_MAP"
|
|
eval "for key in \"\${!${_fp_map_var}[@]}\"; do
|
|
FFPROBE_PATH_MAP[\"\$key\"]=\"\${${_fp_map_var}[\$key]}\"
|
|
done"
|
|
unset _fp_map_var
|
|
|
|
if [[ -z "$FFPROBE_CONTAINER" || -z "$FFPROBE_BIN" ]]; then
|
|
error "FFPROBE_CONTAINER/FFPROBE_BIN not configured on $MY_ID — skipping"
|
|
exit 0
|
|
fi
|
|
|
|
# Every probe is a docker exec into this container. If it is stopped or unhealthy, every
|
|
# exec fails, every file reads as corrupt, and two such runs would clear the strike limit
|
|
# and hand --remediate an entire library to delete. Abort before probing anything.
|
|
check_container_health "$FFPROBE_CONTAINER" "${DOCKER_TIMEOUT:-30}" "Arr Corruption Scan"
|
|
|
|
CORRUPTION_SCAN_STATE_FILE="${CORRUPTION_SCAN_STATE_FILE:-$DATA_DIR/corruption_scan_state.tsv}"
|
|
mkdir -p "$(dirname "$CORRUPTION_SCAN_STATE_FILE")"
|
|
touch "$CORRUPTION_SCAN_STATE_FILE"
|
|
|
|
CORRUPTION_SCAN_STRIKES_FILE="${CORRUPTION_SCAN_STRIKES_FILE:-$DATA_DIR/corruption_scan_strikes.tsv}"
|
|
CORRUPTION_SCAN_STRIKE_LIMIT="${CORRUPTION_SCAN_STRIKE_LIMIT:-2}"
|
|
CORRUPTION_SCAN_MAX_CORRUPT_PCT="${CORRUPTION_SCAN_MAX_CORRUPT_PCT:-10}"
|
|
CORRUPTION_SCAN_MAX_CONSECUTIVE="${CORRUPTION_SCAN_MAX_CONSECUTIVE:-15}"
|
|
CORRUPTION_SCAN_GUARD_MIN_SCANNED="${CORRUPTION_SCAN_GUARD_MIN_SCANNED:-20}"
|
|
mkdir -p "$(dirname "$CORRUPTION_SCAN_STRIKES_FILE")"
|
|
touch "$CORRUPTION_SCAN_STRIKES_FILE"
|
|
|
|
# Thin wrappers around common.sh's wd_state_get/wd_state_set — same shape as
|
|
# stability_watchdog.sh's get_strikes/set_strikes/increment_strikes/reset_strikes, keyed here
|
|
# by host path instead of a watchdog check name. Requires repeat corrupt detections across
|
|
# separate scan runs before --remediate acts, so a one-off ffprobe hiccup (mid-write file,
|
|
# NFS blip) can't trigger an unnecessary delete+re-search on its own.
|
|
get_scan_strikes() {
|
|
wd_state_get "$1" "$CORRUPTION_SCAN_STRIKES_FILE"
|
|
}
|
|
set_scan_strikes() {
|
|
wd_state_set "$1" "$2" "$CORRUPTION_SCAN_STRIKES_FILE"
|
|
}
|
|
increment_scan_strikes() {
|
|
local current
|
|
current=$(get_scan_strikes "$1")
|
|
[[ -z "$current" ]] && current=0
|
|
(( current++ ))
|
|
set_scan_strikes "$1" "$current"
|
|
echo "$current"
|
|
}
|
|
reset_scan_strikes() {
|
|
local current
|
|
current=$(get_scan_strikes "$1")
|
|
[[ -n "$current" && "$current" != "0" ]] && set_scan_strikes "$1" 0
|
|
}
|
|
|
|
# Bails out of the whole run without committing anything. Safe to call at any point before
|
|
# the commit phase: strikes are queued in memory until then, so an abort leaves the strike
|
|
# file exactly as the previous run left it and deletes nothing.
|
|
abort_scan() {
|
|
local why="$1"
|
|
error "Corruption scan ABORTED — $why"
|
|
error "No strikes recorded and nothing remediated this run — the library was not trusted."
|
|
[[ -n "${FRESH_CLEAN_TMP:-}" ]] && rm -f "$FRESH_CLEAN_TMP"
|
|
notify "Corruption scan aborted on $(hostname) ($MY_ID) — $why. Nothing deleted." \
|
|
"Arr Corruption Scan" "warning"
|
|
exit 1
|
|
}
|
|
|
|
# Per-arr API shape differences — everything else in the scan/strike/remediate loop below is
|
|
# identical between Sonarr and Radarr.
|
|
declare -A ARR_FILE_ENDPOINT=( [sonarr]="episodefile" [radarr]="moviefile" )
|
|
declare -A ARR_PARENT_ENDPOINT=( [sonarr]="episode" [radarr]="movie" )
|
|
declare -A ARR_SEARCH_COMMAND=( [sonarr]="EpisodeSearch" [radarr]="MoviesSearch" )
|
|
declare -A ARR_SEARCH_ID_FIELD=( [sonarr]="episodeIds" [radarr]="movieIds" )
|
|
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
for arr in sonarr radarr; do
|
|
url_var="${arr^^}_URL"
|
|
echo "$ICON_GEAR ${arr^} URL: ${!url_var:-not configured}"
|
|
done
|
|
echo "$ICON_GEAR FFprobe container: $FFPROBE_CONTAINER"
|
|
echo "$ICON_GEAR FFprobe binary: $FFPROBE_BIN"
|
|
echo "$ICON_GEAR FFprobe path map: ${#FFPROBE_PATH_MAP[@]} entries"
|
|
echo "$ICON_GEAR State file: $CORRUPTION_SCAN_STATE_FILE"
|
|
echo "$ICON_GEAR Strike limit: $CORRUPTION_SCAN_STRIKE_LIMIT"
|
|
echo "$ICON_GEAR Remediate: $REMEDIATE"
|
|
echo "$ICON_GEAR Corrupt ceiling: ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% of scanned (min ${CORRUPTION_SCAN_GUARD_MIN_SCANNED} scanned)"
|
|
echo "$ICON_GEAR Consecutive trip: $CORRUPTION_SCAN_MAX_CONSECUTIVE"
|
|
echo "$ICON_GEAR Scan limit: ${SCAN_LIMIT:-unlimited} (per arr)"
|
|
echo "$ICON_GEAR Path filter: ${PATH_FILTER:-none}"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
echo ""
|
|
[[ "$REMEDIATE" == true ]] && warn "REMEDIATE MODE — corrupt files will be deleted and re-searched" \
|
|
|| info "Report-only — pass --remediate to act"
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
|
|
check_container_health "$FFPROBE_CONTAINER" 15 "Corruption Scan"
|
|
|
|
# ==============================================================================================
|
|
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
|
|
# Translates a host filesystem path to FFPROBE_CONTAINER's internal path via prefix match
|
|
# against FFPROBE_PATH_MAP. Empty output (return 1) means this file's share isn't covered
|
|
# by the ffprobe container yet — caller must skip, not guess.
|
|
ffprobe_translate_path() {
|
|
local host_path="$1" prefix
|
|
for prefix in "${!FFPROBE_PATH_MAP[@]}"; do
|
|
if [[ "$host_path" == "$prefix"/* ]]; then
|
|
echo "${FFPROBE_PATH_MAP[$prefix]}${host_path#$prefix}"
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Probes one file. Echoes "clean" or "corrupt:<reason>". Never trusts a truncated/garbled
|
|
# stderr as automatically corrupt — only a real non-empty ffprobe stderr counts.
|
|
probe_file() {
|
|
local host_path="$1" container_path output rc
|
|
container_path=$(ffprobe_translate_path "$host_path") || { echo "unmapped"; return; }
|
|
output=$(docker exec "$FFPROBE_CONTAINER" "$FFPROBE_BIN" -v error "$container_path" 2>&1)
|
|
rc=$?
|
|
|
|
# docker exec writes its own failures to the same stream ffprobe uses, so a stopped
|
|
# container or an unreachable daemon is otherwise indistinguishable from a corrupt
|
|
# header. A stopped container exits 1 with a daemon message; a missing binary exits
|
|
# 127 — neither is evidence about the file, so both must be caught.
|
|
if (( rc >= 125 )) \
|
|
|| [[ "$output" == "Error response from daemon:"* \
|
|
|| "$output" == "Cannot connect to the Docker daemon"* \
|
|
|| "$output" == "error during connect:"* ]]; then
|
|
echo "probe_error:${output//$'\n'/ }"
|
|
return
|
|
fi
|
|
|
|
if [[ -z "$output" ]]; then
|
|
echo "clean"
|
|
elif (( rc != 0 )); then
|
|
# ffprobe could not parse the file — EBML header parsing failed, moov atom not found,
|
|
# contradictionary STSC and STCO. This is the only class that may be remediated.
|
|
echo "corrupt:${output//$'\n'/ }"
|
|
else
|
|
# Exit 0 with stderr output: a recoverable muxing complaint, most commonly
|
|
# "Referenced QT chapter track not found", which many recent .mp4 releases emit and
|
|
# which says nothing about playability. Equating any stderr with corruption is what
|
|
# produced 103 "corrupt" files on 2026-08-23 — 28 of 43 newly scanned Radarr items.
|
|
# Reported for visibility, never strike-tracked, never remediated.
|
|
echo "suspect:${output//$'\n'/ }"
|
|
fi
|
|
}
|
|
|
|
# Appends one arr_api() call's output to a batch file, but ONLY on success. arr_api() prints
|
|
# its own error message via error() (a plain `echo`, i.e. stdout, not stderr) on any non-200
|
|
# response — appending its raw output unconditionally means a single failed batch call (one
|
|
# bad seriesId/movieId batch out of hundreds) mixes a plain-text error line into what's
|
|
# otherwise a stream of valid JSON arrays, and `jq -s` then fails to parse the WHOLE file,
|
|
# turning one bad batch into zero usable files for the entire arr. Confirmed live 2026-07-21:
|
|
# Sonarr seriesId=650 returned HTTP 404 (stale/deleted series reference) mid-walk, and that
|
|
# single 404's error text corrupted the full 138MB/1177-series concatenated batch, silently
|
|
# zeroing out the whole Sonarr scan for that run. Capturing output first and gating the
|
|
# append on the actual exit code isolates one bad call to just that call.
|
|
_arr_api_append_on_success() {
|
|
local out
|
|
out=$(arr_api "$1" "$2" "$3" "$4" "$5" 2>/dev/null)
|
|
[[ $? -eq 0 ]] && echo "$out" >> "$6"
|
|
}
|
|
|
|
# Fetches every movie's file(s) via Radarr's moviefile endpoint, batched (movieId=X repeated
|
|
# query param, BATCH_SIZE at a time — a single whole-library request 414s, confirmed live by
|
|
# radarr_cleanup.sh 2026-07-19). Deliberately not just the movie list's embedded .movieFile —
|
|
# that only ever has the primary file, missing Radarr's second-tracked-file-per-movie feature
|
|
# (alternate editions/extras). Echoes the raw moviefile JSON array (id/movieId/path per item).
|
|
fetch_radarr_items() {
|
|
local movies_now
|
|
movies_now=$(arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "movie" "Radarr" 2>/dev/null)
|
|
|
|
local _ids=() _id _qs="" _batch_count=0
|
|
local BATCH_SIZE=200 # 250 confirmed working live 2026-07-19 by radarr_cleanup.sh, margin kept
|
|
mapfile -t _ids < <(echo "$movies_now" | jq -r '.[] | select(.hasFile==true) | .id' 2>/dev/null)
|
|
|
|
local all_tmp; all_tmp=$(mktemp)
|
|
for _id in "${_ids[@]}"; do
|
|
_qs+="movieId=${_id}&"
|
|
(( _batch_count++ ))
|
|
if [[ "$_batch_count" -ge "$BATCH_SIZE" ]]; then
|
|
_arr_api_append_on_success "$RADARR_URL" "$RADARR_API_KEY" "v3" "moviefile?${_qs%&}" "Radarr" "$all_tmp"
|
|
_qs=""
|
|
_batch_count=0
|
|
fi
|
|
done
|
|
if [[ -n "$_qs" ]]; then
|
|
_arr_api_append_on_success "$RADARR_URL" "$RADARR_API_KEY" "v3" "moviefile?${_qs%&}" "Radarr" "$all_tmp"
|
|
fi
|
|
|
|
jq -s 'add // []' "$all_tmp" 2>/dev/null
|
|
rm -f "$all_tmp"
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Load clean-file state (skip cache) ━━━
|
|
# ==============================================================================================
|
|
declare -A CLEAN_STATE
|
|
while IFS=$'\t' read -r _s_path _s_stamp; do
|
|
[[ -n "$_s_path" ]] && CLEAN_STATE["$_s_path"]="$_s_stamp"
|
|
done < "$CORRUPTION_SCAN_STATE_FILE"
|
|
unset _s_path _s_stamp
|
|
info "Loaded ${#CLEAN_STATE[@]} previously-verified-clean entries"
|
|
|
|
# Merges a fresh-clean-stamps temp file into the persistent state file, newest wins per path —
|
|
# reading fresh entries first (before the old base file) means the first occurrence tac/awk
|
|
# keeps is always the newest one for any path re-verified this run. Called once per arr
|
|
# (immediately after that arr's scan, not batched to the very end of the whole script) so a
|
|
# hard-exit partway through the NEXT arr — check_container_health()/check_arr_version() both
|
|
# exit 1 directly on a real failure, not just return — can never wipe out the previous arr's
|
|
# already-computed clean state for this run.
|
|
merge_clean_state() {
|
|
local fresh_tmp="$1" state_tmp
|
|
state_tmp=$(mktemp)
|
|
cat "$fresh_tmp" "$CORRUPTION_SCAN_STATE_FILE" | awk -F'\t' '!seen[$1]++' | sort > "$state_tmp"
|
|
mv "$state_tmp" "$CORRUPTION_SCAN_STATE_FILE"
|
|
}
|
|
|
|
TOTAL_SCANNED=0
|
|
TOTAL_CORRUPT=0
|
|
TOTAL_REMEDIATED=0
|
|
TOTAL_REMEDIATE_FAILED=0
|
|
declare -A ARR_SCANNED ARR_SKIPPED_CACHED ARR_SKIPPED_UNMAPPED ARR_CORRUPT ARR_SUSPECT ARR_PROBE_ERRORS ARR_STRIKE_HELD ARR_REMEDIATED ARR_REMEDIATE_FAILED
|
|
|
|
for arr in sonarr radarr; do
|
|
url_var="${arr^^}_URL"; key_var="${arr^^}_API_KEY"
|
|
# Named arr_url/arr_key, not url/key — build_arr_path_map() below uses a non-local
|
|
# `for key in ...` loop internally (iterating FFPROBE/path-map prefixes) and would
|
|
# silently clobber a plain $key with its last loop value otherwise. Confirmed live
|
|
# 2026-07-21: this exact collision fed a path-map prefix ("/ext-anime-shows") to Sonarr's
|
|
# API calls as the X-Api-Key header instead of the real key, making every Sonarr call
|
|
# this loop made fail with 401 while looking like a connectivity problem.
|
|
arr_url="${!url_var:-}"; arr_key="${!key_var:-}"
|
|
|
|
if [[ -z "$arr_url" || -z "$arr_key" ]]; then
|
|
info "${arr^} not configured on $MY_ID ($LOCAL_SERVER_NAME) — skipping"
|
|
continue
|
|
fi
|
|
|
|
echo ""
|
|
echo " ${arr^} — $arr_url"
|
|
|
|
build_arr_path_map "${arr^^}"
|
|
|
|
check_container_health "${arr^}" 15 "Corruption Scan"
|
|
ver_var="${arr^^}_VERSION_MAJOR"
|
|
check_arr_version "$arr_url" "$arr_key" "v3" "${!ver_var}" "${arr^}" || {
|
|
warn "${arr^} version check failed — skipping this arr"
|
|
continue
|
|
}
|
|
|
|
# ──────────────────────────────────────────────────────────────────────────────────────
|
|
# Fetch tracked files, normalized to {path, file_id, parent_id, title} regardless of arr —
|
|
# everything past this point is arr-agnostic.
|
|
# ──────────────────────────────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Fetching ${arr^} Tracked Files ━━━"
|
|
|
|
if [[ "$arr" == "sonarr" ]]; then
|
|
# Prefer the shared per-episode-file cache written by sonarr_cleanup.sh (has
|
|
# id/episodeId/seriesId/path already) — falls back to a live per-series walk only on
|
|
# a genuine miss.
|
|
RAW_ITEMS=$(arr_get_cached_items "sonarr" 14400)
|
|
if [[ -z "$RAW_ITEMS" || "$RAW_ITEMS" == "null" ]]; then
|
|
info "No fresh cached episode-file data — fetching live (this is the slow path)"
|
|
SERIES_RESPONSE=$(arr_get_tracked_data "sonarr" "$arr_url" "$arr_key" "v3") || {
|
|
error "Failed to fetch series from Sonarr — skipping this arr"
|
|
continue
|
|
}
|
|
SERIES_IDS=$(echo "$SERIES_RESPONSE" | jq -r '.[].id')
|
|
all_tmp=$(mktemp)
|
|
while IFS= read -r sid; do
|
|
[[ -z "$sid" ]] && continue
|
|
_arr_api_append_on_success "$arr_url" "$arr_key" "v3" "episodefile?seriesId=${sid}" "Sonarr" "$all_tmp"
|
|
done <<< "$SERIES_IDS"
|
|
RAW_ITEMS=$(jq -s 'add // []' "$all_tmp" 2>/dev/null)
|
|
rm -f "$all_tmp"
|
|
arr_item_cache_write "sonarr" "$RAW_ITEMS"
|
|
fi
|
|
ITEMS=$(echo "$RAW_ITEMS" | jq -c \
|
|
'[.[] | {path, file_id:.id, parent_id:.episodeId, title:(.sceneName // .relativePath // .path)}]')
|
|
else
|
|
RAW_ITEMS=$(arr_get_cached_items "radarr" 14400)
|
|
if [[ -z "$RAW_ITEMS" || "$RAW_ITEMS" == "null" ]]; then
|
|
info "No fresh cached movie-file data — fetching live (this is the slow path)"
|
|
RAW_ITEMS=$(fetch_radarr_items)
|
|
arr_item_cache_write "radarr" "$RAW_ITEMS"
|
|
fi
|
|
ITEMS=$(echo "$RAW_ITEMS" | jq -c \
|
|
'[.[] | {path, file_id:.id, parent_id:.movieId, title:(.sceneName // .relativePath // .path)}]')
|
|
fi
|
|
|
|
ITEM_COUNT=$(echo "$ITEMS" | jq 'length' 2>/dev/null)
|
|
if [[ -z "$ITEM_COUNT" || "$ITEM_COUNT" -eq 0 ]]; then
|
|
error "0 tracked files for ${arr^} — skipping this arr"
|
|
continue
|
|
fi
|
|
info "$ITEM_COUNT tracked files"
|
|
|
|
# ──────────────────────────────────────────────────────────────────────────────────────
|
|
# Scan
|
|
# ──────────────────────────────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ $ICON_CLEAN Scanning ━━━"
|
|
|
|
SCANNED=0
|
|
SKIPPED_CACHED=0
|
|
SKIPPED_UNMAPPED=0
|
|
CORRUPT_COUNT=0
|
|
STRIKE_HELD=0
|
|
REMEDIATED=0
|
|
REMEDIATE_FAILED=0
|
|
PROBE_ERRORS=0
|
|
SUSPECT_COUNT=0
|
|
CONSECUTIVE_BAD=0
|
|
QUEUE_PATH=()
|
|
QUEUE_STRIKES=()
|
|
QUEUE_ITEM=()
|
|
|
|
FRESH_CLEAN_TMP=$(mktemp)
|
|
|
|
while IFS= read -r item; do
|
|
api_path=$(echo "$item" | jq -r '.path')
|
|
file_id=$(echo "$item" | jq -r '.file_id')
|
|
parent_id=$(echo "$item" | jq -r '.parent_id')
|
|
|
|
host_path=$(translate_path "$api_path")
|
|
[[ -f "$host_path" ]] || continue
|
|
[[ -n "$PATH_FILTER" && "$host_path" != *"$PATH_FILTER"* ]] && continue
|
|
|
|
stamp="$(stat -c '%Y:%s' "$host_path" 2>/dev/null)"
|
|
[[ -z "$stamp" ]] && continue
|
|
|
|
if [[ "${CLEAN_STATE[$host_path]:-}" == "$stamp" ]]; then
|
|
(( SKIPPED_CACHED++ ))
|
|
continue
|
|
fi
|
|
|
|
(( SCANNED++ ))
|
|
if [[ "$SCAN_LIMIT" -gt 0 && "$SCANNED" -gt "$SCAN_LIMIT" ]]; then
|
|
(( SCANNED-- ))
|
|
break
|
|
fi
|
|
|
|
result=$(probe_file "$host_path")
|
|
|
|
if [[ "$result" == "unmapped" ]]; then
|
|
(( SKIPPED_UNMAPPED++ ))
|
|
[[ "$ENABLE_LOGGING" == true ]] && warn " ? $host_path — no FFPROBE_PATH_MAP entry covers this share"
|
|
continue
|
|
fi
|
|
|
|
# A docker-level failure is not evidence about the file. Count it, never queue it.
|
|
if [[ "$result" == probe_error:* ]]; then
|
|
(( PROBE_ERRORS++ ))
|
|
(( CONSECUTIVE_BAD++ ))
|
|
warn " ? $host_path — probe failed, NOT counted as corrupt: ${result#probe_error:}"
|
|
if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then
|
|
abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly"
|
|
fi
|
|
continue
|
|
fi
|
|
|
|
if [[ "$result" == "clean" ]]; then
|
|
CONSECUTIVE_BAD=0
|
|
reset_scan_strikes "$host_path"
|
|
echo -e "${host_path}\t${stamp}" >> "$FRESH_CLEAN_TMP"
|
|
[[ "$ENABLE_LOGGING" == true ]] && echo " $ICON_SUCCESS $host_path"
|
|
continue
|
|
fi
|
|
|
|
# A successful probe that merely warned. Proves the container is alive, so it clears
|
|
# the consecutive-failure tripwire, but it never becomes a strike.
|
|
if [[ "$result" == suspect:* ]]; then
|
|
CONSECUTIVE_BAD=0
|
|
(( SUSPECT_COUNT++ ))
|
|
[[ "$ENABLE_LOGGING" == true ]] && warn " ~ $host_path — ffprobe warning (exit 0), NOT corrupt: ${result#suspect:}"
|
|
continue
|
|
fi
|
|
|
|
# corrupt:<reason> — queued, NOT committed. Nothing reaches the strike file and nothing
|
|
# is deleted until this arr has been fully probed and the guards below have passed. A
|
|
# container that dies mid-scan makes every remaining file read as corrupt, and a delete
|
|
# cannot be undone — so the destructive half has to wait until the corrupt rate for the
|
|
# whole run is known. 2026-08-23: one Jellyfin restart produced 103 false positives.
|
|
reason="${result#corrupt:}"
|
|
(( CORRUPT_COUNT++ ))
|
|
(( CONSECUTIVE_BAD++ ))
|
|
|
|
prev_strikes=$(get_scan_strikes "$host_path")
|
|
prev_strikes="${prev_strikes//[^0-9]/}"
|
|
strikes=$(( ${prev_strikes:-0} + 1 ))
|
|
|
|
QUEUE_PATH+=("$host_path")
|
|
QUEUE_STRIKES+=("$strikes")
|
|
QUEUE_ITEM+=("$item")
|
|
|
|
echo " $ICON_ERROR CORRUPT: $host_path (strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT)"
|
|
[[ "$ENABLE_LOGGING" == true ]] && echo " $reason"
|
|
|
|
if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then
|
|
abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly"
|
|
fi
|
|
done < <(echo "$ITEMS" | jq -c '.[]')
|
|
|
|
# ━━━ False-positive guards — run before anything is committed ━━━
|
|
if (( CORRUPT_COUNT > 0 )); then
|
|
# The pre-flight check only proves the container was up when the scan started.
|
|
# Re-check now: a mid-scan death is exactly what this guard exists to catch.
|
|
check_container_health "$FFPROBE_CONTAINER" "${DOCKER_TIMEOUT:-30}" "Arr Corruption Scan"
|
|
|
|
if (( SCANNED >= CORRUPTION_SCAN_GUARD_MIN_SCANNED )); then
|
|
corrupt_pct=$(( CORRUPT_COUNT * 100 / SCANNED ))
|
|
if (( corrupt_pct >= CORRUPTION_SCAN_MAX_CORRUPT_PCT )); then
|
|
abort_scan "$CORRUPT_COUNT of $SCANNED probed files (${corrupt_pct}%) read as corrupt — at or above the ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% ceiling"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# ━━━ Guards passed — commit strikes, then remediate whatever reached the limit ━━━
|
|
for _q in "${!QUEUE_PATH[@]}"; do
|
|
host_path="${QUEUE_PATH[$_q]}"
|
|
strikes="${QUEUE_STRIKES[$_q]}"
|
|
item="${QUEUE_ITEM[$_q]}"
|
|
|
|
set_scan_strikes "$host_path" "$strikes"
|
|
|
|
[[ "$REMEDIATE" != true ]] && continue
|
|
|
|
if (( strikes < CORRUPTION_SCAN_STRIKE_LIMIT )); then
|
|
warn " $host_path — strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT, not yet remediating (needs repeat confirmation)"
|
|
(( STRIKE_HELD++ ))
|
|
continue
|
|
fi
|
|
reset_scan_strikes "$host_path"
|
|
|
|
file_id=$(echo "$item" | jq -r '.file_id')
|
|
parent_id=$(echo "$item" | jq -r '.parent_id')
|
|
title=$(echo "$item" | jq -r '.title')
|
|
|
|
http_code=$(curl -sf -o /dev/null -w "%{http_code}" -X DELETE \
|
|
--max-time 15 -H "X-Api-Key: $arr_key" \
|
|
"${arr_url}/api/v3/${ARR_FILE_ENDPOINT[$arr]}/${file_id}" 2>/dev/null)
|
|
|
|
if [[ "$http_code" != "200" ]]; then
|
|
error " ✗ $title — delete failed (HTTP $http_code)"
|
|
(( REMEDIATE_FAILED++ ))
|
|
continue
|
|
fi
|
|
|
|
sleep 2
|
|
verify_hasfile=$(arr_api "$arr_url" "$arr_key" "v3" "${ARR_PARENT_ENDPOINT[$arr]}/${parent_id}" "${arr^}" 2>/dev/null \
|
|
| jq -r '.hasFile // "unknown"')
|
|
|
|
if [[ "$verify_hasfile" != "false" ]]; then
|
|
error " ✗ $title — deleted but hasFile still '$verify_hasfile' — not searching, needs review"
|
|
(( REMEDIATE_FAILED++ ))
|
|
continue
|
|
fi
|
|
|
|
search_code=$(curl -sf -o /dev/null -w "%{http_code}" -X POST \
|
|
--max-time 30 -H "X-Api-Key: $arr_key" -H "Content-Type: application/json" \
|
|
-d "{\"name\":\"${ARR_SEARCH_COMMAND[$arr]}\",\"${ARR_SEARCH_ID_FIELD[$arr]}\":[${parent_id}]}" \
|
|
"${arr_url}/api/v3/command" 2>/dev/null)
|
|
|
|
if [[ "$search_code" == "200" || "$search_code" == "201" ]]; then
|
|
echo " $ICON_SUCCESS $title — deleted, verified, search triggered"
|
|
(( REMEDIATED++ ))
|
|
else
|
|
warn " $title — deleted and verified, but search trigger returned HTTP $search_code"
|
|
(( REMEDIATE_FAILED++ ))
|
|
fi
|
|
done
|
|
|
|
merge_clean_state "$FRESH_CLEAN_TMP"
|
|
rm -f "$FRESH_CLEAN_TMP"
|
|
|
|
ARR_SCANNED[$arr]=$SCANNED
|
|
ARR_SKIPPED_CACHED[$arr]=$SKIPPED_CACHED
|
|
ARR_SKIPPED_UNMAPPED[$arr]=$SKIPPED_UNMAPPED
|
|
ARR_CORRUPT[$arr]=$CORRUPT_COUNT
|
|
ARR_SUSPECT[$arr]=$SUSPECT_COUNT
|
|
ARR_PROBE_ERRORS[$arr]=$PROBE_ERRORS
|
|
ARR_STRIKE_HELD[$arr]=$STRIKE_HELD
|
|
ARR_REMEDIATED[$arr]=$REMEDIATED
|
|
ARR_REMEDIATE_FAILED[$arr]=$REMEDIATE_FAILED
|
|
|
|
(( TOTAL_SCANNED += SCANNED ))
|
|
(( TOTAL_CORRUPT += CORRUPT_COUNT ))
|
|
(( TOTAL_REMEDIATED += REMEDIATED ))
|
|
(( TOTAL_REMEDIATE_FAILED += REMEDIATE_FAILED ))
|
|
done
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY CORRUPTION SCAN SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
for arr in sonarr radarr; do
|
|
[[ -z "${ARR_SCANNED[$arr]:-}" ]] && continue
|
|
echo ""
|
|
echo " ${arr^}:"
|
|
echo " $ICON_SYNC Newly scanned: ${ARR_SCANNED[$arr]}"
|
|
echo " $ICON_SUCCESS Skipped (cached): ${ARR_SKIPPED_CACHED[$arr]}"
|
|
echo " $ICON_WARN Skipped (unmapped): ${ARR_SKIPPED_UNMAPPED[$arr]}"
|
|
echo " $ICON_ERROR Corrupt found: ${ARR_CORRUPT[$arr]}"
|
|
echo " $ICON_WARN Warnings (exit 0): ${ARR_SUSPECT[$arr]} (reported, never remediated)"
|
|
echo " $ICON_WARN Probe errors: ${ARR_PROBE_ERRORS[$arr]} (not counted as corrupt)"
|
|
if [[ "$REMEDIATE" == true ]]; then
|
|
echo " $ICON_WARN Held (strikes): ${ARR_STRIKE_HELD[$arr]}"
|
|
echo " $ICON_SUCCESS Remediated: ${ARR_REMEDIATED[$arr]}"
|
|
echo " $ICON_ERROR Remediation failed: ${ARR_REMEDIATE_FAILED[$arr]}"
|
|
fi
|
|
done
|
|
echo ""
|
|
echo " Total:"
|
|
echo " $ICON_SYNC Newly scanned: $TOTAL_SCANNED"
|
|
echo " $ICON_ERROR Corrupt found: $TOTAL_CORRUPT"
|
|
if [[ "$REMEDIATE" == true ]]; then
|
|
echo " $ICON_SUCCESS Remediated: $TOTAL_REMEDIATED"
|
|
echo " $ICON_ERROR Remediation failed: $TOTAL_REMEDIATE_FAILED"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
exit 0
|