Gmer4Lfe
77845829ad
Full-library scans are slow to reach any one show by chance — this lets a test target a specific known-corrupt file/show without waiting on scan order.
398 lines
18 KiB
Bash
Executable File
398 lines
18 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================ Arr Corruption Scan ==============================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Scans Sonarr's tracked video files for corrupt headers (ffprobe-based, same detection
|
|
# method as the third-party Healarr tool) and, in --remediate mode, deletes the bad file
|
|
# from Sonarr and explicitly triggers an EpisodeSearch to replace it.
|
|
#
|
|
# Built after Healarr crashed mid-scan on a genuine Go concurrency bug (unsynchronized
|
|
# map access when multiple corruption events land at once — confirmed via its own crash
|
|
# log, not fixable from our side). The core idea (scan → delete → re-search) isn't hard to
|
|
# replicate; the fix here is architectural: this script processes one file at a time,
|
|
# strictly sequential, so the race condition that killed Healarr can't happen — there's
|
|
# nothing running concurrently to race.
|
|
#
|
|
# ==============================================================================================
|
|
# WHY A SEPARATE CONTAINER FOR FFPROBE
|
|
# ==============================================================================================
|
|
#
|
|
# Neither Sonarr nor Radarr bundle ffprobe. ffprobe runs via `docker exec` into a
|
|
# different container that does — confirmed live 2026-07-18:
|
|
# Jellyfin — working ffprobe, but only mounts Tv_Shows + Movies
|
|
# Emby — mounts everything (kids/anime shares too), but its bundled
|
|
# ffprobe binary is broken (2017-dated, fails to exec — likely a
|
|
# missing dynamic linker dependency, not something to fix here)
|
|
# Jellyfin is what's configured (HOST*_FFPROBE_CONTAINER) — matches the current TV-share
|
|
# testing scope. Kids/anime coverage needs a working ffprobe source with those mounts
|
|
# before this script can cover those shares; not solved yet.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# One file at a time, in this order per file:
|
|
# 1. Skip if unchanged (mtime+size) since the last time it verified clean — state file
|
|
# avoids re-probing the entire library every run, which would take far too long at
|
|
# this library size (90k+ tracked files).
|
|
# 2. ffprobe via `docker exec` into FFPROBE_CONTAINER. Empty stderr + exit 0 = clean.
|
|
# Anything else = corrupt (same signature as Healarr: "Invalid data found when
|
|
# processing input", EBML header errors, etc.)
|
|
# 3. Report-only by default. --remediate additionally:
|
|
# a. DELETE the specific episodefile record via Sonarr's API
|
|
# b. Verify hasFile flipped false (never trust the DELETE response alone)
|
|
# c. Explicitly trigger EpisodeSearch for that episode — this is deliberate, not
|
|
# left to Sonarr's own background missing-search cycle, because that cycle
|
|
# skips unmonitored episodes entirely. An explicit EpisodeSearch call does not
|
|
# have that restriction (confirmed live: two unmonitored episodes Healarr
|
|
# healed both still got successfully re-grabbed via this exact same kind of
|
|
# search call, logged in Sonarr's history as "UserInvokedSearch").
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# host*.conf
|
|
# SONARR_URL / SONARR_API_KEY / SONARR_PATH_MAP — existing, aliased by detect_hosts()
|
|
# HOST*_FFPROBE_CONTAINER — container name with a working ffprobe binary
|
|
# HOST*_FFPROBE_BIN — full path to that binary inside the container
|
|
# HOST*_FFPROBE_PATH_MAP — host path prefix → that container's internal path prefix
|
|
# (separate from the arr's own path map — the ffprobe
|
|
# container almost certainly mounts shares differently)
|
|
#
|
|
# master.conf
|
|
# CORRUPTION_SCAN_STATE_FILE — path to the clean-file skip-cache (default in DATA_DIR)
|
|
# SONARR_VERSION_MAJOR — reused from sonarr_cleanup.sh for the API version check
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# arr_corruption_scan.sh — report-only, scans everything not yet
|
|
# verified clean
|
|
# arr_corruption_scan.sh --remediate — delete + re-search on every corrupt file found
|
|
# arr_corruption_scan.sh --limit=50 — cap this run to 50 newly-probed files (state
|
|
# file makes repeat runs cheap regardless, but
|
|
# useful for a bounded first test)
|
|
# arr_corruption_scan.sh --log — verbose (prints every clean file too)
|
|
# arr_corruption_scan.sh --status — show config and exit
|
|
# arr_corruption_scan.sh --filter=Becker — only consider paths containing this substring
|
|
# (testing/targeting a specific show; state
|
|
# file and everything else behaves normally)
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
# --remediate / --limit are script-local, not recognized by parse_args — check raw args
|
|
# before they get filtered.
|
|
REMEDIATE=false
|
|
SCAN_LIMIT=0
|
|
PATH_FILTER=""
|
|
for _arg in "$@"; do
|
|
case "$_arg" in
|
|
--remediate) REMEDIATE=true ;;
|
|
--limit=*) SCAN_LIMIT="${_arg#*=}" ;;
|
|
--filter=*) PATH_FILTER="${_arg#*=}" ;;
|
|
esac
|
|
done
|
|
unset _arg
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v curl >/dev/null 2>&1; then
|
|
error "curl not found — required for Sonarr API calls"
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v jq >/dev/null 2>&1; then
|
|
error "jq not found — required for JSON parsing"
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker command not found"
|
|
exit 1
|
|
fi
|
|
|
|
acquire_lock "wait"
|
|
TMP_DIR="/tmp/arr_corruption_scan_$$"
|
|
mkdir -p "$TMP_DIR"
|
|
trap "_release_all_locks; rm -rf $TMP_DIR" EXIT
|
|
|
|
detect_hosts
|
|
|
|
if [[ -z "${SONARR_URL:-}" ]] || [[ -z "${SONARR_API_KEY:-}" ]]; then
|
|
info "Sonarr not configured on $MY_ID ($LOCAL_SERVER_NAME) — skipping"
|
|
exit 0
|
|
fi
|
|
|
|
build_arr_path_map "SONARR"
|
|
|
|
# FFPROBE_* aren't part of the shared arr alias set in detect_hosts() — resolve them here,
|
|
# same eval-based pattern build_arr_path_map() uses for the associative array.
|
|
FFPROBE_CONTAINER_VAR="${MY_ID}_FFPROBE_CONTAINER"
|
|
FFPROBE_CONTAINER="${!FFPROBE_CONTAINER_VAR:-}"
|
|
FFPROBE_BIN_VAR="${MY_ID}_FFPROBE_BIN"
|
|
FFPROBE_BIN="${!FFPROBE_BIN_VAR:-}"
|
|
|
|
declare -A FFPROBE_PATH_MAP=()
|
|
_fp_map_var="${MY_ID}_FFPROBE_PATH_MAP"
|
|
eval "for key in \"\${!${_fp_map_var}[@]}\"; do
|
|
FFPROBE_PATH_MAP[\"\$key\"]=\"\${${_fp_map_var}[\$key]}\"
|
|
done"
|
|
unset _fp_map_var
|
|
|
|
if [[ -z "$FFPROBE_CONTAINER" || -z "$FFPROBE_BIN" ]]; then
|
|
error "FFPROBE_CONTAINER/FFPROBE_BIN not configured on $MY_ID — skipping"
|
|
exit 0
|
|
fi
|
|
|
|
CORRUPTION_SCAN_STATE_FILE="${CORRUPTION_SCAN_STATE_FILE:-$DATA_DIR/corruption_scan_state.tsv}"
|
|
mkdir -p "$(dirname "$CORRUPTION_SCAN_STATE_FILE")"
|
|
touch "$CORRUPTION_SCAN_STATE_FILE"
|
|
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_GEAR Sonarr URL: $SONARR_URL"
|
|
echo "$ICON_GEAR FFprobe container: $FFPROBE_CONTAINER"
|
|
echo "$ICON_GEAR FFprobe binary: $FFPROBE_BIN"
|
|
echo "$ICON_GEAR FFprobe path map: ${#FFPROBE_PATH_MAP[@]} entries"
|
|
echo "$ICON_GEAR State file: $CORRUPTION_SCAN_STATE_FILE"
|
|
echo "$ICON_GEAR Remediate: $REMEDIATE"
|
|
echo "$ICON_GEAR Scan limit: ${SCAN_LIMIT:-unlimited}"
|
|
echo "$ICON_GEAR Path filter: ${PATH_FILTER:-none}"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
echo ""
|
|
echo " $MY_ID ($LOCAL_SERVER_NAME) — $SONARR_URL"
|
|
[[ "$REMEDIATE" == true ]] && warn "REMEDIATE MODE — corrupt files will be deleted and re-searched" \
|
|
|| info "Report-only — pass --remediate to act"
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
|
|
check_container_health "$FFPROBE_CONTAINER" 15 "Corruption Scan"
|
|
check_container_health "Sonarr" 15 "Corruption Scan"
|
|
|
|
check_arr_version "$SONARR_URL" "$SONARR_API_KEY" "v3" "$SONARR_VERSION_MAJOR" "Sonarr" || exit 1
|
|
|
|
# ==============================================================================================
|
|
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
|
|
# Translates a host filesystem path to FFPROBE_CONTAINER's internal path via prefix match
|
|
# against FFPROBE_PATH_MAP. Empty output (return 1) means this file's share isn't covered
|
|
# by the ffprobe container yet — caller must skip, not guess.
|
|
ffprobe_translate_path() {
|
|
local host_path="$1" prefix
|
|
for prefix in "${!FFPROBE_PATH_MAP[@]}"; do
|
|
if [[ "$host_path" == "$prefix"/* ]]; then
|
|
echo "${FFPROBE_PATH_MAP[$prefix]}${host_path#$prefix}"
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Probes one file. Echoes "clean" or "corrupt:<reason>". Never trusts a truncated/garbled
|
|
# stderr as automatically corrupt — only a real non-empty ffprobe stderr counts.
|
|
probe_file() {
|
|
local host_path="$1" container_path output
|
|
container_path=$(ffprobe_translate_path "$host_path") || { echo "unmapped"; return; }
|
|
output=$(docker exec "$FFPROBE_CONTAINER" "$FFPROBE_BIN" -v error "$container_path" 2>&1)
|
|
if [[ -z "$output" ]]; then
|
|
echo "clean"
|
|
else
|
|
echo "corrupt:${output//$'\n'/ }"
|
|
fi
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Load clean-file state (skip cache) ━━━
|
|
# ==============================================================================================
|
|
declare -A CLEAN_STATE
|
|
while IFS=$'\t' read -r _s_path _s_stamp; do
|
|
[[ -n "$_s_path" ]] && CLEAN_STATE["$_s_path"]="$_s_stamp"
|
|
done < "$CORRUPTION_SCAN_STATE_FILE"
|
|
unset _s_path _s_stamp
|
|
info "Loaded ${#CLEAN_STATE[@]} previously-verified-clean entries"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Fetch Sonarr's tracked episode files ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Fetching Sonarr Tracked Files ━━━"
|
|
|
|
# Prefer the shared per-episode-file cache written by sonarr_cleanup.sh (has id/episodeId/
|
|
# seriesId/path already) — falls back to a live per-series walk only on a genuine miss.
|
|
EPISODE_FILES=$(arr_get_cached_items "sonarr" 14400)
|
|
if [[ -z "$EPISODE_FILES" || "$EPISODE_FILES" == "null" ]]; then
|
|
info "No fresh cached episode-file data — fetching live (this is the slow path)"
|
|
SERIES_RESPONSE=$(arr_get_tracked_data "sonarr" "$SONARR_URL" "$SONARR_API_KEY" "v3") || {
|
|
error "Failed to fetch series from Sonarr"
|
|
exit 1
|
|
}
|
|
SERIES_IDS=$(echo "$SERIES_RESPONSE" | jq -r '.[].id')
|
|
all_tmp=$(mktemp)
|
|
while IFS= read -r sid; do
|
|
[[ -z "$sid" ]] && continue
|
|
arr_api "$SONARR_URL" "$SONARR_API_KEY" "v3" "episodefile?seriesId=${sid}" "Sonarr" 2>/dev/null \
|
|
>> "$all_tmp"
|
|
done <<< "$SERIES_IDS"
|
|
EPISODE_FILES=$(jq -s 'add // []' "$all_tmp" 2>/dev/null)
|
|
rm -f "$all_tmp"
|
|
arr_item_cache_write "sonarr" "$EPISODE_FILES"
|
|
fi
|
|
|
|
FILE_COUNT=$(echo "$EPISODE_FILES" | jq 'length' 2>/dev/null)
|
|
if [[ -z "$FILE_COUNT" || "$FILE_COUNT" -eq 0 ]]; then
|
|
error "0 tracked episode files — aborting"
|
|
exit 1
|
|
fi
|
|
info "$FILE_COUNT tracked episode files"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Scan ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_CLEAN Scanning ━━━"
|
|
|
|
SCANNED=0
|
|
SKIPPED_CACHED=0
|
|
SKIPPED_UNMAPPED=0
|
|
CORRUPT_COUNT=0
|
|
REMEDIATED=0
|
|
REMEDIATE_FAILED=0
|
|
|
|
# Fresh clean-file stamps are appended here as they're found (O(1) per file) rather than
|
|
# rewritten into the growing state file on every hit — at this library's scale (90k+ files)
|
|
# a per-file rewrite-the-whole-file approach would be O(n²) and far too slow.
|
|
FRESH_CLEAN_TMP=$(mktemp)
|
|
|
|
while IFS= read -r item; do
|
|
api_path=$(echo "$item" | jq -r '.path')
|
|
episode_file_id=$(echo "$item" | jq -r '.id')
|
|
series_id=$(echo "$item" | jq -r '.seriesId')
|
|
|
|
host_path=$(translate_path "$api_path")
|
|
[[ -f "$host_path" ]] || continue
|
|
[[ -n "$PATH_FILTER" && "$host_path" != *"$PATH_FILTER"* ]] && continue
|
|
|
|
stamp="$(stat -c '%Y:%s' "$host_path" 2>/dev/null)"
|
|
[[ -z "$stamp" ]] && continue
|
|
|
|
if [[ "${CLEAN_STATE[$host_path]:-}" == "$stamp" ]]; then
|
|
(( SKIPPED_CACHED++ ))
|
|
continue
|
|
fi
|
|
|
|
(( SCANNED++ ))
|
|
if [[ "$SCAN_LIMIT" -gt 0 && "$SCANNED" -gt "$SCAN_LIMIT" ]]; then
|
|
(( SCANNED-- ))
|
|
break
|
|
fi
|
|
|
|
result=$(probe_file "$host_path")
|
|
|
|
if [[ "$result" == "unmapped" ]]; then
|
|
(( SKIPPED_UNMAPPED++ ))
|
|
[[ "$ENABLE_LOGGING" == true ]] && warn " ? $host_path — no FFPROBE_PATH_MAP entry covers this share"
|
|
continue
|
|
fi
|
|
|
|
if [[ "$result" == "clean" ]]; then
|
|
echo -e "${host_path}\t${stamp}" >> "$FRESH_CLEAN_TMP"
|
|
[[ "$ENABLE_LOGGING" == true ]] && echo " $ICON_SUCCESS $host_path"
|
|
continue
|
|
fi
|
|
|
|
# corrupt:<reason>
|
|
reason="${result#corrupt:}"
|
|
(( CORRUPT_COUNT++ ))
|
|
echo " $ICON_ERROR CORRUPT: $host_path"
|
|
[[ "$ENABLE_LOGGING" == true ]] && echo " $reason"
|
|
|
|
if [[ "$REMEDIATE" != true ]]; then
|
|
continue
|
|
fi
|
|
|
|
episode_id=$(echo "$item" | jq -r '.episodeId // empty')
|
|
title=$(echo "$item" | jq -r '.sceneName // .relativePath // .path')
|
|
|
|
http_code=$(curl -sf -o /dev/null -w "%{http_code}" -X DELETE \
|
|
--max-time 15 -H "X-Api-Key: $SONARR_API_KEY" \
|
|
"${SONARR_URL}/api/v3/episodefile/${episode_file_id}" 2>/dev/null)
|
|
|
|
if [[ "$http_code" != "200" ]]; then
|
|
error " ✗ $title — delete failed (HTTP $http_code)"
|
|
(( REMEDIATE_FAILED++ ))
|
|
continue
|
|
fi
|
|
|
|
sleep 2
|
|
verify_hasfile=$(arr_api "$SONARR_URL" "$SONARR_API_KEY" "v3" "episode/${episode_id}" "Sonarr" 2>/dev/null \
|
|
| jq -r '.hasFile // "unknown"')
|
|
|
|
if [[ "$verify_hasfile" != "false" ]]; then
|
|
error " ✗ $title — deleted but hasFile still '$verify_hasfile' — not searching, needs review"
|
|
(( REMEDIATE_FAILED++ ))
|
|
continue
|
|
fi
|
|
|
|
search_code=$(curl -sf -o /dev/null -w "%{http_code}" -X POST \
|
|
--max-time 30 -H "X-Api-Key: $SONARR_API_KEY" -H "Content-Type: application/json" \
|
|
-d "{\"name\":\"EpisodeSearch\",\"episodeIds\":[${episode_id}]}" \
|
|
"${SONARR_URL}/api/v3/command" 2>/dev/null)
|
|
|
|
if [[ "$search_code" == "200" || "$search_code" == "201" ]]; then
|
|
echo " $ICON_SUCCESS $title — deleted, verified, search triggered"
|
|
(( REMEDIATED++ ))
|
|
else
|
|
warn " $title — deleted and verified, but search trigger returned HTTP $search_code"
|
|
(( REMEDIATE_FAILED++ ))
|
|
fi
|
|
done < <(echo "$EPISODE_FILES" | jq -c '.[]')
|
|
|
|
# Merge fresh clean-file stamps with the existing state, newest wins per path — reading
|
|
# fresh entries first (before the old base file) means the first occurrence tac/awk keeps
|
|
# is always the newest one for any path re-verified this run.
|
|
STATE_TMP=$(mktemp)
|
|
cat "$FRESH_CLEAN_TMP" "$CORRUPTION_SCAN_STATE_FILE" | awk -F'\t' '!seen[$1]++' | sort > "$STATE_TMP"
|
|
mv "$STATE_TMP" "$CORRUPTION_SCAN_STATE_FILE"
|
|
rm -f "$FRESH_CLEAN_TMP"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY CORRUPTION SCAN SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_SYNC Newly scanned: $SCANNED"
|
|
echo "$ICON_SUCCESS Skipped (cached): $SKIPPED_CACHED"
|
|
echo "$ICON_WARN Skipped (unmapped): $SKIPPED_UNMAPPED"
|
|
echo "$ICON_ERROR Corrupt found: $CORRUPT_COUNT"
|
|
if [[ "$REMEDIATE" == true ]]; then
|
|
echo "$ICON_SUCCESS Remediated: $REMEDIATED"
|
|
echo "$ICON_ERROR Remediation failed: $REMEDIATE_FAILED"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
exit 0
|