#!/bin/bash # ============================================================================================== # ========================= Arrs Failed / Stalled Recovery ===================================== # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Detect and recover failed imports and stalled downloads across Sonarr, Radarr, # and Lidarr. Blocklists the bad release and triggers a re-search — hands-free # overnight recovery. # # ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # # Five problem types detected from the arr queue API: # importFailed — downloaded but arr couldn't import the file # importPending — downloaded, stuck waiting to import (will not self-resolve) # importBlocked — downloaded, but arr matched the release to the wrong media # by grab-history ID instead of by title and refuses to import # (permanent block, never self-resolves) # error status — serious failure not covered by the above two states # stalled — download stuck with no connections or no progress # # Never touches items with state "downloading" or "imported" — safe to run anytime. # Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first. # # importBlocked items get one extra check first (try_smart_import, Sonarr/Radarr only): # most are junk/duplicates and fall straight through to the normal 3-step response below, # but some are a release arr already correctly parsed — episode/movie identified, quality # and language known — that's just tripping the title-vs-grab-history safety net. If the # target has no file yet (missing) or the candidate is a same-language resolution upgrade # over what's already there, it's imported directly instead of being discarded. See # ARR_SMART_IMPORT_ENABLED in CONFIGURATION below. # # Per problem item that isn't smart-imported (3-step response): # 1. Blocklist the release — prevents re-grabbing the same bad release # 2. Remove from queue — cleans up the failed item # 3. Trigger new search — finds a different release automatically # # ============================================================================================== # DESIGN PRINCIPLES # ============================================================================================== # # Hands-Free Recovery # The script completes the full recovery cycle autonomously — blocklist, remove, # re-search. No operator decision required. A failed import at midnight resolves # itself before morning without any intervention. # # Smart Import Is Conservative By Design # try_smart_import only acts when every file in the download is unambiguous: no # rejections from arr's own analysis, and (no existing file) or (matching language # plus a strictly higher resolution). Any ambiguity — mixed multi-episode files, # unknown language, equal-or-lower quality, wrong language — falls straight through # to blocklist+research, exactly today's behavior. It only ever adds a chance to # keep something worth keeping; it never makes the no-smart-import case worse. # # Age Gate Before Action # Items newer than ARR_IMPORT_RECOVERY_AGE are skipped. Arrs have their own # retry logic — acting immediately would race against it. The age gate gives # the arr time to self-resolve before this script escalates. # # Blocklist First # The bad release is blocklisted before removal and re-search. Without this, # the re-search can re-grab the same release that just failed. # # Circuit Breaker Per Media Item # Some items can never resolve via blind retry — e.g. an album missing 1-2 # tracks where every available release is a different edition that doesn't # match. Without a limit, the same media ID gets blocklisted + re-searched # forever, every run, burning bandwidth and indexer queries for nothing. # After ARR_RECOVERY_MAX_ATTEMPTS consecutive failures for the same # (arr_type, media_id), the item is still blocklisted/cleaned from the queue # but search is no longer auto-triggered — it's flagged chronic and left for # manual review instead. # # "Consecutive" is enforced, not just counted — a media_id's failure count # is pruned at the end of every process_arr() pass if it no longer appears # in that run's problem-item set (2026-07-19 fix: counts were never reset on # success, so an item that failed a few times months apart and then imported # fine could still get stuck permanently chronic from stale history). # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # acquire_lock — prevents concurrent runs overlapping # jq validation — exits if jq not installed (required for JSON parsing) # API pre-flight — checks each arr is reachable before querying queue # Version check — check_arr_version() verifies running arr matches master.conf major # version; exits rather than silently misoperating after upgrade # Age threshold — skips items newer than ARR_IMPORT_RECOVERY_AGE (default 6hr) # Silent by default — only problems produce output, clean arrs stay silent # # API version mapping (endpoint paths differ from major version labels): # Sonarr v4 → /api/v3/ (v3 endpoint retained in v4) # Radarr v6 → /api/v3/ (v3 endpoint retained in v6) # Lidarr v3 → /api/v1/ (different from Sonarr/Radarr) # # ============================================================================================== # STATE FILES # ============================================================================================== # # ARR_RECOVERY_STATS — stats file written after each run (read by coffee report) # ARR_RECOVERY_FAILURE_COUNTS — per (arr_type, media_id) consecutive-failure counts, # persists across runs so the circuit breaker survives restarts. # # ============================================================================================== # CONFIGURATION # ============================================================================================== # # host*.conf # # HOST*_SONARR_URL / HOST*_SONARR_API_KEY / HOST*_SONARR_RECOVERY # HOST*_RADARR_URL / HOST*_RADARR_API_KEY / HOST*_RADARR_RECOVERY # HOST1_LIDARR_URL / HOST1_LIDARR_API_KEY / HOST1_LIDARR_RECOVERY # All aliased by detect_hosts() — script uses unprefixed names # # master.conf # # ARR_IMPORT_RECOVERY_AGE — hours before item is eligible for recovery (default: 6) # ARR_RECOVERY_MAX_ATTEMPTS — consecutive failures before an item is flagged chronic # and auto re-search stops (default: 3) # ARR_SMART_IMPORT_ENABLED — try_smart_import gate for importBlocked items, Sonarr/ # Radarr only — Lidarr's manual-import matching doesn't # reliably resolve album/track context (default: true) # ARR_SMART_IMPORT_PREFERRED_LANGUAGE — only import as a match/upgrade if the # candidate is this language; existing files in a # different language are always treated as upgradeable # (default: English) # SONARR_VERSION_MAJOR — expected Sonarr major version (e.g. 4) # RADARR_VERSION_MAJOR — expected Radarr major version (e.g. 6) # LIDARR_VERSION_MAJOR — expected Lidarr major version (e.g. 3) # ARR_RECOVERY_STATS — stats file path (read by coffee report) # ARR_RECOVERY_FAILURE_COUNTS — failure-count state file path # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # arrs_failed_stalled_recovery.sh — normal run # arrs_failed_stalled_recovery.sh --dry-run — show what would be actioned, no changes # arrs_failed_stalled_recovery.sh --log — verbose output # arrs_failed_stalled_recovery.sh --status — show config and exit # # Recommended schedule: 0 5 * * * (5am daily) # Or every 6hr: 0 */6 * * * (matches ARR_IMPORT_RECOVERY_AGE default) # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi acquire_lock # detect_hosts() sets MY_ID and aliases SONARR_*, RADARR_*, LIDARR_* vars detect_hosts # jq is required — not optional — for JSON parsing if ! command -v jq >/dev/null 2>&1; then error "jq is not installed — required for arr API JSON parsing" error "Install: apt-get install jq or brew install jq" notify "arrs_failed_stalled_recovery failed on $(hostname) — jq not installed" \ "Arr Recovery" "warning" exit 1 fi log "jq found" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no items will be blocklisted or searched" # Age threshold in seconds AGE_THRESHOLD_SECONDS=$(( ARR_IMPORT_RECOVERY_AGE * 3600 )) ARR_RECOVERY_MAX_ATTEMPTS="${ARR_RECOVERY_MAX_ATTEMPTS:-3}" ARR_RECOVERY_FAILURE_COUNTS="${ARR_RECOVERY_FAILURE_COUNTS:-$DATA_DIR/arr_recovery_failure_counts.db}" log "$ICON_GEAR Config: age-threshold=${ARR_IMPORT_RECOVERY_AGE}hr max-attempts=${ARR_RECOVERY_MAX_ATTEMPTS} sonarr-v${SONARR_VERSION_MAJOR} radarr-v${RADARR_VERSION_MAJOR} lidarr-v${LIDARR_VERSION_MAJOR:-?}" # Load persisted per-item failure counts — key is "arr_type:media_id" declare -A FAILURE_COUNTS if [[ -f "$ARR_RECOVERY_FAILURE_COUNTS" ]]; then while IFS='|' read -r _key _count; do [[ -z "$_key" ]] && continue FAILURE_COUNTS["$_key"]="$_count" done < "$ARR_RECOVERY_FAILURE_COUNTS" fi TOTAL_ACTIONED=0 TOTAL_SKIPPED=0 TOTAL_CHRONIC=0 TOTAL_SMART_IMPORTED=0 ARR_SUMMARIES=() # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_SYNC Sonarr: ${SONARR_URL:-not configured} (recovery: ${SONARR_RECOVERY:-true})" echo "$ICON_SYNC Radarr: ${RADARR_URL:-not configured} (recovery: ${RADARR_RECOVERY:-true})" echo "$ICON_SYNC Lidarr: ${LIDARR_URL:-not configured on this host} (recovery: ${LIDARR_RECOVERY:-false})" echo "$ICON_TIME Age thresh: ${ARR_IMPORT_RECOVERY_AGE}hr" echo "$ICON_GEAR Max attempts: ${ARR_RECOVERY_MAX_ATTEMPTS:-3} (chronic after this many)" echo "$ICON_GEAR Smart import: ${ARR_SMART_IMPORT_ENABLED:-true} (preferred language: ${ARR_SMART_IMPORT_PREFERRED_LANGUAGE:-English})" echo "$ICON_GEAR Sonarr ver: v${SONARR_VERSION_MAJOR} expected" echo "$ICON_GEAR Radarr ver: v${RADARR_VERSION_MAJOR} expected" echo "$ICON_GEAR Lidarr ver: v${LIDARR_VERSION_MAJOR} expected" echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)" echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Check if a queue item is older than ARR_IMPORT_RECOVERY_AGE # Returns 0 (old enough) or 1 (too new — skip) item_is_old_enough() { local added="$1" [[ -z "$added" ]] && return 0 # no date = treat as old enough, safe to act local added_epoch added_epoch=$(date -d "$added" +%s 2>/dev/null) || return 0 local age_seconds=$(( $(date +%s) - added_epoch )) [[ "$age_seconds" -ge "$AGE_THRESHOLD_SECONDS" ]] } # Query the arr queue API and return all records, paginated. # A single page=1&pageSize=200 request silently misses everything past record 200 — # on a busy Sonarr instance the queue can run into the thousands (e.g. a large # missing-episode search campaign), which pushed every importBlocked/warning item # past page 1 and made this whole script blind to them despite matching correctly. # Args: url, api_key, api_version get_queue_data() { local url="$1" api_key="$2" api_version="$3" local page=1 page_size=250 max_pages=50 local page_data page_count # Accumulate pages as files rather than growing a shell variable — on a large # queue (thousands of records) passing the combined JSON through --argjson # blows past ARG_MAX ("Argument list too long"). jq -s reads files instead. local tmp_dir tmp_dir=$(mktemp -d) trap 'rm -rf "$tmp_dir"' RETURN while [[ "$page" -le "$max_pages" ]]; do page_data=$(curl -sf --max-time 15 \ -H "X-Api-Key: $api_key" \ "${url}/api/${api_version}/queue?page=${page}&pageSize=${page_size}&includeUnknownSeriesItems=true&includeUnknownArtistItems=true" \ 2>/dev/null) [[ -z "$page_data" ]] && break page_count=$(echo "$page_data" | jq '.records // [] | length' 2>/dev/null) [[ -z "$page_count" || "$page_count" -eq 0 ]] && break echo "$page_data" | jq -c '.records // []' > "$tmp_dir/page_${page}.json" [[ "$page_count" -lt "$page_size" ]] && break (( page++ )) done jq -c -s '{totalRecords: ([.[][]] | length), records: [.[][]]}' "$tmp_dir"/page_*.json 2>/dev/null \ || echo '{"totalRecords":0,"records":[]}' } # Blocklist and remove a queue item # Args: url, api_key, api_version, queue_id blocklist_item() { local url="$1" api_key="$2" api_version="$3" queue_id="$4" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would blocklist queue item $queue_id" return 0 fi curl -sf --max-time 15 \ -X DELETE \ -H "X-Api-Key: $api_key" \ "${url}/api/${api_version}/queue/${queue_id}?removeFromClient=true&blocklist=true&skipRedownload=false" \ >/dev/null 2>&1 } # Trigger a new search for the media item # Args: url, api_key, api_version, arr_type, media_id trigger_search() { local url="$1" api_key="$2" api_version="$3" arr_type="$4" media_id="$5" local command body case "$arr_type" in sonarr) command="EpisodeSearch"; body="{\"name\":\"EpisodeSearch\",\"episodeIds\":[$media_id]}" ;; radarr) command="MoviesSearch"; body="{\"name\":\"MoviesSearch\",\"movieIds\":[$media_id]}" ;; lidarr) command="AlbumSearch"; body="{\"name\":\"AlbumSearch\",\"albumIds\":[$media_id]}" ;; esac if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would trigger $command for media ID $media_id" return 0 fi curl -sf --max-time 15 \ -X POST \ -H "X-Api-Key: $api_key" \ -H "Content-Type: application/json" \ -d "$body" \ "${url}/api/${api_version}/command" \ >/dev/null 2>&1 } # Decide whether an importBlocked download is actually worth keeping, and import # it directly if so — instead of always discarding it via blocklist+research. # # Fires the ManualImport command and returns as soon as it's accepted (HTTP 201) # rather than polling for completion. Deliberately does NOT fall back to # blocklist_item() after a successful trigger: Radarr/Sonarr's import runs async # in the background, and blocklisting (which deletes the source via # removeFromClient) right after firing it would race a still-in-progress import # for anything but the smallest files. If the import silently fails, the item # simply reappears as importBlocked next run and gets tried again — safe, if not # maximally fast, since the source file is never touched by this function. # # Returns 0 if a smart-import was triggered (caller should skip the normal # blocklist+research path for this item), 1 if declined or failed (caller should # fall through to the normal path exactly as before this function existed). # Args: url, api_key, api_version, arr_type, download_id, title try_smart_import() { local url="$1" api_key="$2" api_version="$3" arr_type="$4" download_id="$5" title="$6" # Lidarr's manual-import matching doesn't reliably resolve album/track context # (confirmed 2026-07-15 — 659/659 track candidates came back with no album # match at all) — not worth attempting, always fall through to normal handling. [[ "$arr_type" == "lidarr" ]] && return 1 [[ -z "$download_id" ]] && return 1 local candidates candidates=$(curl -sf --max-time 30 \ -H "X-Api-Key: $api_key" \ "${url}/api/${api_version}/manualimport?downloadId=${download_id}" \ 2>/dev/null) [[ -z "$candidates" || "$candidates" == "[]" || "$candidates" == "null" ]] && return 1 # Any rejected file (Sample, Unknown Movie/Series, "Not an upgrade", etc.) # disqualifies the whole download — conservative by design. local rejected_count rejected_count=$(echo "$candidates" | jq '[.[] | select(.rejections | length > 0)] | length' 2>/dev/null) [[ -z "$rejected_count" || "$rejected_count" -gt 0 ]] && return 1 local file_count file_count=$(echo "$candidates" | jq 'length' 2>/dev/null) [[ -z "$file_count" || "$file_count" -eq 0 ]] && return 1 local preferred_lang="${ARR_SMART_IMPORT_PREFERRED_LANGUAGE:-English}" local qualifying_files=() local i entry target_id has_file existing existing_res existing_lang candidate_res candidate_lang decision for (( i=0; i/dev/null) [[ -z "$entry" ]] && return 1 candidate_res=$(echo "$entry" | jq -r '.quality.quality.resolution // 0' 2>/dev/null) candidate_lang=$(echo "$entry" | jq -r '.languages[0].name // "Unknown"' 2>/dev/null) existing_res=0 existing_lang="Unknown" target_id="" has_file="false" case "$arr_type" in sonarr) # Multi-episode files complicate the existing-quality comparison per # episode — skip rather than guess when a release covers more than one. [[ "$(echo "$entry" | jq '.episodes | length' 2>/dev/null)" != "1" ]] && return 1 target_id=$(echo "$entry" | jq -r '.episodes[0].id // empty' 2>/dev/null) has_file=$(echo "$entry" | jq -r '.episodes[0].hasFile // false' 2>/dev/null) if [[ "$has_file" == "true" ]]; then existing=$(curl -sf --max-time 15 -H "X-Api-Key: $api_key" \ "${url}/api/${api_version}/episode/${target_id}?includeEpisodeFile=true" 2>/dev/null) existing_res=$(echo "$existing" | jq -r '.episodeFile.quality.quality.resolution // 0' 2>/dev/null) existing_lang=$(echo "$existing" | jq -r '.episodeFile.languages[0].name // "Unknown"' 2>/dev/null) fi # /manualimport only nests the IDs under .series.id / .episodes[].id — # the ManualImport command body needs them flattened to top-level # seriesId/episodeIds or Sonarr rejects the whole command with # "Series with ID 0 does not exist" (confirmed live 2026-07-19: every # smart-import this run reported as successful had actually failed # this way, silently, since the caller only checks the HTTP 201 accept). entry=$(echo "$entry" | jq -c --argjson eid "$target_id" \ '. + {seriesId: .series.id, episodeIds: [$eid]}' 2>/dev/null) ;; radarr) target_id=$(echo "$entry" | jq -r '.movie.id // empty' 2>/dev/null) has_file=$(echo "$entry" | jq -r '.movie.hasFile // false' 2>/dev/null) if [[ "$has_file" == "true" ]]; then existing=$(echo "$entry" | jq -c '.movie.movieFile // empty' 2>/dev/null) if [[ -z "$existing" || "$existing" == "null" ]]; then existing=$(curl -sf --max-time 15 -H "X-Api-Key: $api_key" \ "${url}/api/${api_version}/movie/${target_id}" 2>/dev/null | jq -c '.movieFile // empty') fi existing_res=$(echo "$existing" | jq -r '.quality.quality.resolution // 0' 2>/dev/null) existing_lang=$(echo "$existing" | jq -r '.languages[0].name // "Unknown"' 2>/dev/null) fi # Same flattening issue as Sonarr above — command body needs a # top-level movieId or Radarr rejects it with "Movie with ID 0 # does not exist". entry=$(echo "$entry" | jq -c --argjson mid "$target_id" '. + {movieId: $mid}' 2>/dev/null) ;; esac [[ -z "$target_id" || -z "$entry" ]] && return 1 if [[ "$has_file" != "true" ]]; then decision="import" # nothing there yet — fills a real gap elif [[ "$candidate_lang" != "$preferred_lang" ]]; then decision="decline" # never replace anything with a non-preferred language elif [[ "$existing_lang" != "$preferred_lang" ]]; then decision="import" # existing is wrong-language, candidate is right — upgrade elif [[ "$candidate_res" -gt "$existing_res" ]]; then decision="import" # same language, strictly higher resolution — upgrade else decision="decline" # same or worse, same language — no benefit fi [[ "$decision" == "decline" ]] && return 1 qualifying_files+=("$entry") done [[ "${#qualifying_files[@]}" -eq 0 ]] && return 1 if [[ "$DRY_RUN" == true ]]; then warn " DRY RUN — would smart-import: $title" return 0 fi local files_json cmd_body response http_code cmd_id cmd_status files_json=$(printf '%s\n' "${qualifying_files[@]}" | jq -s -c '.' 2>/dev/null) [[ -z "$files_json" ]] && return 1 cmd_body=$(jq -c -n --argjson files "$files_json" \ '{name:"ManualImport", files:$files, importMode:"auto"}' 2>/dev/null) [[ -z "$cmd_body" ]] && return 1 response=$(curl -s -w '\n%{http_code}' -X POST \ -H "X-Api-Key: $api_key" -H "Content-Type: application/json" \ -d "$cmd_body" \ "${url}/api/${api_version}/command" 2>/dev/null) http_code=$(echo "$response" | tail -1) cmd_id=$(echo "$response" | head -n -1 | jq -r '.id // empty' 2>/dev/null) [[ "$http_code" != "201" || -z "$cmd_id" ]] && return 1 # A bad payload (e.g. the seriesId/movieId=0 bug this was written to catch) # fails in ~10ms — well before any real file copy would even start — so a # brief poll here catches that failure class without racing a genuinely # long-running import, which is the reason this doesn't poll to completion. local _i for _i in 1 2 3; do sleep 1 cmd_status=$(curl -sf --max-time 10 -H "X-Api-Key: $api_key" \ "${url}/api/${api_version}/command/${cmd_id}" 2>/dev/null | jq -r '.status // empty') [[ "$cmd_status" == "failed" ]] && return 1 [[ "$cmd_status" == "completed" ]] && break done return 0 } # ============================================================================================== # ── PROCESS AN ARR ──────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Args: display_name, arr_type, url, api_key, api_version, enabled, # version_major, version_api_prefix # # Exits cleanly if disabled. # Checks API reachability and version before touching queue. # Processes each problem item: blocklist + trigger new search. # Silent when clean — only warns when problems found or actioned. process_arr() { local arr_name="$1" local arr_type="$2" local url="$3" local api_key="$4" local api_version="$5" local enabled="$6" local version_major="$7" local version_api_prefix="$8" local actioned=0 skipped_new=0 chronic=0 smart_imported=0 local is_chronic fail_key fail_count local -A seen_media_ids # media_ids appearing as a problem this run — anything NOT in # here by the end has stopped being a problem and has its # FAILURE_COUNTS entry pruned below echo "" echo "━━━ $ICON_SYNC $arr_name ━━━" # Disabled — skip cleanly if [[ "$enabled" != "true" ]]; then log "$arr_name recovery disabled — skipping" ARR_SUMMARIES+=("$arr_name: disabled") return fi # URL not configured on this host — skip cleanly if [[ -z "$url" ]]; then log "$arr_name not configured on $MY_ID — skipping" ARR_SUMMARIES+=("$arr_name: not configured on $MY_ID") return fi # API reachability if ! check_api "$url" "$arr_name" 10; then warn "$arr_name API unreachable — skipping" ARR_SUMMARIES+=("$arr_name: unreachable") return fi # Version check — exit if API structure may have changed if ! check_arr_version "$url" "$api_key" "$version_api_prefix" \ "$version_major" "$arr_name"; then ARR_SUMMARIES+=("$arr_name: version mismatch — skipped") return fi # Fetch queue local queue_data queue_data=$(get_queue_data "$url" "$api_key" "$api_version") if [[ -z "$queue_data" ]]; then warn "$arr_name — could not retrieve queue data" ARR_SUMMARIES+=("$arr_name: queue fetch failed") return fi local total_records total_records=$(echo "$queue_data" | jq '.totalRecords // 0' 2>/dev/null) log "$arr_name queue: $total_records total items" # Filter for problem items — never touch downloading or imported local problem_items problem_items=$(echo "$queue_data" | jq -c ' .records // [] | .[] | select( .trackedDownloadState != "downloading" and .trackedDownloadState != "imported" and ( .trackedDownloadState == "importFailed" or .trackedDownloadState == "importPending" or .trackedDownloadState == "importBlocked" or .trackedDownloadStatus == "error" or (.status == "warning" and ( (.errorMessage // "" | ascii_downcase | contains("stalled")) or (.statusMessages // [] | .[] | .messages // [] | .[] | ascii_downcase | contains("stalled")) )) ) ) ' 2>/dev/null) if [[ -z "$problem_items" ]]; then echo "$arr_name — clean ✅ no failed imports or stalled downloads" ARR_SUMMARIES+=("$arr_name: clean ✅") return fi local problem_count problem_count=$(echo "$problem_items" | wc -l) warn "$arr_name — found $problem_count problem item(s)" # Process each problem item while IFS= read -r item; do [[ -z "$item" ]] && continue local queue_id title added tracked_state tracked_status problem_type media_id download_id queue_id=$(echo "$item" | jq -r '.id // empty' 2>/dev/null) title=$(echo "$item" | jq -r '.title // "Unknown"' 2>/dev/null) added=$(echo "$item" | jq -r '.added // empty' 2>/dev/null) tracked_state=$(echo "$item" | jq -r '.trackedDownloadState // ""' 2>/dev/null) tracked_status=$(echo "$item" | jq -r '.trackedDownloadStatus // ""' 2>/dev/null) # Human-readable problem type case "$tracked_state" in importFailed) problem_type="import failed" ;; importPending) problem_type="import pending/stuck" ;; importBlocked) problem_type="import blocked (matched by ID)" ;; *) [[ "$tracked_status" == "error" ]] && \ problem_type="error" || problem_type="stalled" ;; esac # Media ID for search trigger case "$arr_type" in sonarr) media_id=$(echo "$item" | jq -r '.episodeId // .episode.id // empty' 2>/dev/null) ;; radarr) media_id=$(echo "$item" | jq -r '.movieId // .movie.id // empty' 2>/dev/null) ;; lidarr) media_id=$(echo "$item" | jq -r '.albumId // .album.id // empty' 2>/dev/null) ;; esac [[ -n "$media_id" ]] && seen_media_ids["$media_id"]=1 [[ -z "$queue_id" ]] && continue # Age check — skip items that are too new to have self-resolved if ! item_is_old_enough "$added"; then log " Skipping (too new < ${ARR_IMPORT_RECOVERY_AGE}hr): $title" (( skipped_new++ )) (( TOTAL_SKIPPED++ )) continue fi # importBlocked gets one extra chance before the normal blocklist path — # most are junk/duplicates and fall straight through unchanged, but a # clean same-language upgrade or gap-fill gets imported directly instead # of discarded. See try_smart_import() for the full decision logic. if [[ "${ARR_SMART_IMPORT_ENABLED:-true}" == "true" ]] && \ [[ "$tracked_state" == "importBlocked" ]]; then download_id=$(echo "$item" | jq -r '.downloadId // empty' 2>/dev/null) if try_smart_import "$url" "$api_key" "$api_version" "$arr_type" "$download_id" "$title"; then log " $ICON_DONE Smart-imported (upgrade/gap-fill): $title" (( smart_imported++ )) (( TOTAL_SMART_IMPORTED++ )) (( actioned++ )) (( TOTAL_ACTIONED++ )) continue fi fi warn " $ICON_TRASH $problem_type — $title" # Step 1: Blocklist + remove from queue if ! blocklist_item "$url" "$api_key" "$api_version" "$queue_id"; then warn " Failed to blocklist: $title" (( TOTAL_SKIPPED++ )) continue fi log " Blocklisted: $queue_id" # Step 2: Circuit breaker — track consecutive failures per (arr_type, media_id). # Some items can never resolve via blind retry (e.g. an album missing 1-2 tracks # where no available release matches the existing edition) — without this, the # same item gets blocklisted + re-searched forever, every run. is_chronic=false if [[ -n "$media_id" ]]; then fail_key="${arr_type}:${media_id}" fail_count=$(( ${FAILURE_COUNTS[$fail_key]:-0} + 1 )) FAILURE_COUNTS[$fail_key]="$fail_count" if [[ "$fail_count" -gt "$ARR_RECOVERY_MAX_ATTEMPTS" ]]; then is_chronic=true (( chronic++ )) (( TOTAL_CHRONIC++ )) warn " Chronic (${fail_count} consecutive failures) — needs manual review: $title" [[ "$fail_count" -eq $(( ARR_RECOVERY_MAX_ATTEMPTS + 1 )) ]] && \ notify "$arr_name item now chronic after ${ARR_RECOVERY_MAX_ATTEMPTS} failed attempts — needs manual review: $title" \ "Arr Recovery" "warning" fi fi # Step 3: Trigger new search — skipped for chronic items if [[ "$is_chronic" == true ]]; then log " Skipping auto re-search (chronic): $title" elif [[ -n "$media_id" ]]; then if trigger_search "$url" "$api_key" "$api_version" "$arr_type" "$media_id"; then log " New search triggered: $title" else warn " Blocklisted but search trigger failed: $title" fi else warn " Blocklisted but no media ID found — search not triggered: $title" fi (( actioned++ )) (( TOTAL_ACTIONED++ )) done <<< "$problem_items" # Prune stale failure counts — anything for this arr_type that isn't a problem in this # run's queue snapshot has either imported successfully or is otherwise no longer stuck. # FAILURE_COUNTS never decremented on success (confirmed live 2026-07-19: Sekirei S06E04 # sat chronic at count 4 despite hasFile=true, already fully resolved) — "consecutive # failures" is supposed to mean consecutive since it last wasn't a problem, not a # cumulative count for all time. Age-skipped items are still in seen_media_ids (added # before the age check above), so a too-new item correctly keeps its count instead of # being reset just for not having been acted on yet. local _fc_key _fc_id for _fc_key in "${!FAILURE_COUNTS[@]}"; do [[ "$_fc_key" == "${arr_type}:"* ]] || continue _fc_id="${_fc_key#${arr_type}:}" [[ -z "${seen_media_ids[$_fc_id]:-}" ]] && unset "FAILURE_COUNTS[$_fc_key]" done if [[ "$actioned" -gt 0 ]]; then warn "$arr_name — actioned: $actioned (smart-imported: $smart_imported) | skipped (too new): $skipped_new | chronic: $chronic" else log "$arr_name — nothing actioned | skipped (too new): $skipped_new" fi ARR_SUMMARIES+=("$arr_name: actioned $actioned (smart-imported $smart_imported) | too new $skipped_new | chronic $chronic") } # ============================================================================================== # ━━━ Process Each Arr ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_SYNC Arrs Failed/Stalled Recovery — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)" log "Age threshold: ${ARR_IMPORT_RECOVERY_AGE}hr" START=$(date +%s) # Sonarr — uses aliased vars set by detect_hosts() process_arr \ "Sonarr" \ "sonarr" \ "${SONARR_URL:-}" \ "${SONARR_API_KEY:-}" \ "v3" \ "${SONARR_RECOVERY:-true}" \ "${SONARR_VERSION_MAJOR:-4}" \ "v3" # Radarr — uses aliased vars set by detect_hosts() process_arr \ "Radarr" \ "radarr" \ "${RADARR_URL:-}" \ "${RADARR_API_KEY:-}" \ "v3" \ "${RADARR_RECOVERY:-true}" \ "${RADARR_VERSION_MAJOR:-6}" \ "v3" # Lidarr — HOST1 only, LIDARR_URL empty on HOST2 → exits cleanly via "not configured" guard process_arr \ "Lidarr" \ "lidarr" \ "${LIDARR_URL:-}" \ "${LIDARR_API_KEY:-}" \ "v1" \ "${LIDARR_RECOVERY:-false}" \ "${LIDARR_VERSION_MAJOR:-3}" \ "v1" END=$(date +%s) # Persist updated failure counts — skipped in dry-run so nothing is recorded for a preview if [[ "$DRY_RUN" == false ]]; then mkdir -p "$(dirname "$ARR_RECOVERY_FAILURE_COUNTS")" 2>/dev/null : > "$ARR_RECOVERY_FAILURE_COUNTS" for key in "${!FAILURE_COUNTS[@]}"; do echo "${key}|${FAILURE_COUNTS[$key]}" >> "$ARR_RECOVERY_FAILURE_COUNTS" done fi # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY ARR RECOVERY SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "$ICON_TRASH Actioned: $TOTAL_ACTIONED items blocklisted + searched" echo "$ICON_DONE Smart-imported: $TOTAL_SMART_IMPORTED items (upgrade/gap-fill, kept instead of discarded)" echo "$ICON_SKIP Skipped: $TOTAL_SKIPPED items (too new)" echo "$ICON_WARN Chronic: $TOTAL_CHRONIC items (blocklisted, auto re-search stopped)" echo "" for summary in "${ARR_SUMMARIES[@]}"; do echo " $ICON_SUMMARY $summary" done echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" elif [[ "$TOTAL_ACTIONED" -gt 0 ]]; then warn "$ICON_DONE Done — $TOTAL_ACTIONED item(s) blocklisted and re-searched" notify "Arr recovery on $(hostname) — $TOTAL_ACTIONED item(s) blocklisted and re-searched" \ "Arr Recovery" "warning" else echo "$ICON_DONE Done — nothing to recover (all arrs clean)" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Write stats for sunday_morning_coffee_report.sh if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_RECOVERY_STATS:-}" ]]; then echo "$(date '+%Y-%m-%d')|$(date '+%H:%M')|${TOTAL_ACTIONED}|${TOTAL_SKIPPED}|${TOTAL_CHRONIC}" \ >> "$ARR_RECOVERY_STATS" 2>/dev/null || true fi exit 0