#!/bin/bash # ============================================================================================== # ========================= Arrs Failed / Stalled Recovery ===================================== # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Detect and recover failed imports and stalled downloads across Sonarr, Radarr, # and Lidarr. Blocklists the bad release and triggers a re-search — hands-free # overnight recovery. # # ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # # Four problem types detected from the arr queue API: # importFailed — downloaded but arr couldn't import the file # importPending — downloaded, stuck waiting to import (will not self-resolve) # error status — serious failure not covered by the above two states # stalled — download stuck with no connections or no progress # # Never touches items with state "downloading" or "imported" — safe to run anytime. # Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first. # # Per problem item (3-step response): # 1. Blocklist the release — prevents re-grabbing the same bad release # 2. Remove from queue — cleans up the failed item # 3. Trigger new search — finds a different release automatically # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # acquire_lock — prevents concurrent runs overlapping # jq validation — exits if jq not installed (required for JSON parsing) # API pre-flight — checks each arr is reachable before querying queue # Version check — check_arr_version() verifies running arr matches master.conf major # version; exits rather than silently misoperating after upgrade # Age threshold — skips items newer than ARR_IMPORT_RECOVERY_AGE (default 6hr) # Silent by default — only problems produce output, clean arrs stay silent # # API version mapping (endpoint paths differ from major version labels): # Sonarr v4 → /api/v3/ (v3 endpoint retained in v4) # Radarr v6 → /api/v3/ (v3 endpoint retained in v6) # Lidarr v3 → /api/v1/ (different from Sonarr/Radarr) # # ============================================================================================== # STATE FILES # ============================================================================================== # # ARR_RECOVERY_STATS — stats file written after each run (read by coffee report) # # ============================================================================================== # CONFIGURATION # ============================================================================================== # # master_host*.conf # # HOST*_SONARR_URL / HOST*_SONARR_API_KEY / HOST*_SONARR_RECOVERY # HOST*_RADARR_URL / HOST*_RADARR_API_KEY / HOST*_RADARR_RECOVERY # HOST1_LIDARR_URL / HOST1_LIDARR_API_KEY / HOST1_LIDARR_RECOVERY # All aliased by detect_hosts() — script uses unprefixed names # # master.conf # # ARR_IMPORT_RECOVERY_AGE — hours before item is eligible for recovery (default: 6) # SONARR_VERSION_MAJOR — expected Sonarr major version (e.g. 4) # RADARR_VERSION_MAJOR — expected Radarr major version (e.g. 6) # LIDARR_VERSION_MAJOR — expected Lidarr major version (e.g. 3) # ARR_RECOVERY_STATS — stats file path (read by coffee report) # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # arrs_failed_stalled_recovery.sh — normal run # arrs_failed_stalled_recovery.sh --dry-run — show what would be actioned, no changes # arrs_failed_stalled_recovery.sh --log — verbose output # arrs_failed_stalled_recovery.sh --status — show config and exit # # Recommended schedule: 0 5 * * * (5am daily) # Or every 6hr: 0 */6 * * * (matches ARR_IMPORT_RECOVERY_AGE default) # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi acquire_lock # detect_hosts() sets MY_ID and aliases SONARR_*, RADARR_*, LIDARR_* vars detect_hosts # jq is required — not optional — for JSON parsing if ! command -v jq >/dev/null 2>&1; then error "jq is not installed — required for arr API JSON parsing" error "Install: apt-get install jq or brew install jq" notify "arrs_failed_stalled_recovery failed on $(hostname) — jq not installed" \ "Arr Recovery" "warning" exit 1 fi log "jq found" validate_unraid_cmd \ "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ "" "" \ "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no items will be blocklisted or searched" # Age threshold in seconds AGE_THRESHOLD_SECONDS=$(( ARR_IMPORT_RECOVERY_AGE * 3600 )) TOTAL_ACTIONED=0 TOTAL_SKIPPED=0 ARR_SUMMARIES=() # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_SYNC Sonarr: ${SONARR_URL:-not configured} (recovery: ${SONARR_RECOVERY:-true})" echo "$ICON_SYNC Radarr: ${RADARR_URL:-not configured} (recovery: ${RADARR_RECOVERY:-true})" echo "$ICON_SYNC Lidarr: ${LIDARR_URL:-not configured on this host} (recovery: ${LIDARR_RECOVERY:-false})" echo "$ICON_TIME Age thresh: ${ARR_IMPORT_RECOVERY_AGE}hr" echo "$ICON_GEAR Sonarr ver: v${SONARR_VERSION_MAJOR} expected" echo "$ICON_GEAR Radarr ver: v${RADARR_VERSION_MAJOR} expected" echo "$ICON_GEAR Lidarr ver: v${LIDARR_VERSION_MAJOR} expected" echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)" echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Check if a queue item is older than ARR_IMPORT_RECOVERY_AGE # Returns 0 (old enough) or 1 (too new — skip) item_is_old_enough() { local added="$1" [[ -z "$added" ]] && return 0 # no date = treat as old enough, safe to act local added_epoch added_epoch=$(date -d "$added" +%s 2>/dev/null) || return 0 local age_seconds=$(( $(date +%s) - added_epoch )) [[ "$age_seconds" -ge "$AGE_THRESHOLD_SECONDS" ]] } # Query the arr queue API and return all records # Args: url, api_key, api_version get_queue_data() { local url="$1" api_key="$2" api_version="$3" curl -sf --max-time 15 \ -H "X-Api-Key: $api_key" \ "${url}/api/${api_version}/queue?page=1&pageSize=200&includeUnknownSeriesItems=true&includeUnknownArtistItems=true" \ 2>/dev/null } # Blocklist and remove a queue item # Args: url, api_key, api_version, queue_id blocklist_item() { local url="$1" api_key="$2" api_version="$3" queue_id="$4" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would blocklist queue item $queue_id" return 0 fi curl -sf --max-time 15 \ -X DELETE \ -H "X-Api-Key: $api_key" \ "${url}/api/${api_version}/queue/${queue_id}?removeFromClient=true&blocklist=true&skipRedownload=false" \ >/dev/null 2>&1 } # Trigger a new search for the media item # Args: url, api_key, api_version, arr_type, media_id trigger_search() { local url="$1" api_key="$2" api_version="$3" arr_type="$4" media_id="$5" local command body case "$arr_type" in sonarr) command="EpisodeSearch"; body="{\"name\":\"EpisodeSearch\",\"episodeIds\":[$media_id]}" ;; radarr) command="MoviesSearch"; body="{\"name\":\"MoviesSearch\",\"movieIds\":[$media_id]}" ;; lidarr) command="AlbumSearch"; body="{\"name\":\"AlbumSearch\",\"albumIds\":[$media_id]}" ;; esac if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would trigger $command for media ID $media_id" return 0 fi curl -sf --max-time 15 \ -X POST \ -H "X-Api-Key: $api_key" \ -H "Content-Type: application/json" \ -d "$body" \ "${url}/api/${api_version}/command" \ >/dev/null 2>&1 } # ============================================================================================== # ── PROCESS AN ARR ──────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Args: display_name, arr_type, url, api_key, api_version, enabled, # version_major, version_api_prefix # # Exits cleanly if disabled. # Checks API reachability and version before touching queue. # Processes each problem item: blocklist + trigger new search. # Silent when clean — only warns when problems found or actioned. process_arr() { local arr_name="$1" local arr_type="$2" local url="$3" local api_key="$4" local api_version="$5" local enabled="$6" local version_major="$7" local version_api_prefix="$8" local actioned=0 skipped_new=0 echo "" echo "━━━ $ICON_SYNC $arr_name ━━━" # Disabled — skip cleanly if [[ "$enabled" != "true" ]]; then log "$arr_name recovery disabled — skipping" ARR_SUMMARIES+=("$arr_name: disabled") return fi # URL not configured on this host — skip cleanly if [[ -z "$url" ]]; then log "$arr_name not configured on $MY_ID — skipping" ARR_SUMMARIES+=("$arr_name: not configured on $MY_ID") return fi # API reachability if ! check_api "$url" "$arr_name" 10; then warn "$arr_name API unreachable — skipping" ARR_SUMMARIES+=("$arr_name: unreachable") return fi # Version check — exit if API structure may have changed if ! check_arr_version "$url" "$api_key" "$version_api_prefix" \ "$version_major" "$arr_name"; then ARR_SUMMARIES+=("$arr_name: version mismatch — skipped") return fi # Fetch queue local queue_data queue_data=$(get_queue_data "$url" "$api_key" "$api_version") if [[ -z "$queue_data" ]]; then warn "$arr_name — could not retrieve queue data" ARR_SUMMARIES+=("$arr_name: queue fetch failed") return fi local total_records total_records=$(echo "$queue_data" | jq '.totalRecords // 0' 2>/dev/null) log "$arr_name queue: $total_records total items" # Filter for problem items — never touch downloading or imported local problem_items problem_items=$(echo "$queue_data" | jq -c ' .records // [] | .[] | select( .trackedDownloadState != "downloading" and .trackedDownloadState != "imported" and ( .trackedDownloadState == "importFailed" or .trackedDownloadState == "importPending" or .trackedDownloadStatus == "error" or (.status == "warning" and ( (.errorMessage // "" | ascii_downcase | contains("stalled")) or (.statusMessages // [] | .[] | .messages // [] | .[] | ascii_downcase | contains("stalled")) )) ) ) ' 2>/dev/null) if [[ -z "$problem_items" ]]; then log "$arr_name — clean ✅ no failed imports or stalled downloads" ARR_SUMMARIES+=("$arr_name: clean ✅") return fi local problem_count problem_count=$(echo "$problem_items" | wc -l) warn "$arr_name — found $problem_count problem item(s)" # Process each problem item while IFS= read -r item; do [[ -z "$item" ]] && continue local queue_id title added tracked_state tracked_status problem_type media_id queue_id=$(echo "$item" | jq -r '.id // empty' 2>/dev/null) title=$(echo "$item" | jq -r '.title // "Unknown"' 2>/dev/null) added=$(echo "$item" | jq -r '.added // empty' 2>/dev/null) tracked_state=$(echo "$item" | jq -r '.trackedDownloadState // ""' 2>/dev/null) tracked_status=$(echo "$item" | jq -r '.trackedDownloadStatus // ""' 2>/dev/null) # Human-readable problem type case "$tracked_state" in importFailed) problem_type="import failed" ;; importPending) problem_type="import pending/stuck" ;; *) [[ "$tracked_status" == "error" ]] && \ problem_type="error" || problem_type="stalled" ;; esac # Media ID for search trigger case "$arr_type" in sonarr) media_id=$(echo "$item" | jq -r '.episodeId // .episode.id // empty' 2>/dev/null) ;; radarr) media_id=$(echo "$item" | jq -r '.movieId // .movie.id // empty' 2>/dev/null) ;; lidarr) media_id=$(echo "$item" | jq -r '.albumId // .album.id // empty' 2>/dev/null) ;; esac [[ -z "$queue_id" ]] && continue # Age check — skip items that are too new to have self-resolved if ! item_is_old_enough "$added"; then log " Skipping (too new < ${ARR_IMPORT_RECOVERY_AGE}hr): $title" (( skipped_new++ )) (( TOTAL_SKIPPED++ )) continue fi warn " $ICON_TRASH $problem_type — $title" # Step 1: Blocklist + remove from queue if ! blocklist_item "$url" "$api_key" "$api_version" "$queue_id"; then warn " Failed to blocklist: $title" (( TOTAL_SKIPPED++ )) continue fi log " Blocklisted: $queue_id" # Step 2: Trigger new search if [[ -n "$media_id" ]]; then if trigger_search "$url" "$api_key" "$api_version" "$arr_type" "$media_id"; then log " New search triggered: $title" else warn " Blocklisted but search trigger failed: $title" fi else warn " Blocklisted but no media ID found — search not triggered: $title" fi (( actioned++ )) (( TOTAL_ACTIONED++ )) done <<< "$problem_items" if [[ "$actioned" -gt 0 ]]; then warn "$arr_name — actioned: $actioned | skipped (too new): $skipped_new" else log "$arr_name — nothing actioned | skipped (too new): $skipped_new" fi ARR_SUMMARIES+=("$arr_name: actioned $actioned | too new $skipped_new") } # ============================================================================================== # ━━━ Process Each Arr ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_SYNC Arrs Failed/Stalled Recovery — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)" log "Age threshold: ${ARR_IMPORT_RECOVERY_AGE}hr" START=$(date +%s) # Sonarr — uses aliased vars set by detect_hosts() process_arr \ "Sonarr" \ "sonarr" \ "${SONARR_URL:-}" \ "${SONARR_API_KEY:-}" \ "v3" \ "${SONARR_RECOVERY:-true}" \ "${SONARR_VERSION_MAJOR:-4}" \ "v3" # Radarr — uses aliased vars set by detect_hosts() process_arr \ "Radarr" \ "radarr" \ "${RADARR_URL:-}" \ "${RADARR_API_KEY:-}" \ "v3" \ "${RADARR_RECOVERY:-true}" \ "${RADARR_VERSION_MAJOR:-6}" \ "v3" # Lidarr — HOST1 only, LIDARR_URL empty on HOST2 → exits cleanly via "not configured" guard process_arr \ "Lidarr" \ "lidarr" \ "${LIDARR_URL:-}" \ "${LIDARR_API_KEY:-}" \ "v1" \ "${LIDARR_RECOVERY:-false}" \ "${LIDARR_VERSION_MAJOR:-3}" \ "v1" END=$(date +%s) # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY ARR RECOVERY SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "$ICON_TRASH Actioned: $TOTAL_ACTIONED items blocklisted + searched" echo "$ICON_SKIP Skipped: $TOTAL_SKIPPED items (too new)" echo "" for summary in "${ARR_SUMMARIES[@]}"; do echo " $ICON_SUMMARY $summary" done echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" elif [[ "$TOTAL_ACTIONED" -gt 0 ]]; then warn "$ICON_DONE Done — $TOTAL_ACTIONED item(s) blocklisted and re-searched" notify "Arr recovery on $(hostname) — $TOTAL_ACTIONED item(s) blocklisted and re-searched" \ "Arr Recovery" "warning" else log "$ICON_DONE Done — nothing to recover (all arrs clean)" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Write stats for sunday_morning_coffee_report.sh if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_RECOVERY_STATS:-}" ]]; then echo "$(date '+%Y-%m-%d')|$(date '+%H:%M')|${TOTAL_ACTIONED}|${TOTAL_SKIPPED}" \ >> "$ARR_RECOVERY_STATS" 2>/dev/null || true fi exit 0