#!/bin/bash # ----------------------------------------------------------------------------------------------- # --------------------------------- Arrs Failed Stalled Recovery -------------------------------- # ----------------------------------------------------------------------------------------------- # Automatically detects and recovers from failed imports and stalled downloads # across Sonarr, Radarr, and Lidarr. Blocklists the bad release and triggers # a new search — hands free recovery while you sleep. # Schedule: 0 */6 * * * (every 6 hours) # # What it checks (per-arr toggles in Master.conf): # HOST1: Sonarr (Tv_Shows) — /api/v3/ # HOST1: Radarr (Movies) — /api/v3/ # HOST1: Lidarr (Music) — /api/v1/ ← HOST1 only, exits cleanly on HOST2 # HOST2: Sonarr (Anime_Shows) — /api/v3/ # HOST2: Radarr (Anime_Movies) — /api/v3/ # # Targets four problem types from the queue API: # importFailed — downloaded successfully but arr couldn't import the file # importPending — downloaded, stuck waiting to import (won't self-resolve) # error status — serious failure not covered by importFailed/importPending # stalled — download stuck with no connections or no progress # # Items newer than ARR_IMPORT_RECOVERY_AGE (6hr) are skipped — gives arr time to retry. # Never touches items with state "downloading" or "imported" — safe to run anytime. # # Action per problem item: # 1. Blocklist the release — prevents re-grabbing the same bad release # 2. Remove from queue — cleans up the failed item # 3. Trigger new search — finds a different release automatically # # Configuration in Master.conf: # ARR_IMPORT_RECOVERY_AGE — hours before item is eligible # HOST1/2_SONARR_RECOVERY — enable/disable per arr # HOST1/2_RADARR_RECOVERY — enable/disable per arr # HOST1_LIDARR_RECOVERY — enable/disable Lidarr (HOST1 only) # # Age threshold (ARR_IMPORT_RECOVERY_AGE): # Items newer than threshold are skipped — gives the arr time to retry on its own # Items older than threshold have not self-resolved — safe to intervene # Default: 12 hours # # API versions: # Sonarr v4 → /api/v3/ (v3 endpoint retained in v4) # Radarr v6 → /api/v3/ (v3 endpoint retained in v6) # Lidarr v3 → /api/v1/ (different from Sonarr/Radarr) # # Per-arr enable/disable toggles in Master.conf. # Lidarr runs on HOST1 only — exits cleanly on HOST2. # detect_hosts() selects correct URL and API key per server at runtime. # # Recommended schedule: 0 5 * * * (5am daily) # Supports --dry-run to show what would be actioned without making changes. # ----------------------------------------------------------------------------------------------- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../Master.conf" source "$SCRIPT_DIR/../common.sh" parse_args "$@" # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_GEAR Setup ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_GEAR Setup ━━━" if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi success "Running as root" acquire_lock detect_hosts [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no items will be blocklisted or searched" # Age threshold in seconds for comparison AGE_THRESHOLD_SECONDS=$(( ARR_IMPORT_RECOVERY_AGE * 3600 )) # Tracking totals TOTAL_ACTIONED=0 TOTAL_SKIPPED=0 ARR_SUMMARIES=() # ----------------------------------------------------------------------------------------------- # CORE FUNCTIONS # ----------------------------------------------------------------------------------------------- # Check if a queue item is older than ARR_IMPORT_RECOVERY_AGE item_is_old_enough() { local added="$1" [[ -z "$added" ]] && return 0 # no date = treat as old enough local added_epoch added_epoch=$(date -d "$added" +%s 2>/dev/null) || return 0 local now_epoch now_epoch=$(date +%s) local age_seconds=$(( now_epoch - added_epoch )) [[ "$age_seconds" -ge "$AGE_THRESHOLD_SECONDS" ]] } # Query the arr queue and return failed/stalled items # Args: url, api_key, api_version get_problem_items() { local url="$1" api_key="$2" api_version="$3" curl -sf --max-time 15 \ -H "X-Api-Key: $api_key" \ "${url}/api/${api_version}/queue?page=1&pageSize=200&includeUnknownSeriesItems=true&includeUnknownArtistItems=true" \ 2>/dev/null } # Blocklist and remove item from queue # Args: url, api_key, api_version, queue_id blocklist_item() { local url="$1" api_key="$2" api_version="$3" queue_id="$4" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would blocklist queue item $queue_id" return 0 fi curl -sf --max-time 15 \ -X DELETE \ -H "X-Api-Key: $api_key" \ "${url}/api/${api_version}/queue/${queue_id}?removeFromClient=true&blocklist=true&skipRedownload=false" \ >/dev/null 2>&1 } # Trigger new search # Args: url, api_key, api_version, arr_type, media_id trigger_search() { local url="$1" api_key="$2" api_version="$3" arr_type="$4" media_id="$5" local command body case "$arr_type" in sonarr) command="EpisodeSearch"; body="{\"name\":\"EpisodeSearch\",\"episodeIds\":[$media_id]}" ;; radarr) command="MoviesSearch"; body="{\"name\":\"MoviesSearch\",\"movieIds\":[$media_id]}" ;; lidarr) command="AlbumSearch"; body="{\"name\":\"AlbumSearch\",\"albumIds\":[$media_id]}" ;; esac if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would trigger $command for ID $media_id" return 0 fi curl -sf --max-time 15 \ -X POST \ -H "X-Api-Key: $api_key" \ -H "Content-Type: application/json" \ -d "$body" \ "${url}/api/${api_version}/command" \ >/dev/null 2>&1 } # ----------------------------------------------------------------------------------------------- # PROCESS AN ARR # Args: arr_name, arr_type, url, api_key, api_version, enabled # ----------------------------------------------------------------------------------------------- process_arr() { local arr_name="$1" arr_type="$2" url="$3" api_key="$4" api_version="$5" enabled="$6" local actioned=0 skipped=0 too_new=0 echo "" echo "━━━ $ICON_SYNC $arr_name ━━━" if [[ "$enabled" != "true" ]]; then info "$arr_name recovery disabled — skipping" ARR_SUMMARIES+=("$arr_name: disabled") return fi # API pre-flight if ! check_api "$url" "$arr_name" 10; then warn "$arr_name API unreachable — skipping" ARR_SUMMARIES+=("$arr_name: unreachable") return fi # Get queue local queue_data queue_data=$(get_problem_items "$url" "$api_key" "$api_version") if [[ -z "$queue_data" ]] || ! command -v jq >/dev/null 2>&1; then warn "$arr_name — could not retrieve queue data" ARR_SUMMARIES+=("$arr_name: queue fetch failed") return fi local total_records total_records=$(echo "$queue_data" | jq '.totalRecords // 0' 2>/dev/null) info "Queue: $total_records total items" # Filter for problem items — never touch downloading or imported # importFailed = tried to import, actually failed # importPending = downloaded, stuck waiting to import (won't self-resolve) # error status = serious failure not covered by above states # stalled = download stuck with no connections or progress local problem_items problem_items=$(echo "$queue_data" | jq -c ' .records // [] | .[] | select( .trackedDownloadState != "downloading" and .trackedDownloadState != "imported" and ( .trackedDownloadState == "importFailed" or .trackedDownloadState == "importPending" or .trackedDownloadStatus == "error" or (.status == "warning" and ( (.errorMessage // "" | ascii_downcase | contains("stalled")) or (.statusMessages // [] | .[] | .messages // [] | .[] | ascii_downcase | contains("stalled")) )) ) ) ' 2>/dev/null) if [[ -z "$problem_items" ]]; then success "$arr_name — no failed imports or stalled downloads found" ARR_SUMMARIES+=("$arr_name: clean ✅") return fi local problem_count problem_count=$(echo "$problem_items" | wc -l) info "Found $problem_count problem item(s)" # Process each problem item while IFS= read -r item; do [[ -z "$item" ]] && continue local queue_id title added problem_type media_id queue_id=$(echo "$item" | jq -r '.id // empty' 2>/dev/null) title=$(echo "$item" | jq -r '.title // "Unknown"' 2>/dev/null) added=$(echo "$item" | jq -r '.added // empty' 2>/dev/null) # Determine problem type for display local tracked_state tracked_status tracked_state=$(echo "$item" | jq -r '.trackedDownloadState // ""' 2>/dev/null) tracked_status=$(echo "$item" | jq -r '.trackedDownloadStatus // ""' 2>/dev/null) case "$tracked_state" in importFailed) problem_type="import failed" ;; importPending) problem_type="import pending/stuck" ;; *) if [[ "$tracked_status" == "error" ]]; then problem_type="error" else problem_type="stalled" fi ;; esac # Get media ID for search trigger (episode, movie, or album) case "$arr_type" in sonarr) media_id=$(echo "$item" | jq -r '.episodeId // .episode.id // empty' 2>/dev/null) ;; radarr) media_id=$(echo "$item" | jq -r '.movieId // .movie.id // empty' 2>/dev/null) ;; lidarr) media_id=$(echo "$item" | jq -r '.albumId // .album.id // empty' 2>/dev/null) ;; esac [[ -z "$queue_id" ]] && continue # Age check if ! item_is_old_enough "$added"; then info " $ICON_TIME Skipping (too new < ${ARR_IMPORT_RECOVERY_AGE}hr): $title" ((too_new++)) ((skipped++)) continue fi info " $ICON_TRASH $problem_type: $title" # Blocklist + remove if blocklist_item "$url" "$api_key" "$api_version" "$queue_id"; then log " Blocklisted queue item: $queue_id" else warn " Failed to blocklist: $title" ((skipped++)) continue fi # Trigger new search if we have a media ID if [[ -n "$media_id" ]]; then if trigger_search "$url" "$api_key" "$api_version" "$arr_type" "$media_id"; then log " New search triggered for: $title" ((actioned++)) ((TOTAL_ACTIONED++)) else warn " Blocklisted but search trigger failed: $title" ((actioned++)) ((TOTAL_ACTIONED++)) fi else warn " Blocklisted but no media ID found for search: $title" ((actioned++)) ((TOTAL_ACTIONED++)) fi done <<< "$problem_items" if [[ "$DRY_RUN" == true ]]; then info "$arr_name — dry run complete" else success "$arr_name — actioned: $actioned | skipped (too new): $too_new" fi ARR_SUMMARIES+=("$arr_name: actioned $actioned | too new $too_new") TOTAL_SKIPPED=$(( TOTAL_SKIPPED + skipped )) } # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SYNC Process Each Arr ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_SYNC Arrs Failed Stalled Recovery — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_TIME Age threshold: ${ARR_IMPORT_RECOVERY_AGE}hr (items newer than this are skipped)" echo "" START=$(date +%s) # Sonarr if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then process_arr "Sonarr (Tv_Shows)" "sonarr" \ "$HOST1_SONARR_URL" "$HOST1_SONARR_API_KEY" "v3" \ "${HOST1_SONARR_RECOVERY:-true}" else process_arr "Sonarr (Anime_Shows)" "sonarr" \ "$HOST2_SONARR_URL" "$HOST2_SONARR_API_KEY" "v3" \ "${HOST2_SONARR_RECOVERY:-true}" fi # Radarr if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then process_arr "Radarr (Movies)" "radarr" \ "$HOST1_RADARR_URL" "$HOST1_RADARR_API_KEY" "v3" \ "${HOST1_RADARR_RECOVERY:-true}" else process_arr "Radarr (Anime_Movies)" "radarr" \ "$HOST2_RADARR_URL" "$HOST2_RADARR_API_KEY" "v3" \ "${HOST2_RADARR_RECOVERY:-true}" fi # Lidarr — HOST1 only if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then process_arr "Lidarr (Music)" "lidarr" \ "$HOST1_LIDARR_URL" "$HOST1_LIDARR_API_KEY" "v1" \ "${HOST1_LIDARR_RECOVERY:-true}" else info "Lidarr runs on $HOST1 only — skipping on $LOCAL_SERVER_NAME" fi END=$(date +%s) # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SUMMARY Summary ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━━━ $ICON_SUMMARY ARR IMPORT RECOVERY SUMMARY ━━━━━" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "$ICON_TRASH Actioned: $TOTAL_ACTIONED items blocklisted + searched" echo "$ICON_RUNNING Skipped: $TOTAL_SKIPPED items (too new or unreachable)" echo "" echo " Problem types detected: importFailed | importPending | error | stalled" echo "" for summary in "${ARR_SUMMARIES[@]}"; do echo " $ICON_SUMMARY $summary" done echo "" if [[ "$DRY_RUN" == true ]]; then echo "$ICON_WARN Status: DRY RUN — no changes made" elif [[ "$TOTAL_ACTIONED" -gt 0 ]]; then echo "$ICON_DONE Status: $ICON_SUCCESS DONE" notify "Arr import recovery on $(hostname) — $TOTAL_ACTIONED item(s) blocklisted and re-searched. Check arrs for new downloads." "Arr Recovery" "normal" else echo "$ICON_DONE Status: $ICON_SUCCESS DONE — nothing to recover" fi # Write stats for sunday_morning_coffee_report.sh if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_RECOVERY_STATS:-}" ]]; then DATE=$(date '+%Y-%m-%d') TIME=$(date '+%H:%M') echo "${DATE}|${TIME}|${TOTAL_ACTIONED}|${TOTAL_SKIPPED}" \ >> "$ARR_RECOVERY_STATS" 2>/dev/null || true fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"