From 39f2525d0f9c45e3684f908c53c6ccfbc61b76a5 Mon Sep 17 00:00:00 2001 From: FailedProxy Date: Wed, 22 Apr 2026 18:22:29 -0400 Subject: [PATCH] added a arrs atalled failled handler --- Master.conf | 37 ++- Media/arrs_faile-stalled_ recovery.sh | 337 ++++++++++++++++++++++++++ user_script_plug-in.sh | 7 +- 3 files changed, 375 insertions(+), 6 deletions(-) create mode 100644 Media/arrs_faile-stalled_ recovery.sh diff --git a/Master.conf b/Master.conf index f5bfb3e..226a0a6 100644 --- a/Master.conf +++ b/Master.conf @@ -818,7 +818,7 @@ NETWORK_CONNECT_CONTAINERS=( ) NETWORK_CONNECT_NETWORKS=( - "high-availability" # Docker network name — must exist before array start + "nextcloud-aio" # Docker network name — must exist before array start ) # ============================================================================================== @@ -948,9 +948,9 @@ MEDIA_MAINTENANCE_JOBS=( "Media/media_shares_permissions.sh" # apply permissions first "Media/media_cleaner.sh anime" # remove junk from anime shares "Media/media_cleaner.sh media" # remove junk from media shares -# "Media/lidarr_cleanup.sh" # remove orphaned music files -# "Media/sonarr_cleanup.sh" # remove orphaned TV files -# "Media/radarr_cleanup.sh" # remove orphaned movie files + "Media/lidarr_cleanup.sh" # remove orphaned music files + "Media/sonarr_cleanup.sh" # remove orphaned TV files + "Media/radarr_cleanup.sh" # remove orphaned movie files ) # ━━━ Arr Cleanup ━━━ @@ -1020,6 +1020,35 @@ RADARR_ORPHAN_AGE=7 RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") RADARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa") +# ━━━ Arr Import Recovery ━━━ +# Automatically blocklists and re-searches failed imports and stalled downloads. +# Runs daily at 5am — by this time overnight downloads are complete and any +# failures have had time to surface. Items under ARR_IMPORT_RECOVERY_AGE are +# skipped — gives the arr time to retry on its own before we intervene. +# +# Targets two problem types from the queue API: +# importFailed — downloaded successfully but arr couldn't import it +# stalled — download stuck with no connections or progress +# +# Action: blocklist the release + remove from queue + trigger new search +# Blocklist prevents the same bad release being grabbed again +# New search finds a different release automatically +# If all releases are bad → arr will exhaust options, manual check needed +# Notification tells you what was actioned so you can monitor +# +# Per-arr toggles — disable temporarily if an arr is having issues +# detect_hosts() selects correct URL and API key per server at runtime +# Lidarr runs on HOST1 only — exits cleanly on HOST2 + +ARR_IMPORT_RECOVERY_AGE=12 # hours — skip items newer than this, give arr time to retry + +# Per-arr enable/disable +HOST1_SONARR_RECOVERY=true # Tv_Shows import recovery +HOST1_RADARR_RECOVERY=true # Movies import recovery +HOST1_LIDARR_RECOVERY=true # Music import recovery — HOST1 only +HOST2_SONARR_RECOVERY=true # Anime_Shows import recovery +HOST2_RADARR_RECOVERY=true # Anime_Movies import recovery + # ============================================================================================== # ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── # ============================================================================================== diff --git a/Media/arrs_faile-stalled_ recovery.sh b/Media/arrs_faile-stalled_ recovery.sh new file mode 100644 index 0000000..8ca7445 --- /dev/null +++ b/Media/arrs_faile-stalled_ recovery.sh @@ -0,0 +1,337 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Arrs Failed Stalled Recovery ---------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Automatically detects and recovers from failed imports and stalled downloads +# across Sonarr, Radarr, and Lidarr. Blocklists the bad release and triggers +# a new search — hands free recovery while you sleep. +# +# Targets two problem types from the queue API: +# importFailed — downloaded successfully but arr couldn't import the file +# stalled — download stuck with no connections or no progress +# +# Action per problem item: +# 1. Blocklist the release — prevents re-grabbing the same bad release +# 2. Remove from queue — cleans up the failed item +# 3. Trigger new search — finds a different release automatically +# +# Age threshold (ARR_IMPORT_RECOVERY_AGE): +# Items newer than threshold are skipped — gives the arr time to retry on its own +# Items older than threshold have not self-resolved — safe to intervene +# Default: 12 hours +# +# API versions: +# Sonarr v4 → /api/v3/ (v3 endpoint retained in v4) +# Radarr v6 → /api/v3/ (v3 endpoint retained in v6) +# Lidarr v3 → /api/v1/ (different from Sonarr/Radarr) +# +# Per-arr enable/disable toggles in Master.conf. +# Lidarr runs on HOST1 only — exits cleanly on HOST2. +# detect_hosts() selects correct URL and API key per server at runtime. +# +# Recommended schedule: 0 5 * * * (5am daily) +# Supports --dry-run to show what would be actioned without making changes. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +acquire_lock + +detect_hosts + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no items will be blocklisted or searched" + +# Age threshold in seconds for comparison +AGE_THRESHOLD_SECONDS=$(( ARR_IMPORT_RECOVERY_AGE * 3600 )) + +# Tracking totals +TOTAL_ACTIONED=0 +TOTAL_SKIPPED=0 +ARR_SUMMARIES=() + +# ----------------------------------------------------------------------------------------------- +# CORE FUNCTIONS +# ----------------------------------------------------------------------------------------------- + +# Check if a queue item is older than ARR_IMPORT_RECOVERY_AGE +item_is_old_enough() { + local added="$1" + [[ -z "$added" ]] && return 0 # no date = treat as old enough + local added_epoch + added_epoch=$(date -d "$added" +%s 2>/dev/null) || return 0 + local now_epoch + now_epoch=$(date +%s) + local age_seconds=$(( now_epoch - added_epoch )) + [[ "$age_seconds" -ge "$AGE_THRESHOLD_SECONDS" ]] +} + +# Query the arr queue and return failed/stalled items +# Args: url, api_key, api_version +get_problem_items() { + local url="$1" api_key="$2" api_version="$3" + curl -sf --max-time 15 \ + -H "X-Api-Key: $api_key" \ + "${url}/api/${api_version}/queue?page=1&pageSize=200&includeUnknownSeriesItems=true&includeUnknownArtistItems=true" \ + 2>/dev/null +} + +# Blocklist and remove item from queue +# Args: url, api_key, api_version, queue_id +blocklist_item() { + local url="$1" api_key="$2" api_version="$3" queue_id="$4" + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would blocklist queue item $queue_id" + return 0 + fi + curl -sf --max-time 15 \ + -X DELETE \ + -H "X-Api-Key: $api_key" \ + "${url}/api/${api_version}/queue/${queue_id}?removeFromClient=true&blocklist=true&skipRedownload=false" \ + >/dev/null 2>&1 +} + +# Trigger new search +# Args: url, api_key, api_version, arr_type, media_id +trigger_search() { + local url="$1" api_key="$2" api_version="$3" arr_type="$4" media_id="$5" + local command body + case "$arr_type" in + sonarr) command="EpisodeSearch"; body="{\"name\":\"EpisodeSearch\",\"episodeIds\":[$media_id]}" ;; + radarr) command="MoviesSearch"; body="{\"name\":\"MoviesSearch\",\"movieIds\":[$media_id]}" ;; + lidarr) command="AlbumSearch"; body="{\"name\":\"AlbumSearch\",\"albumIds\":[$media_id]}" ;; + esac + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would trigger $command for ID $media_id" + return 0 + fi + curl -sf --max-time 15 \ + -X POST \ + -H "X-Api-Key: $api_key" \ + -H "Content-Type: application/json" \ + -d "$body" \ + "${url}/api/${api_version}/command" \ + >/dev/null 2>&1 +} + +# ----------------------------------------------------------------------------------------------- +# PROCESS AN ARR +# Args: arr_name, arr_type, url, api_key, api_version, enabled +# ----------------------------------------------------------------------------------------------- +process_arr() { + local arr_name="$1" arr_type="$2" url="$3" api_key="$4" api_version="$5" enabled="$6" + local actioned=0 skipped=0 too_new=0 + + echo "" + echo "━━━ $ICON_SYNC $arr_name ━━━" + + if [[ "$enabled" != "true" ]]; then + info "$arr_name recovery disabled — skipping" + ARR_SUMMARIES+=("$arr_name: disabled") + return + fi + + # API pre-flight + if ! check_api "$url" "$arr_name" 10; then + warn "$arr_name API unreachable — skipping" + ARR_SUMMARIES+=("$arr_name: unreachable") + return + fi + + # Get queue + local queue_data + queue_data=$(get_problem_items "$url" "$api_key" "$api_version") + if [[ -z "$queue_data" ]] || ! command -v jq >/dev/null 2>&1; then + warn "$arr_name — could not retrieve queue data" + ARR_SUMMARIES+=("$arr_name: queue fetch failed") + return + fi + + local total_records + total_records=$(echo "$queue_data" | jq '.totalRecords // 0' 2>/dev/null) + info "Queue: $total_records total items" + + # Filter for problem items — importFailed or stalled + local problem_items + problem_items=$(echo "$queue_data" | jq -c ' + .records // [] | + .[] | + select( + .trackedDownloadState == "importFailed" or + (.status == "warning" and ( + (.errorMessage // "" | ascii_downcase | contains("stalled")) or + (.statusMessages // [] | .[] | .messages // [] | .[] | ascii_downcase | contains("stalled")) + )) + ) + ' 2>/dev/null) + + if [[ -z "$problem_items" ]]; then + success "$arr_name — no failed imports or stalled downloads found" + ARR_SUMMARIES+=("$arr_name: clean ✅") + return + fi + + local problem_count + problem_count=$(echo "$problem_items" | wc -l) + info "Found $problem_count problem item(s)" + + # Process each problem item + while IFS= read -r item; do + [[ -z "$item" ]] && continue + + local queue_id title added problem_type media_id + queue_id=$(echo "$item" | jq -r '.id // empty' 2>/dev/null) + title=$(echo "$item" | jq -r '.title // "Unknown"' 2>/dev/null) + added=$(echo "$item" | jq -r '.added // empty' 2>/dev/null) + + # Determine problem type + local tracked_state + tracked_state=$(echo "$item" | jq -r '.trackedDownloadState // ""' 2>/dev/null) + if [[ "$tracked_state" == "importFailed" ]]; then + problem_type="import failed" + else + problem_type="stalled" + fi + + # Get media ID for search trigger (episode, movie, or album) + case "$arr_type" in + sonarr) media_id=$(echo "$item" | jq -r '.episodeId // .episode.id // empty' 2>/dev/null) ;; + radarr) media_id=$(echo "$item" | jq -r '.movieId // .movie.id // empty' 2>/dev/null) ;; + lidarr) media_id=$(echo "$item" | jq -r '.albumId // .album.id // empty' 2>/dev/null) ;; + esac + + [[ -z "$queue_id" ]] && continue + + # Age check + if ! item_is_old_enough "$added"; then + info " $ICON_TIME Skipping (too new < ${ARR_IMPORT_RECOVERY_AGE}hr): $title" + ((too_new++)) + ((skipped++)) + continue + fi + + info " $ICON_TRASH $problem_type: $title" + + # Blocklist + remove + if blocklist_item "$url" "$api_key" "$api_version" "$queue_id"; then + log " Blocklisted queue item: $queue_id" + else + warn " Failed to blocklist: $title" + ((skipped++)) + continue + fi + + # Trigger new search if we have a media ID + if [[ -n "$media_id" ]]; then + if trigger_search "$url" "$api_key" "$api_version" "$arr_type" "$media_id"; then + log " New search triggered for: $title" + ((actioned++)) + ((TOTAL_ACTIONED++)) + else + warn " Blocklisted but search trigger failed: $title" + ((actioned++)) + ((TOTAL_ACTIONED++)) + fi + else + warn " Blocklisted but no media ID found for search: $title" + ((actioned++)) + ((TOTAL_ACTIONED++)) + fi + + done <<< "$problem_items" + + if [[ "$DRY_RUN" == true ]]; then + info "$arr_name — dry run complete" + else + success "$arr_name — actioned: $actioned | skipped (too new): $too_new" + fi + + ARR_SUMMARIES+=("$arr_name: actioned $actioned | too new $too_new") + TOTAL_SKIPPED=$(( TOTAL_SKIPPED + skipped )) +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SYNC Process Each Arr ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_SYNC Arrs Failed Stalled Recovery — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "$ICON_TIME Age threshold: ${ARR_IMPORT_RECOVERY_AGE}hr (items newer than this are skipped)" +echo "" + +START=$(date +%s) + +# Sonarr +if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then + process_arr "Sonarr (Tv_Shows)" "sonarr" \ + "$HOST1_SONARR_URL" "$HOST1_SONARR_API_KEY" "v3" \ + "${HOST1_SONARR_RECOVERY:-true}" +else + process_arr "Sonarr (Anime_Shows)" "sonarr" \ + "$HOST2_SONARR_URL" "$HOST2_SONARR_API_KEY" "v3" \ + "${HOST2_SONARR_RECOVERY:-true}" +fi + +# Radarr +if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then + process_arr "Radarr (Movies)" "radarr" \ + "$HOST1_RADARR_URL" "$HOST1_RADARR_API_KEY" "v3" \ + "${HOST1_RADARR_RECOVERY:-true}" +else + process_arr "Radarr (Anime_Movies)" "radarr" \ + "$HOST2_RADARR_URL" "$HOST2_RADARR_API_KEY" "v3" \ + "${HOST2_RADARR_RECOVERY:-true}" +fi + +# Lidarr — HOST1 only +if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then + process_arr "Lidarr (Music)" "lidarr" \ + "$HOST1_LIDARR_URL" "$HOST1_LIDARR_API_KEY" "v1" \ + "${HOST1_LIDARR_RECOVERY:-true}" +else + info "Lidarr runs on $HOST1 only — skipping on $LOCAL_SERVER_NAME" +fi + +END=$(date +%s) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━━━ $ICON_SUMMARY ARR IMPORT RECOVERY SUMMARY ━━━━━" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" +echo "$ICON_TRASH Actioned: $TOTAL_ACTIONED items blocklisted + searched" +echo "$ICON_RUNNING Skipped: $TOTAL_SKIPPED items (too new or unreachable)" +echo "" + +for summary in "${ARR_SUMMARIES[@]}"; do + echo " $ICON_SUMMARY $summary" +done + +echo "" + +if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Status: DRY RUN — no changes made" +elif [[ "$TOTAL_ACTIONED" -gt 0 ]]; then + echo "$ICON_DONE Status: $ICON_SUCCESS DONE" + notify "Arr import recovery on $(hostname) — $TOTAL_ACTIONED item(s) blocklisted and re-searched. Check arrs for new downloads." "Arr Recovery" "normal" +else + echo "$ICON_DONE Status: $ICON_SUCCESS DONE — nothing to recover" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/user_script_plug-in.sh b/user_script_plug-in.sh index 518972c..ddfa517 100644 --- a/user_script_plug-in.sh +++ b/user_script_plug-in.sh @@ -216,6 +216,9 @@ #/mnt/user/appdata/unraid_scripts/Media/sonarr_cleanup.sh #/mnt/user/appdata/unraid_scripts/Media/radarr_cleanup.sh --dry-run --log #/mnt/user/appdata/unraid_scripts/Media/radarr_cleanup.sh +#/mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh --dry-run +#/mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh +# ^^ schedule: 0 5 * * * (5am daily) — always dry-run first # # ━━━ Transcodes ━━━ # transcode_management.sh runs cleanup then manager — schedule that, not the individuals. @@ -298,10 +301,10 @@ # */15 * * * * system_watchdog.sh # # ━━━ Daily ━━━ -# 0 1 * * * media_shares_sync.sh (media shares both directions) +# 0 1 * * * media_shares_sync.sh (media shares both directions) # 0 2 * * * media_management.sh (permissions + cleaners + arrs) -# 30 2 * * 0 critical_shares_maintenance.sh (clean Emby + auth stack sync) # 0 3 * * * docker_daily_restart.sh +# 0 5 * * * arrs_failed_stalled_recovery.sh (blocklist + re-search failed imports + stalled) # 0 8 * * * weekly_health_digest.sh (profile controls if it sends) # # ━━━ Rsync profiles — schedule individually ━━━