added a arrs atalled failled handler
This commit is contained in:
@@ -0,0 +1,337 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Arrs Failed Stalled Recovery ----------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Automatically detects and recovers from failed imports and stalled downloads
|
||||
# across Sonarr, Radarr, and Lidarr. Blocklists the bad release and triggers
|
||||
# a new search — hands free recovery while you sleep.
|
||||
#
|
||||
# Targets two problem types from the queue API:
|
||||
# importFailed — downloaded successfully but arr couldn't import the file
|
||||
# stalled — download stuck with no connections or no progress
|
||||
#
|
||||
# Action per problem item:
|
||||
# 1. Blocklist the release — prevents re-grabbing the same bad release
|
||||
# 2. Remove from queue — cleans up the failed item
|
||||
# 3. Trigger new search — finds a different release automatically
|
||||
#
|
||||
# Age threshold (ARR_IMPORT_RECOVERY_AGE):
|
||||
# Items newer than threshold are skipped — gives the arr time to retry on its own
|
||||
# Items older than threshold have not self-resolved — safe to intervene
|
||||
# Default: 12 hours
|
||||
#
|
||||
# API versions:
|
||||
# Sonarr v4 → /api/v3/ (v3 endpoint retained in v4)
|
||||
# Radarr v6 → /api/v3/ (v3 endpoint retained in v6)
|
||||
# Lidarr v3 → /api/v1/ (different from Sonarr/Radarr)
|
||||
#
|
||||
# Per-arr enable/disable toggles in Master.conf.
|
||||
# Lidarr runs on HOST1 only — exits cleanly on HOST2.
|
||||
# detect_hosts() selects correct URL and API key per server at runtime.
|
||||
#
|
||||
# Recommended schedule: 0 5 * * * (5am daily)
|
||||
# Supports --dry-run to show what would be actioned without making changes.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no items will be blocklisted or searched"
|
||||
|
||||
# Age threshold in seconds for comparison
|
||||
AGE_THRESHOLD_SECONDS=$(( ARR_IMPORT_RECOVERY_AGE * 3600 ))
|
||||
|
||||
# Tracking totals
|
||||
TOTAL_ACTIONED=0
|
||||
TOTAL_SKIPPED=0
|
||||
ARR_SUMMARIES=()
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# CORE FUNCTIONS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# Check if a queue item is older than ARR_IMPORT_RECOVERY_AGE
|
||||
item_is_old_enough() {
|
||||
local added="$1"
|
||||
[[ -z "$added" ]] && return 0 # no date = treat as old enough
|
||||
local added_epoch
|
||||
added_epoch=$(date -d "$added" +%s 2>/dev/null) || return 0
|
||||
local now_epoch
|
||||
now_epoch=$(date +%s)
|
||||
local age_seconds=$(( now_epoch - added_epoch ))
|
||||
[[ "$age_seconds" -ge "$AGE_THRESHOLD_SECONDS" ]]
|
||||
}
|
||||
|
||||
# Query the arr queue and return failed/stalled items
|
||||
# Args: url, api_key, api_version
|
||||
get_problem_items() {
|
||||
local url="$1" api_key="$2" api_version="$3"
|
||||
curl -sf --max-time 15 \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
"${url}/api/${api_version}/queue?page=1&pageSize=200&includeUnknownSeriesItems=true&includeUnknownArtistItems=true" \
|
||||
2>/dev/null
|
||||
}
|
||||
|
||||
# Blocklist and remove item from queue
|
||||
# Args: url, api_key, api_version, queue_id
|
||||
blocklist_item() {
|
||||
local url="$1" api_key="$2" api_version="$3" queue_id="$4"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would blocklist queue item $queue_id"
|
||||
return 0
|
||||
fi
|
||||
curl -sf --max-time 15 \
|
||||
-X DELETE \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
"${url}/api/${api_version}/queue/${queue_id}?removeFromClient=true&blocklist=true&skipRedownload=false" \
|
||||
>/dev/null 2>&1
|
||||
}
|
||||
|
||||
# Trigger new search
|
||||
# Args: url, api_key, api_version, arr_type, media_id
|
||||
trigger_search() {
|
||||
local url="$1" api_key="$2" api_version="$3" arr_type="$4" media_id="$5"
|
||||
local command body
|
||||
case "$arr_type" in
|
||||
sonarr) command="EpisodeSearch"; body="{\"name\":\"EpisodeSearch\",\"episodeIds\":[$media_id]}" ;;
|
||||
radarr) command="MoviesSearch"; body="{\"name\":\"MoviesSearch\",\"movieIds\":[$media_id]}" ;;
|
||||
lidarr) command="AlbumSearch"; body="{\"name\":\"AlbumSearch\",\"albumIds\":[$media_id]}" ;;
|
||||
esac
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would trigger $command for ID $media_id"
|
||||
return 0
|
||||
fi
|
||||
curl -sf --max-time 15 \
|
||||
-X POST \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$body" \
|
||||
"${url}/api/${api_version}/command" \
|
||||
>/dev/null 2>&1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# PROCESS AN ARR
|
||||
# Args: arr_name, arr_type, url, api_key, api_version, enabled
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
process_arr() {
|
||||
local arr_name="$1" arr_type="$2" url="$3" api_key="$4" api_version="$5" enabled="$6"
|
||||
local actioned=0 skipped=0 too_new=0
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC $arr_name ━━━"
|
||||
|
||||
if [[ "$enabled" != "true" ]]; then
|
||||
info "$arr_name recovery disabled — skipping"
|
||||
ARR_SUMMARIES+=("$arr_name: disabled")
|
||||
return
|
||||
fi
|
||||
|
||||
# API pre-flight
|
||||
if ! check_api "$url" "$arr_name" 10; then
|
||||
warn "$arr_name API unreachable — skipping"
|
||||
ARR_SUMMARIES+=("$arr_name: unreachable")
|
||||
return
|
||||
fi
|
||||
|
||||
# Get queue
|
||||
local queue_data
|
||||
queue_data=$(get_problem_items "$url" "$api_key" "$api_version")
|
||||
if [[ -z "$queue_data" ]] || ! command -v jq >/dev/null 2>&1; then
|
||||
warn "$arr_name — could not retrieve queue data"
|
||||
ARR_SUMMARIES+=("$arr_name: queue fetch failed")
|
||||
return
|
||||
fi
|
||||
|
||||
local total_records
|
||||
total_records=$(echo "$queue_data" | jq '.totalRecords // 0' 2>/dev/null)
|
||||
info "Queue: $total_records total items"
|
||||
|
||||
# Filter for problem items — importFailed or stalled
|
||||
local problem_items
|
||||
problem_items=$(echo "$queue_data" | jq -c '
|
||||
.records // [] |
|
||||
.[] |
|
||||
select(
|
||||
.trackedDownloadState == "importFailed" or
|
||||
(.status == "warning" and (
|
||||
(.errorMessage // "" | ascii_downcase | contains("stalled")) or
|
||||
(.statusMessages // [] | .[] | .messages // [] | .[] | ascii_downcase | contains("stalled"))
|
||||
))
|
||||
)
|
||||
' 2>/dev/null)
|
||||
|
||||
if [[ -z "$problem_items" ]]; then
|
||||
success "$arr_name — no failed imports or stalled downloads found"
|
||||
ARR_SUMMARIES+=("$arr_name: clean ✅")
|
||||
return
|
||||
fi
|
||||
|
||||
local problem_count
|
||||
problem_count=$(echo "$problem_items" | wc -l)
|
||||
info "Found $problem_count problem item(s)"
|
||||
|
||||
# Process each problem item
|
||||
while IFS= read -r item; do
|
||||
[[ -z "$item" ]] && continue
|
||||
|
||||
local queue_id title added problem_type media_id
|
||||
queue_id=$(echo "$item" | jq -r '.id // empty' 2>/dev/null)
|
||||
title=$(echo "$item" | jq -r '.title // "Unknown"' 2>/dev/null)
|
||||
added=$(echo "$item" | jq -r '.added // empty' 2>/dev/null)
|
||||
|
||||
# Determine problem type
|
||||
local tracked_state
|
||||
tracked_state=$(echo "$item" | jq -r '.trackedDownloadState // ""' 2>/dev/null)
|
||||
if [[ "$tracked_state" == "importFailed" ]]; then
|
||||
problem_type="import failed"
|
||||
else
|
||||
problem_type="stalled"
|
||||
fi
|
||||
|
||||
# Get media ID for search trigger (episode, movie, or album)
|
||||
case "$arr_type" in
|
||||
sonarr) media_id=$(echo "$item" | jq -r '.episodeId // .episode.id // empty' 2>/dev/null) ;;
|
||||
radarr) media_id=$(echo "$item" | jq -r '.movieId // .movie.id // empty' 2>/dev/null) ;;
|
||||
lidarr) media_id=$(echo "$item" | jq -r '.albumId // .album.id // empty' 2>/dev/null) ;;
|
||||
esac
|
||||
|
||||
[[ -z "$queue_id" ]] && continue
|
||||
|
||||
# Age check
|
||||
if ! item_is_old_enough "$added"; then
|
||||
info " $ICON_TIME Skipping (too new < ${ARR_IMPORT_RECOVERY_AGE}hr): $title"
|
||||
((too_new++))
|
||||
((skipped++))
|
||||
continue
|
||||
fi
|
||||
|
||||
info " $ICON_TRASH $problem_type: $title"
|
||||
|
||||
# Blocklist + remove
|
||||
if blocklist_item "$url" "$api_key" "$api_version" "$queue_id"; then
|
||||
log " Blocklisted queue item: $queue_id"
|
||||
else
|
||||
warn " Failed to blocklist: $title"
|
||||
((skipped++))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Trigger new search if we have a media ID
|
||||
if [[ -n "$media_id" ]]; then
|
||||
if trigger_search "$url" "$api_key" "$api_version" "$arr_type" "$media_id"; then
|
||||
log " New search triggered for: $title"
|
||||
((actioned++))
|
||||
((TOTAL_ACTIONED++))
|
||||
else
|
||||
warn " Blocklisted but search trigger failed: $title"
|
||||
((actioned++))
|
||||
((TOTAL_ACTIONED++))
|
||||
fi
|
||||
else
|
||||
warn " Blocklisted but no media ID found for search: $title"
|
||||
((actioned++))
|
||||
((TOTAL_ACTIONED++))
|
||||
fi
|
||||
|
||||
done <<< "$problem_items"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
info "$arr_name — dry run complete"
|
||||
else
|
||||
success "$arr_name — actioned: $actioned | skipped (too new): $too_new"
|
||||
fi
|
||||
|
||||
ARR_SUMMARIES+=("$arr_name: actioned $actioned | too new $too_new")
|
||||
TOTAL_SKIPPED=$(( TOTAL_SKIPPED + skipped ))
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SYNC Process Each Arr ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Arrs Failed Stalled Recovery — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_TIME Age threshold: ${ARR_IMPORT_RECOVERY_AGE}hr (items newer than this are skipped)"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
# Sonarr
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
process_arr "Sonarr (Tv_Shows)" "sonarr" \
|
||||
"$HOST1_SONARR_URL" "$HOST1_SONARR_API_KEY" "v3" \
|
||||
"${HOST1_SONARR_RECOVERY:-true}"
|
||||
else
|
||||
process_arr "Sonarr (Anime_Shows)" "sonarr" \
|
||||
"$HOST2_SONARR_URL" "$HOST2_SONARR_API_KEY" "v3" \
|
||||
"${HOST2_SONARR_RECOVERY:-true}"
|
||||
fi
|
||||
|
||||
# Radarr
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
process_arr "Radarr (Movies)" "radarr" \
|
||||
"$HOST1_RADARR_URL" "$HOST1_RADARR_API_KEY" "v3" \
|
||||
"${HOST1_RADARR_RECOVERY:-true}"
|
||||
else
|
||||
process_arr "Radarr (Anime_Movies)" "radarr" \
|
||||
"$HOST2_RADARR_URL" "$HOST2_RADARR_API_KEY" "v3" \
|
||||
"${HOST2_RADARR_RECOVERY:-true}"
|
||||
fi
|
||||
|
||||
# Lidarr — HOST1 only
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
process_arr "Lidarr (Music)" "lidarr" \
|
||||
"$HOST1_LIDARR_URL" "$HOST1_LIDARR_API_KEY" "v1" \
|
||||
"${HOST1_LIDARR_RECOVERY:-true}"
|
||||
else
|
||||
info "Lidarr runs on $HOST1 only — skipping on $LOCAL_SERVER_NAME"
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ARR IMPORT RECOVERY SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo "$ICON_TRASH Actioned: $TOTAL_ACTIONED items blocklisted + searched"
|
||||
echo "$ICON_RUNNING Skipped: $TOTAL_SKIPPED items (too new or unreachable)"
|
||||
echo ""
|
||||
|
||||
for summary in "${ARR_SUMMARIES[@]}"; do
|
||||
echo " $ICON_SUMMARY $summary"
|
||||
done
|
||||
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_ACTIONED" -gt 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
notify "Arr import recovery on $(hostname) — $TOTAL_ACTIONED item(s) blocklisted and re-searched. Check arrs for new downloads." "Arr Recovery" "normal"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE — nothing to recover"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
Reference in New Issue
Block a user