360 lines
14 KiB
Bash
360 lines
14 KiB
Bash
#!/bin/bash
|
|
# -----------------------------------------------------------------------------------------------
|
|
# --------------------------------- Arrs Failed Stalled Recovery ----------------------------------------
|
|
# -----------------------------------------------------------------------------------------------
|
|
# Automatically detects and recovers from failed imports and stalled downloads
|
|
# across Sonarr, Radarr, and Lidarr. Blocklists the bad release and triggers
|
|
# a new search — hands free recovery while you sleep.
|
|
#
|
|
# Targets four problem types from the queue API:
|
|
# importFailed — downloaded successfully but arr couldn't import the file
|
|
# importPending — downloaded, stuck waiting to import (won't self-resolve)
|
|
# error status — serious failure not covered by importFailed/importPending
|
|
# stalled — download stuck with no connections or no progress
|
|
#
|
|
# Never touches items with state "downloading" or "imported" — safe to run anytime.
|
|
#
|
|
# Action per problem item:
|
|
# 1. Blocklist the release — prevents re-grabbing the same bad release
|
|
# 2. Remove from queue — cleans up the failed item
|
|
# 3. Trigger new search — finds a different release automatically
|
|
#
|
|
# Age threshold (ARR_IMPORT_RECOVERY_AGE):
|
|
# Items newer than threshold are skipped — gives the arr time to retry on its own
|
|
# Items older than threshold have not self-resolved — safe to intervene
|
|
# Default: 12 hours
|
|
#
|
|
# API versions:
|
|
# Sonarr v4 → /api/v3/ (v3 endpoint retained in v4)
|
|
# Radarr v6 → /api/v3/ (v3 endpoint retained in v6)
|
|
# Lidarr v3 → /api/v1/ (different from Sonarr/Radarr)
|
|
#
|
|
# Per-arr enable/disable toggles in Master.conf.
|
|
# Lidarr runs on HOST1 only — exits cleanly on HOST2.
|
|
# detect_hosts() selects correct URL and API key per server at runtime.
|
|
#
|
|
# Recommended schedule: 0 5 * * * (5am daily)
|
|
# Supports --dry-run to show what would be actioned without making changes.
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../Master.conf"
|
|
source "$SCRIPT_DIR/../common.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_GEAR Setup ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
|
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
success "Running as root"
|
|
|
|
acquire_lock
|
|
|
|
detect_hosts
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no items will be blocklisted or searched"
|
|
|
|
# Age threshold in seconds for comparison
|
|
AGE_THRESHOLD_SECONDS=$(( ARR_IMPORT_RECOVERY_AGE * 3600 ))
|
|
|
|
# Tracking totals
|
|
TOTAL_ACTIONED=0
|
|
TOTAL_SKIPPED=0
|
|
ARR_SUMMARIES=()
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# CORE FUNCTIONS
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
# Check if a queue item is older than ARR_IMPORT_RECOVERY_AGE
|
|
item_is_old_enough() {
|
|
local added="$1"
|
|
[[ -z "$added" ]] && return 0 # no date = treat as old enough
|
|
local added_epoch
|
|
added_epoch=$(date -d "$added" +%s 2>/dev/null) || return 0
|
|
local now_epoch
|
|
now_epoch=$(date +%s)
|
|
local age_seconds=$(( now_epoch - added_epoch ))
|
|
[[ "$age_seconds" -ge "$AGE_THRESHOLD_SECONDS" ]]
|
|
}
|
|
|
|
# Query the arr queue and return failed/stalled items
|
|
# Args: url, api_key, api_version
|
|
get_problem_items() {
|
|
local url="$1" api_key="$2" api_version="$3"
|
|
curl -sf --max-time 15 \
|
|
-H "X-Api-Key: $api_key" \
|
|
"${url}/api/${api_version}/queue?page=1&pageSize=200&includeUnknownSeriesItems=true&includeUnknownArtistItems=true" \
|
|
2>/dev/null
|
|
}
|
|
|
|
# Blocklist and remove item from queue
|
|
# Args: url, api_key, api_version, queue_id
|
|
blocklist_item() {
|
|
local url="$1" api_key="$2" api_version="$3" queue_id="$4"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would blocklist queue item $queue_id"
|
|
return 0
|
|
fi
|
|
curl -sf --max-time 15 \
|
|
-X DELETE \
|
|
-H "X-Api-Key: $api_key" \
|
|
"${url}/api/${api_version}/queue/${queue_id}?removeFromClient=true&blocklist=true&skipRedownload=false" \
|
|
>/dev/null 2>&1
|
|
}
|
|
|
|
# Trigger new search
|
|
# Args: url, api_key, api_version, arr_type, media_id
|
|
trigger_search() {
|
|
local url="$1" api_key="$2" api_version="$3" arr_type="$4" media_id="$5"
|
|
local command body
|
|
case "$arr_type" in
|
|
sonarr) command="EpisodeSearch"; body="{\"name\":\"EpisodeSearch\",\"episodeIds\":[$media_id]}" ;;
|
|
radarr) command="MoviesSearch"; body="{\"name\":\"MoviesSearch\",\"movieIds\":[$media_id]}" ;;
|
|
lidarr) command="AlbumSearch"; body="{\"name\":\"AlbumSearch\",\"albumIds\":[$media_id]}" ;;
|
|
esac
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would trigger $command for ID $media_id"
|
|
return 0
|
|
fi
|
|
curl -sf --max-time 15 \
|
|
-X POST \
|
|
-H "X-Api-Key: $api_key" \
|
|
-H "Content-Type: application/json" \
|
|
-d "$body" \
|
|
"${url}/api/${api_version}/command" \
|
|
>/dev/null 2>&1
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# PROCESS AN ARR
|
|
# Args: arr_name, arr_type, url, api_key, api_version, enabled
|
|
# -----------------------------------------------------------------------------------------------
|
|
process_arr() {
|
|
local arr_name="$1" arr_type="$2" url="$3" api_key="$4" api_version="$5" enabled="$6"
|
|
local actioned=0 skipped=0 too_new=0
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC $arr_name ━━━"
|
|
|
|
if [[ "$enabled" != "true" ]]; then
|
|
info "$arr_name recovery disabled — skipping"
|
|
ARR_SUMMARIES+=("$arr_name: disabled")
|
|
return
|
|
fi
|
|
|
|
# API pre-flight
|
|
if ! check_api "$url" "$arr_name" 10; then
|
|
warn "$arr_name API unreachable — skipping"
|
|
ARR_SUMMARIES+=("$arr_name: unreachable")
|
|
return
|
|
fi
|
|
|
|
# Get queue
|
|
local queue_data
|
|
queue_data=$(get_problem_items "$url" "$api_key" "$api_version")
|
|
if [[ -z "$queue_data" ]] || ! command -v jq >/dev/null 2>&1; then
|
|
warn "$arr_name — could not retrieve queue data"
|
|
ARR_SUMMARIES+=("$arr_name: queue fetch failed")
|
|
return
|
|
fi
|
|
|
|
local total_records
|
|
total_records=$(echo "$queue_data" | jq '.totalRecords // 0' 2>/dev/null)
|
|
info "Queue: $total_records total items"
|
|
|
|
# Filter for problem items — never touch downloading or imported
|
|
# importFailed = tried to import, actually failed
|
|
# importPending = downloaded, stuck waiting to import (won't self-resolve)
|
|
# error status = serious failure not covered by above states
|
|
# stalled = download stuck with no connections or progress
|
|
local problem_items
|
|
problem_items=$(echo "$queue_data" | jq -c '
|
|
.records // [] |
|
|
.[] |
|
|
select(
|
|
.trackedDownloadState != "downloading" and
|
|
.trackedDownloadState != "imported" and
|
|
(
|
|
.trackedDownloadState == "importFailed" or
|
|
.trackedDownloadState == "importPending" or
|
|
.trackedDownloadStatus == "error" or
|
|
(.status == "warning" and (
|
|
(.errorMessage // "" | ascii_downcase | contains("stalled")) or
|
|
(.statusMessages // [] | .[] | .messages // [] | .[] | ascii_downcase | contains("stalled"))
|
|
))
|
|
)
|
|
)
|
|
' 2>/dev/null)
|
|
|
|
if [[ -z "$problem_items" ]]; then
|
|
success "$arr_name — no failed imports or stalled downloads found"
|
|
ARR_SUMMARIES+=("$arr_name: clean ✅")
|
|
return
|
|
fi
|
|
|
|
local problem_count
|
|
problem_count=$(echo "$problem_items" | wc -l)
|
|
info "Found $problem_count problem item(s)"
|
|
|
|
# Process each problem item
|
|
while IFS= read -r item; do
|
|
[[ -z "$item" ]] && continue
|
|
|
|
local queue_id title added problem_type media_id
|
|
queue_id=$(echo "$item" | jq -r '.id // empty' 2>/dev/null)
|
|
title=$(echo "$item" | jq -r '.title // "Unknown"' 2>/dev/null)
|
|
added=$(echo "$item" | jq -r '.added // empty' 2>/dev/null)
|
|
|
|
# Determine problem type for display
|
|
local tracked_state tracked_status
|
|
tracked_state=$(echo "$item" | jq -r '.trackedDownloadState // ""' 2>/dev/null)
|
|
tracked_status=$(echo "$item" | jq -r '.trackedDownloadStatus // ""' 2>/dev/null)
|
|
case "$tracked_state" in
|
|
importFailed) problem_type="import failed" ;;
|
|
importPending) problem_type="import pending/stuck" ;;
|
|
*)
|
|
if [[ "$tracked_status" == "error" ]]; then
|
|
problem_type="error"
|
|
else
|
|
problem_type="stalled"
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
# Get media ID for search trigger (episode, movie, or album)
|
|
case "$arr_type" in
|
|
sonarr) media_id=$(echo "$item" | jq -r '.episodeId // .episode.id // empty' 2>/dev/null) ;;
|
|
radarr) media_id=$(echo "$item" | jq -r '.movieId // .movie.id // empty' 2>/dev/null) ;;
|
|
lidarr) media_id=$(echo "$item" | jq -r '.albumId // .album.id // empty' 2>/dev/null) ;;
|
|
esac
|
|
|
|
[[ -z "$queue_id" ]] && continue
|
|
|
|
# Age check
|
|
if ! item_is_old_enough "$added"; then
|
|
info " $ICON_TIME Skipping (too new < ${ARR_IMPORT_RECOVERY_AGE}hr): $title"
|
|
((too_new++))
|
|
((skipped++))
|
|
continue
|
|
fi
|
|
|
|
info " $ICON_TRASH $problem_type: $title"
|
|
|
|
# Blocklist + remove
|
|
if blocklist_item "$url" "$api_key" "$api_version" "$queue_id"; then
|
|
log " Blocklisted queue item: $queue_id"
|
|
else
|
|
warn " Failed to blocklist: $title"
|
|
((skipped++))
|
|
continue
|
|
fi
|
|
|
|
# Trigger new search if we have a media ID
|
|
if [[ -n "$media_id" ]]; then
|
|
if trigger_search "$url" "$api_key" "$api_version" "$arr_type" "$media_id"; then
|
|
log " New search triggered for: $title"
|
|
((actioned++))
|
|
((TOTAL_ACTIONED++))
|
|
else
|
|
warn " Blocklisted but search trigger failed: $title"
|
|
((actioned++))
|
|
((TOTAL_ACTIONED++))
|
|
fi
|
|
else
|
|
warn " Blocklisted but no media ID found for search: $title"
|
|
((actioned++))
|
|
((TOTAL_ACTIONED++))
|
|
fi
|
|
|
|
done <<< "$problem_items"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
info "$arr_name — dry run complete"
|
|
else
|
|
success "$arr_name — actioned: $actioned | skipped (too new): $too_new"
|
|
fi
|
|
|
|
ARR_SUMMARIES+=("$arr_name: actioned $actioned | too new $too_new")
|
|
TOTAL_SKIPPED=$(( TOTAL_SKIPPED + skipped ))
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SYNC Process Each Arr ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Arrs Failed Stalled Recovery — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_TIME Age threshold: ${ARR_IMPORT_RECOVERY_AGE}hr (items newer than this are skipped)"
|
|
echo ""
|
|
|
|
START=$(date +%s)
|
|
|
|
# Sonarr
|
|
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
|
process_arr "Sonarr (Tv_Shows)" "sonarr" \
|
|
"$HOST1_SONARR_URL" "$HOST1_SONARR_API_KEY" "v3" \
|
|
"${HOST1_SONARR_RECOVERY:-true}"
|
|
else
|
|
process_arr "Sonarr (Anime_Shows)" "sonarr" \
|
|
"$HOST2_SONARR_URL" "$HOST2_SONARR_API_KEY" "v3" \
|
|
"${HOST2_SONARR_RECOVERY:-true}"
|
|
fi
|
|
|
|
# Radarr
|
|
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
|
process_arr "Radarr (Movies)" "radarr" \
|
|
"$HOST1_RADARR_URL" "$HOST1_RADARR_API_KEY" "v3" \
|
|
"${HOST1_RADARR_RECOVERY:-true}"
|
|
else
|
|
process_arr "Radarr (Anime_Movies)" "radarr" \
|
|
"$HOST2_RADARR_URL" "$HOST2_RADARR_API_KEY" "v3" \
|
|
"${HOST2_RADARR_RECOVERY:-true}"
|
|
fi
|
|
|
|
# Lidarr — HOST1 only
|
|
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
|
process_arr "Lidarr (Music)" "lidarr" \
|
|
"$HOST1_LIDARR_URL" "$HOST1_LIDARR_API_KEY" "v1" \
|
|
"${HOST1_LIDARR_RECOVERY:-true}"
|
|
else
|
|
info "Lidarr runs on $HOST1 only — skipping on $LOCAL_SERVER_NAME"
|
|
fi
|
|
|
|
END=$(date +%s)
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SUMMARY Summary ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY ARR IMPORT RECOVERY SUMMARY ━━━━━"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
echo "$ICON_TRASH Actioned: $TOTAL_ACTIONED items blocklisted + searched"
|
|
echo "$ICON_RUNNING Skipped: $TOTAL_SKIPPED items (too new or unreachable)"
|
|
echo ""
|
|
echo " Problem types detected: importFailed | importPending | error | stalled"
|
|
echo ""
|
|
|
|
for summary in "${ARR_SUMMARIES[@]}"; do
|
|
echo " $ICON_SUMMARY $summary"
|
|
done
|
|
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
|
elif [[ "$TOTAL_ACTIONED" -gt 0 ]]; then
|
|
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
|
notify "Arr import recovery on $(hostname) — $TOTAL_ACTIONED item(s) blocklisted and re-searched. Check arrs for new downloads." "Arr Recovery" "normal"
|
|
else
|
|
echo "$ICON_DONE Status: $ICON_SUCCESS DONE — nothing to recover"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" |