461 lines
19 KiB
Bash
Executable File
461 lines
19 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ========================= Arrs Failed / Stalled Recovery =====================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Detect and recover failed imports and stalled downloads across Sonarr, Radarr,
|
|
# and Lidarr. Blocklists the bad release and triggers a re-search — hands-free
|
|
# overnight recovery.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# Four problem types detected from the arr queue API:
|
|
# importFailed — downloaded but arr couldn't import the file
|
|
# importPending — downloaded, stuck waiting to import (will not self-resolve)
|
|
# error status — serious failure not covered by the above two states
|
|
# stalled — download stuck with no connections or no progress
|
|
#
|
|
# Never touches items with state "downloading" or "imported" — safe to run anytime.
|
|
# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first.
|
|
#
|
|
# Per problem item (3-step response):
|
|
# 1. Blocklist the release — prevents re-grabbing the same bad release
|
|
# 2. Remove from queue — cleans up the failed item
|
|
# 3. Trigger new search — finds a different release automatically
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# acquire_lock — prevents concurrent runs overlapping
|
|
# jq validation — exits if jq not installed (required for JSON parsing)
|
|
# API pre-flight — checks each arr is reachable before querying queue
|
|
# Version check — check_arr_version() verifies running arr matches master.conf major
|
|
# version; exits rather than silently misoperating after upgrade
|
|
# Age threshold — skips items newer than ARR_IMPORT_RECOVERY_AGE (default 6hr)
|
|
# Silent by default — only problems produce output, clean arrs stay silent
|
|
#
|
|
# API version mapping (endpoint paths differ from major version labels):
|
|
# Sonarr v4 → /api/v3/ (v3 endpoint retained in v4)
|
|
# Radarr v6 → /api/v3/ (v3 endpoint retained in v6)
|
|
# Lidarr v3 → /api/v1/ (different from Sonarr/Radarr)
|
|
#
|
|
# ==============================================================================================
|
|
# STATE FILES
|
|
# ==============================================================================================
|
|
#
|
|
# ARR_RECOVERY_STATS — stats file written after each run (read by coffee report)
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# host*.conf
|
|
#
|
|
# HOST*_SONARR_URL / HOST*_SONARR_API_KEY / HOST*_SONARR_RECOVERY
|
|
# HOST*_RADARR_URL / HOST*_RADARR_API_KEY / HOST*_RADARR_RECOVERY
|
|
# HOST1_LIDARR_URL / HOST1_LIDARR_API_KEY / HOST1_LIDARR_RECOVERY
|
|
# All aliased by detect_hosts() — script uses unprefixed names
|
|
#
|
|
# master.conf
|
|
#
|
|
# ARR_IMPORT_RECOVERY_AGE — hours before item is eligible for recovery (default: 6)
|
|
# SONARR_VERSION_MAJOR — expected Sonarr major version (e.g. 4)
|
|
# RADARR_VERSION_MAJOR — expected Radarr major version (e.g. 6)
|
|
# LIDARR_VERSION_MAJOR — expected Lidarr major version (e.g. 3)
|
|
# ARR_RECOVERY_STATS — stats file path (read by coffee report)
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# arrs_failed_stalled_recovery.sh — normal run
|
|
# arrs_failed_stalled_recovery.sh --dry-run — show what would be actioned, no changes
|
|
# arrs_failed_stalled_recovery.sh --log — verbose output
|
|
# arrs_failed_stalled_recovery.sh --status — show config and exit
|
|
#
|
|
# Recommended schedule: 0 5 * * * (5am daily)
|
|
# Or every 6hr: 0 */6 * * * (matches ARR_IMPORT_RECOVERY_AGE default)
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
acquire_lock
|
|
|
|
# detect_hosts() sets MY_ID and aliases SONARR_*, RADARR_*, LIDARR_* vars
|
|
detect_hosts
|
|
|
|
# jq is required — not optional — for JSON parsing
|
|
if ! command -v jq >/dev/null 2>&1; then
|
|
error "jq is not installed — required for arr API JSON parsing"
|
|
error "Install: apt-get install jq or brew install jq"
|
|
notify "arrs_failed_stalled_recovery failed on $(hostname) — jq not installed" \
|
|
"Arr Recovery" "warning"
|
|
exit 1
|
|
fi
|
|
log "jq found"
|
|
|
|
validate_unraid_cmd \
|
|
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
|
"" "" \
|
|
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no items will be blocklisted or searched"
|
|
|
|
# Age threshold in seconds
|
|
AGE_THRESHOLD_SECONDS=$(( ARR_IMPORT_RECOVERY_AGE * 3600 ))
|
|
|
|
TOTAL_ACTIONED=0
|
|
TOTAL_SKIPPED=0
|
|
ARR_SUMMARIES=()
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_SYNC Sonarr: ${SONARR_URL:-not configured} (recovery: ${SONARR_RECOVERY:-true})"
|
|
echo "$ICON_SYNC Radarr: ${RADARR_URL:-not configured} (recovery: ${RADARR_RECOVERY:-true})"
|
|
echo "$ICON_SYNC Lidarr: ${LIDARR_URL:-not configured on this host} (recovery: ${LIDARR_RECOVERY:-false})"
|
|
echo "$ICON_TIME Age thresh: ${ARR_IMPORT_RECOVERY_AGE}hr"
|
|
echo "$ICON_GEAR Sonarr ver: v${SONARR_VERSION_MAJOR} expected"
|
|
echo "$ICON_GEAR Radarr ver: v${RADARR_VERSION_MAJOR} expected"
|
|
echo "$ICON_GEAR Lidarr ver: v${LIDARR_VERSION_MAJOR} expected"
|
|
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
|
|
# Check if a queue item is older than ARR_IMPORT_RECOVERY_AGE
|
|
# Returns 0 (old enough) or 1 (too new — skip)
|
|
item_is_old_enough() {
|
|
local added="$1"
|
|
[[ -z "$added" ]] && return 0 # no date = treat as old enough, safe to act
|
|
local added_epoch
|
|
added_epoch=$(date -d "$added" +%s 2>/dev/null) || return 0
|
|
local age_seconds=$(( $(date +%s) - added_epoch ))
|
|
[[ "$age_seconds" -ge "$AGE_THRESHOLD_SECONDS" ]]
|
|
}
|
|
|
|
# Query the arr queue API and return all records
|
|
# Args: url, api_key, api_version
|
|
get_queue_data() {
|
|
local url="$1" api_key="$2" api_version="$3"
|
|
curl -sf --max-time 15 \
|
|
-H "X-Api-Key: $api_key" \
|
|
"${url}/api/${api_version}/queue?page=1&pageSize=200&includeUnknownSeriesItems=true&includeUnknownArtistItems=true" \
|
|
2>/dev/null
|
|
}
|
|
|
|
# Blocklist and remove a queue item
|
|
# Args: url, api_key, api_version, queue_id
|
|
blocklist_item() {
|
|
local url="$1" api_key="$2" api_version="$3" queue_id="$4"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would blocklist queue item $queue_id"
|
|
return 0
|
|
fi
|
|
curl -sf --max-time 15 \
|
|
-X DELETE \
|
|
-H "X-Api-Key: $api_key" \
|
|
"${url}/api/${api_version}/queue/${queue_id}?removeFromClient=true&blocklist=true&skipRedownload=false" \
|
|
>/dev/null 2>&1
|
|
}
|
|
|
|
# Trigger a new search for the media item
|
|
# Args: url, api_key, api_version, arr_type, media_id
|
|
trigger_search() {
|
|
local url="$1" api_key="$2" api_version="$3" arr_type="$4" media_id="$5"
|
|
local command body
|
|
case "$arr_type" in
|
|
sonarr) command="EpisodeSearch"; body="{\"name\":\"EpisodeSearch\",\"episodeIds\":[$media_id]}" ;;
|
|
radarr) command="MoviesSearch"; body="{\"name\":\"MoviesSearch\",\"movieIds\":[$media_id]}" ;;
|
|
lidarr) command="AlbumSearch"; body="{\"name\":\"AlbumSearch\",\"albumIds\":[$media_id]}" ;;
|
|
esac
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would trigger $command for media ID $media_id"
|
|
return 0
|
|
fi
|
|
curl -sf --max-time 15 \
|
|
-X POST \
|
|
-H "X-Api-Key: $api_key" \
|
|
-H "Content-Type: application/json" \
|
|
-d "$body" \
|
|
"${url}/api/${api_version}/command" \
|
|
>/dev/null 2>&1
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ── PROCESS AN ARR ────────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
# Args: display_name, arr_type, url, api_key, api_version, enabled,
|
|
# version_major, version_api_prefix
|
|
#
|
|
# Exits cleanly if disabled.
|
|
# Checks API reachability and version before touching queue.
|
|
# Processes each problem item: blocklist + trigger new search.
|
|
# Silent when clean — only warns when problems found or actioned.
|
|
|
|
process_arr() {
|
|
local arr_name="$1"
|
|
local arr_type="$2"
|
|
local url="$3"
|
|
local api_key="$4"
|
|
local api_version="$5"
|
|
local enabled="$6"
|
|
local version_major="$7"
|
|
local version_api_prefix="$8"
|
|
|
|
local actioned=0 skipped_new=0
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC $arr_name ━━━"
|
|
|
|
# Disabled — skip cleanly
|
|
if [[ "$enabled" != "true" ]]; then
|
|
log "$arr_name recovery disabled — skipping"
|
|
ARR_SUMMARIES+=("$arr_name: disabled")
|
|
return
|
|
fi
|
|
|
|
# URL not configured on this host — skip cleanly
|
|
if [[ -z "$url" ]]; then
|
|
log "$arr_name not configured on $MY_ID — skipping"
|
|
ARR_SUMMARIES+=("$arr_name: not configured on $MY_ID")
|
|
return
|
|
fi
|
|
|
|
# API reachability
|
|
if ! check_api "$url" "$arr_name" 10; then
|
|
warn "$arr_name API unreachable — skipping"
|
|
ARR_SUMMARIES+=("$arr_name: unreachable")
|
|
return
|
|
fi
|
|
|
|
# Version check — exit if API structure may have changed
|
|
if ! check_arr_version "$url" "$api_key" "$version_api_prefix" \
|
|
"$version_major" "$arr_name"; then
|
|
ARR_SUMMARIES+=("$arr_name: version mismatch — skipped")
|
|
return
|
|
fi
|
|
|
|
# Fetch queue
|
|
local queue_data
|
|
queue_data=$(get_queue_data "$url" "$api_key" "$api_version")
|
|
if [[ -z "$queue_data" ]]; then
|
|
warn "$arr_name — could not retrieve queue data"
|
|
ARR_SUMMARIES+=("$arr_name: queue fetch failed")
|
|
return
|
|
fi
|
|
|
|
local total_records
|
|
total_records=$(echo "$queue_data" | jq '.totalRecords // 0' 2>/dev/null)
|
|
log "$arr_name queue: $total_records total items"
|
|
|
|
# Filter for problem items — never touch downloading or imported
|
|
local problem_items
|
|
problem_items=$(echo "$queue_data" | jq -c '
|
|
.records // [] |
|
|
.[] |
|
|
select(
|
|
.trackedDownloadState != "downloading" and
|
|
.trackedDownloadState != "imported" and
|
|
(
|
|
.trackedDownloadState == "importFailed" or
|
|
.trackedDownloadState == "importPending" or
|
|
.trackedDownloadStatus == "error" or
|
|
(.status == "warning" and (
|
|
(.errorMessage // "" | ascii_downcase | contains("stalled")) or
|
|
(.statusMessages // [] | .[] | .messages // [] | .[] |
|
|
ascii_downcase | contains("stalled"))
|
|
))
|
|
)
|
|
)
|
|
' 2>/dev/null)
|
|
|
|
if [[ -z "$problem_items" ]]; then
|
|
log "$arr_name — clean ✅ no failed imports or stalled downloads"
|
|
ARR_SUMMARIES+=("$arr_name: clean ✅")
|
|
return
|
|
fi
|
|
|
|
local problem_count
|
|
problem_count=$(echo "$problem_items" | wc -l)
|
|
warn "$arr_name — found $problem_count problem item(s)"
|
|
|
|
# Process each problem item
|
|
while IFS= read -r item; do
|
|
[[ -z "$item" ]] && continue
|
|
|
|
local queue_id title added tracked_state tracked_status problem_type media_id
|
|
|
|
queue_id=$(echo "$item" | jq -r '.id // empty' 2>/dev/null)
|
|
title=$(echo "$item" | jq -r '.title // "Unknown"' 2>/dev/null)
|
|
added=$(echo "$item" | jq -r '.added // empty' 2>/dev/null)
|
|
tracked_state=$(echo "$item" | jq -r '.trackedDownloadState // ""' 2>/dev/null)
|
|
tracked_status=$(echo "$item" | jq -r '.trackedDownloadStatus // ""' 2>/dev/null)
|
|
|
|
# Human-readable problem type
|
|
case "$tracked_state" in
|
|
importFailed) problem_type="import failed" ;;
|
|
importPending) problem_type="import pending/stuck" ;;
|
|
*)
|
|
[[ "$tracked_status" == "error" ]] && \
|
|
problem_type="error" || problem_type="stalled"
|
|
;;
|
|
esac
|
|
|
|
# Media ID for search trigger
|
|
case "$arr_type" in
|
|
sonarr) media_id=$(echo "$item" | jq -r '.episodeId // .episode.id // empty' 2>/dev/null) ;;
|
|
radarr) media_id=$(echo "$item" | jq -r '.movieId // .movie.id // empty' 2>/dev/null) ;;
|
|
lidarr) media_id=$(echo "$item" | jq -r '.albumId // .album.id // empty' 2>/dev/null) ;;
|
|
esac
|
|
|
|
[[ -z "$queue_id" ]] && continue
|
|
|
|
# Age check — skip items that are too new to have self-resolved
|
|
if ! item_is_old_enough "$added"; then
|
|
log " Skipping (too new < ${ARR_IMPORT_RECOVERY_AGE}hr): $title"
|
|
(( skipped_new++ ))
|
|
(( TOTAL_SKIPPED++ ))
|
|
continue
|
|
fi
|
|
|
|
warn " $ICON_TRASH $problem_type — $title"
|
|
|
|
# Step 1: Blocklist + remove from queue
|
|
if ! blocklist_item "$url" "$api_key" "$api_version" "$queue_id"; then
|
|
warn " Failed to blocklist: $title"
|
|
(( TOTAL_SKIPPED++ ))
|
|
continue
|
|
fi
|
|
log " Blocklisted: $queue_id"
|
|
|
|
# Step 2: Trigger new search
|
|
if [[ -n "$media_id" ]]; then
|
|
if trigger_search "$url" "$api_key" "$api_version" "$arr_type" "$media_id"; then
|
|
log " New search triggered: $title"
|
|
else
|
|
warn " Blocklisted but search trigger failed: $title"
|
|
fi
|
|
else
|
|
warn " Blocklisted but no media ID found — search not triggered: $title"
|
|
fi
|
|
|
|
(( actioned++ ))
|
|
(( TOTAL_ACTIONED++ ))
|
|
|
|
done <<< "$problem_items"
|
|
|
|
if [[ "$actioned" -gt 0 ]]; then
|
|
warn "$arr_name — actioned: $actioned | skipped (too new): $skipped_new"
|
|
else
|
|
log "$arr_name — nothing actioned | skipped (too new): $skipped_new"
|
|
fi
|
|
|
|
ARR_SUMMARIES+=("$arr_name: actioned $actioned | too new $skipped_new")
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Process Each Arr ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Arrs Failed/Stalled Recovery — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
|
log "Age threshold: ${ARR_IMPORT_RECOVERY_AGE}hr"
|
|
|
|
START=$(date +%s)
|
|
|
|
# Sonarr — uses aliased vars set by detect_hosts()
|
|
process_arr \
|
|
"Sonarr" \
|
|
"sonarr" \
|
|
"${SONARR_URL:-}" \
|
|
"${SONARR_API_KEY:-}" \
|
|
"v3" \
|
|
"${SONARR_RECOVERY:-true}" \
|
|
"${SONARR_VERSION_MAJOR:-4}" \
|
|
"v3"
|
|
|
|
# Radarr — uses aliased vars set by detect_hosts()
|
|
process_arr \
|
|
"Radarr" \
|
|
"radarr" \
|
|
"${RADARR_URL:-}" \
|
|
"${RADARR_API_KEY:-}" \
|
|
"v3" \
|
|
"${RADARR_RECOVERY:-true}" \
|
|
"${RADARR_VERSION_MAJOR:-6}" \
|
|
"v3"
|
|
|
|
# Lidarr — HOST1 only, LIDARR_URL empty on HOST2 → exits cleanly via "not configured" guard
|
|
process_arr \
|
|
"Lidarr" \
|
|
"lidarr" \
|
|
"${LIDARR_URL:-}" \
|
|
"${LIDARR_API_KEY:-}" \
|
|
"v1" \
|
|
"${LIDARR_RECOVERY:-false}" \
|
|
"${LIDARR_VERSION_MAJOR:-3}" \
|
|
"v1"
|
|
|
|
END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY ARR RECOVERY SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
echo "$ICON_TRASH Actioned: $TOTAL_ACTIONED items blocklisted + searched"
|
|
echo "$ICON_SKIP Skipped: $TOTAL_SKIPPED items (too new)"
|
|
echo ""
|
|
for summary in "${ARR_SUMMARIES[@]}"; do
|
|
echo " $ICON_SUMMARY $summary"
|
|
done
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no changes made"
|
|
elif [[ "$TOTAL_ACTIONED" -gt 0 ]]; then
|
|
warn "$ICON_DONE Done — $TOTAL_ACTIONED item(s) blocklisted and re-searched"
|
|
notify "Arr recovery on $(hostname) — $TOTAL_ACTIONED item(s) blocklisted and re-searched" \
|
|
"Arr Recovery" "warning"
|
|
else
|
|
echo "$ICON_DONE Done — nothing to recover (all arrs clean)"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
# Write stats for sunday_morning_coffee_report.sh
|
|
if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_RECOVERY_STATS:-}" ]]; then
|
|
echo "$(date '+%Y-%m-%d')|$(date '+%H:%M')|${TOTAL_ACTIONED}|${TOTAL_SKIPPED}" \
|
|
>> "$ARR_RECOVERY_STATS" 2>/dev/null || true
|
|
fi
|
|
|
|
exit 0 |