Gmer4Lfe
f30863f452
page=1&pageSize=200 silently truncated anything past record 200. Sonarr's queue currently runs 1700+ during a large search campaign, which pushed every importBlocked/warning item past page 1 — the script logged 'clean' every run while 52 stuck imports sat completely unseen despite yesterday's importBlocked fix matching them correctly once actually queried.
574 lines
25 KiB
Bash
Executable File
574 lines
25 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ========================= Arrs Failed / Stalled Recovery =====================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Detect and recover failed imports and stalled downloads across Sonarr, Radarr,
|
|
# and Lidarr. Blocklists the bad release and triggers a re-search — hands-free
|
|
# overnight recovery.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# Five problem types detected from the arr queue API:
|
|
# importFailed — downloaded but arr couldn't import the file
|
|
# importPending — downloaded, stuck waiting to import (will not self-resolve)
|
|
# importBlocked — downloaded, but arr matched the release to the wrong media
|
|
# by grab-history ID instead of by title and refuses to import
|
|
# (permanent block, never self-resolves — same handling as
|
|
# importFailed since most real-world cases are junk/duplicate
|
|
# releases; the rare case where the file is actually good but
|
|
# mis-parsed will just get re-searched instead of manually
|
|
# imported, an acceptable tradeoff for hands-free operation)
|
|
# error status — serious failure not covered by the above two states
|
|
# stalled — download stuck with no connections or no progress
|
|
#
|
|
# Never touches items with state "downloading" or "imported" — safe to run anytime.
|
|
# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first.
|
|
#
|
|
# Per problem item (3-step response):
|
|
# 1. Blocklist the release — prevents re-grabbing the same bad release
|
|
# 2. Remove from queue — cleans up the failed item
|
|
# 3. Trigger new search — finds a different release automatically
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Hands-Free Recovery
|
|
# The script completes the full recovery cycle autonomously — blocklist, remove,
|
|
# re-search. No operator decision required. A failed import at midnight resolves
|
|
# itself before morning without any intervention.
|
|
#
|
|
# Age Gate Before Action
|
|
# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped. Arrs have their own
|
|
# retry logic — acting immediately would race against it. The age gate gives
|
|
# the arr time to self-resolve before this script escalates.
|
|
#
|
|
# Blocklist First
|
|
# The bad release is blocklisted before removal and re-search. Without this,
|
|
# the re-search can re-grab the same release that just failed.
|
|
#
|
|
# Circuit Breaker Per Media Item
|
|
# Some items can never resolve via blind retry — e.g. an album missing 1-2
|
|
# tracks where every available release is a different edition that doesn't
|
|
# match. Without a limit, the same media ID gets blocklisted + re-searched
|
|
# forever, every run, burning bandwidth and indexer queries for nothing.
|
|
# After ARR_RECOVERY_MAX_ATTEMPTS consecutive failures for the same
|
|
# (arr_type, media_id), the item is still blocklisted/cleaned from the queue
|
|
# but search is no longer auto-triggered — it's flagged chronic and left for
|
|
# manual review instead.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# acquire_lock — prevents concurrent runs overlapping
|
|
# jq validation — exits if jq not installed (required for JSON parsing)
|
|
# API pre-flight — checks each arr is reachable before querying queue
|
|
# Version check — check_arr_version() verifies running arr matches master.conf major
|
|
# version; exits rather than silently misoperating after upgrade
|
|
# Age threshold — skips items newer than ARR_IMPORT_RECOVERY_AGE (default 6hr)
|
|
# Silent by default — only problems produce output, clean arrs stay silent
|
|
#
|
|
# API version mapping (endpoint paths differ from major version labels):
|
|
# Sonarr v4 → /api/v3/ (v3 endpoint retained in v4)
|
|
# Radarr v6 → /api/v3/ (v3 endpoint retained in v6)
|
|
# Lidarr v3 → /api/v1/ (different from Sonarr/Radarr)
|
|
#
|
|
# ==============================================================================================
|
|
# STATE FILES
|
|
# ==============================================================================================
|
|
#
|
|
# ARR_RECOVERY_STATS — stats file written after each run (read by coffee report)
|
|
# ARR_RECOVERY_FAILURE_COUNTS — per (arr_type, media_id) consecutive-failure counts,
|
|
# persists across runs so the circuit breaker survives restarts.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# host*.conf
|
|
#
|
|
# HOST*_SONARR_URL / HOST*_SONARR_API_KEY / HOST*_SONARR_RECOVERY
|
|
# HOST*_RADARR_URL / HOST*_RADARR_API_KEY / HOST*_RADARR_RECOVERY
|
|
# HOST1_LIDARR_URL / HOST1_LIDARR_API_KEY / HOST1_LIDARR_RECOVERY
|
|
# All aliased by detect_hosts() — script uses unprefixed names
|
|
#
|
|
# master.conf
|
|
#
|
|
# ARR_IMPORT_RECOVERY_AGE — hours before item is eligible for recovery (default: 6)
|
|
# ARR_RECOVERY_MAX_ATTEMPTS — consecutive failures before an item is flagged chronic
|
|
# and auto re-search stops (default: 3)
|
|
# SONARR_VERSION_MAJOR — expected Sonarr major version (e.g. 4)
|
|
# RADARR_VERSION_MAJOR — expected Radarr major version (e.g. 6)
|
|
# LIDARR_VERSION_MAJOR — expected Lidarr major version (e.g. 3)
|
|
# ARR_RECOVERY_STATS — stats file path (read by coffee report)
|
|
# ARR_RECOVERY_FAILURE_COUNTS — failure-count state file path
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# arrs_failed_stalled_recovery.sh — normal run
|
|
# arrs_failed_stalled_recovery.sh --dry-run — show what would be actioned, no changes
|
|
# arrs_failed_stalled_recovery.sh --log — verbose output
|
|
# arrs_failed_stalled_recovery.sh --status — show config and exit
|
|
#
|
|
# Recommended schedule: 0 5 * * * (5am daily)
|
|
# Or every 6hr: 0 */6 * * * (matches ARR_IMPORT_RECOVERY_AGE default)
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
acquire_lock
|
|
|
|
# detect_hosts() sets MY_ID and aliases SONARR_*, RADARR_*, LIDARR_* vars
|
|
detect_hosts
|
|
|
|
# jq is required — not optional — for JSON parsing
|
|
if ! command -v jq >/dev/null 2>&1; then
|
|
error "jq is not installed — required for arr API JSON parsing"
|
|
error "Install: apt-get install jq or brew install jq"
|
|
notify "arrs_failed_stalled_recovery failed on $(hostname) — jq not installed" \
|
|
"Arr Recovery" "warning"
|
|
exit 1
|
|
fi
|
|
log "jq found"
|
|
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no items will be blocklisted or searched"
|
|
|
|
# Age threshold in seconds
|
|
AGE_THRESHOLD_SECONDS=$(( ARR_IMPORT_RECOVERY_AGE * 3600 ))
|
|
ARR_RECOVERY_MAX_ATTEMPTS="${ARR_RECOVERY_MAX_ATTEMPTS:-3}"
|
|
ARR_RECOVERY_FAILURE_COUNTS="${ARR_RECOVERY_FAILURE_COUNTS:-$DATA_DIR/arr_recovery_failure_counts.db}"
|
|
|
|
log "$ICON_GEAR Config: age-threshold=${ARR_IMPORT_RECOVERY_AGE}hr max-attempts=${ARR_RECOVERY_MAX_ATTEMPTS} sonarr-v${SONARR_VERSION_MAJOR} radarr-v${RADARR_VERSION_MAJOR} lidarr-v${LIDARR_VERSION_MAJOR:-?}"
|
|
|
|
# Load persisted per-item failure counts — key is "arr_type:media_id"
|
|
declare -A FAILURE_COUNTS
|
|
if [[ -f "$ARR_RECOVERY_FAILURE_COUNTS" ]]; then
|
|
while IFS='|' read -r _key _count; do
|
|
[[ -z "$_key" ]] && continue
|
|
FAILURE_COUNTS["$_key"]="$_count"
|
|
done < "$ARR_RECOVERY_FAILURE_COUNTS"
|
|
fi
|
|
|
|
TOTAL_ACTIONED=0
|
|
TOTAL_SKIPPED=0
|
|
TOTAL_CHRONIC=0
|
|
ARR_SUMMARIES=()
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_SYNC Sonarr: ${SONARR_URL:-not configured} (recovery: ${SONARR_RECOVERY:-true})"
|
|
echo "$ICON_SYNC Radarr: ${RADARR_URL:-not configured} (recovery: ${RADARR_RECOVERY:-true})"
|
|
echo "$ICON_SYNC Lidarr: ${LIDARR_URL:-not configured on this host} (recovery: ${LIDARR_RECOVERY:-false})"
|
|
echo "$ICON_TIME Age thresh: ${ARR_IMPORT_RECOVERY_AGE}hr"
|
|
echo "$ICON_GEAR Max attempts: ${ARR_RECOVERY_MAX_ATTEMPTS:-3} (chronic after this many)"
|
|
echo "$ICON_GEAR Sonarr ver: v${SONARR_VERSION_MAJOR} expected"
|
|
echo "$ICON_GEAR Radarr ver: v${RADARR_VERSION_MAJOR} expected"
|
|
echo "$ICON_GEAR Lidarr ver: v${LIDARR_VERSION_MAJOR} expected"
|
|
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
|
|
# Check if a queue item is older than ARR_IMPORT_RECOVERY_AGE
|
|
# Returns 0 (old enough) or 1 (too new — skip)
|
|
item_is_old_enough() {
|
|
local added="$1"
|
|
[[ -z "$added" ]] && return 0 # no date = treat as old enough, safe to act
|
|
local added_epoch
|
|
added_epoch=$(date -d "$added" +%s 2>/dev/null) || return 0
|
|
local age_seconds=$(( $(date +%s) - added_epoch ))
|
|
[[ "$age_seconds" -ge "$AGE_THRESHOLD_SECONDS" ]]
|
|
}
|
|
|
|
# Query the arr queue API and return all records, paginated.
|
|
# A single page=1&pageSize=200 request silently misses everything past record 200 —
|
|
# on a busy Sonarr instance the queue can run into the thousands (e.g. a large
|
|
# missing-episode search campaign), which pushed every importBlocked/warning item
|
|
# past page 1 and made this whole script blind to them despite matching correctly.
|
|
# Args: url, api_key, api_version
|
|
get_queue_data() {
|
|
local url="$1" api_key="$2" api_version="$3"
|
|
local page=1 page_size=250 max_pages=50
|
|
local page_data page_count
|
|
# Accumulate pages as files rather than growing a shell variable — on a large
|
|
# queue (thousands of records) passing the combined JSON through --argjson
|
|
# blows past ARG_MAX ("Argument list too long"). jq -s reads files instead.
|
|
local tmp_dir
|
|
tmp_dir=$(mktemp -d)
|
|
trap 'rm -rf "$tmp_dir"' RETURN
|
|
|
|
while [[ "$page" -le "$max_pages" ]]; do
|
|
page_data=$(curl -sf --max-time 15 \
|
|
-H "X-Api-Key: $api_key" \
|
|
"${url}/api/${api_version}/queue?page=${page}&pageSize=${page_size}&includeUnknownSeriesItems=true&includeUnknownArtistItems=true" \
|
|
2>/dev/null)
|
|
[[ -z "$page_data" ]] && break
|
|
|
|
page_count=$(echo "$page_data" | jq '.records // [] | length' 2>/dev/null)
|
|
[[ -z "$page_count" || "$page_count" -eq 0 ]] && break
|
|
|
|
echo "$page_data" | jq -c '.records // []' > "$tmp_dir/page_${page}.json"
|
|
|
|
[[ "$page_count" -lt "$page_size" ]] && break
|
|
(( page++ ))
|
|
done
|
|
|
|
jq -c -s '{totalRecords: ([.[][]] | length), records: [.[][]]}' "$tmp_dir"/page_*.json 2>/dev/null \
|
|
|| echo '{"totalRecords":0,"records":[]}'
|
|
}
|
|
|
|
# Blocklist and remove a queue item
|
|
# Args: url, api_key, api_version, queue_id
|
|
blocklist_item() {
|
|
local url="$1" api_key="$2" api_version="$3" queue_id="$4"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would blocklist queue item $queue_id"
|
|
return 0
|
|
fi
|
|
curl -sf --max-time 15 \
|
|
-X DELETE \
|
|
-H "X-Api-Key: $api_key" \
|
|
"${url}/api/${api_version}/queue/${queue_id}?removeFromClient=true&blocklist=true&skipRedownload=false" \
|
|
>/dev/null 2>&1
|
|
}
|
|
|
|
# Trigger a new search for the media item
|
|
# Args: url, api_key, api_version, arr_type, media_id
|
|
trigger_search() {
|
|
local url="$1" api_key="$2" api_version="$3" arr_type="$4" media_id="$5"
|
|
local command body
|
|
case "$arr_type" in
|
|
sonarr) command="EpisodeSearch"; body="{\"name\":\"EpisodeSearch\",\"episodeIds\":[$media_id]}" ;;
|
|
radarr) command="MoviesSearch"; body="{\"name\":\"MoviesSearch\",\"movieIds\":[$media_id]}" ;;
|
|
lidarr) command="AlbumSearch"; body="{\"name\":\"AlbumSearch\",\"albumIds\":[$media_id]}" ;;
|
|
esac
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would trigger $command for media ID $media_id"
|
|
return 0
|
|
fi
|
|
curl -sf --max-time 15 \
|
|
-X POST \
|
|
-H "X-Api-Key: $api_key" \
|
|
-H "Content-Type: application/json" \
|
|
-d "$body" \
|
|
"${url}/api/${api_version}/command" \
|
|
>/dev/null 2>&1
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ── PROCESS AN ARR ────────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
# Args: display_name, arr_type, url, api_key, api_version, enabled,
|
|
# version_major, version_api_prefix
|
|
#
|
|
# Exits cleanly if disabled.
|
|
# Checks API reachability and version before touching queue.
|
|
# Processes each problem item: blocklist + trigger new search.
|
|
# Silent when clean — only warns when problems found or actioned.
|
|
|
|
process_arr() {
|
|
local arr_name="$1"
|
|
local arr_type="$2"
|
|
local url="$3"
|
|
local api_key="$4"
|
|
local api_version="$5"
|
|
local enabled="$6"
|
|
local version_major="$7"
|
|
local version_api_prefix="$8"
|
|
|
|
local actioned=0 skipped_new=0 chronic=0
|
|
local is_chronic fail_key fail_count
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC $arr_name ━━━"
|
|
|
|
# Disabled — skip cleanly
|
|
if [[ "$enabled" != "true" ]]; then
|
|
log "$arr_name recovery disabled — skipping"
|
|
ARR_SUMMARIES+=("$arr_name: disabled")
|
|
return
|
|
fi
|
|
|
|
# URL not configured on this host — skip cleanly
|
|
if [[ -z "$url" ]]; then
|
|
log "$arr_name not configured on $MY_ID — skipping"
|
|
ARR_SUMMARIES+=("$arr_name: not configured on $MY_ID")
|
|
return
|
|
fi
|
|
|
|
# API reachability
|
|
if ! check_api "$url" "$arr_name" 10; then
|
|
warn "$arr_name API unreachable — skipping"
|
|
ARR_SUMMARIES+=("$arr_name: unreachable")
|
|
return
|
|
fi
|
|
|
|
# Version check — exit if API structure may have changed
|
|
if ! check_arr_version "$url" "$api_key" "$version_api_prefix" \
|
|
"$version_major" "$arr_name"; then
|
|
ARR_SUMMARIES+=("$arr_name: version mismatch — skipped")
|
|
return
|
|
fi
|
|
|
|
# Fetch queue
|
|
local queue_data
|
|
queue_data=$(get_queue_data "$url" "$api_key" "$api_version")
|
|
if [[ -z "$queue_data" ]]; then
|
|
warn "$arr_name — could not retrieve queue data"
|
|
ARR_SUMMARIES+=("$arr_name: queue fetch failed")
|
|
return
|
|
fi
|
|
|
|
local total_records
|
|
total_records=$(echo "$queue_data" | jq '.totalRecords // 0' 2>/dev/null)
|
|
log "$arr_name queue: $total_records total items"
|
|
|
|
# Filter for problem items — never touch downloading or imported
|
|
local problem_items
|
|
problem_items=$(echo "$queue_data" | jq -c '
|
|
.records // [] |
|
|
.[] |
|
|
select(
|
|
.trackedDownloadState != "downloading" and
|
|
.trackedDownloadState != "imported" and
|
|
(
|
|
.trackedDownloadState == "importFailed" or
|
|
.trackedDownloadState == "importPending" or
|
|
.trackedDownloadState == "importBlocked" or
|
|
.trackedDownloadStatus == "error" or
|
|
(.status == "warning" and (
|
|
(.errorMessage // "" | ascii_downcase | contains("stalled")) or
|
|
(.statusMessages // [] | .[] | .messages // [] | .[] |
|
|
ascii_downcase | contains("stalled"))
|
|
))
|
|
)
|
|
)
|
|
' 2>/dev/null)
|
|
|
|
if [[ -z "$problem_items" ]]; then
|
|
echo "$arr_name — clean ✅ no failed imports or stalled downloads"
|
|
ARR_SUMMARIES+=("$arr_name: clean ✅")
|
|
return
|
|
fi
|
|
|
|
local problem_count
|
|
problem_count=$(echo "$problem_items" | wc -l)
|
|
warn "$arr_name — found $problem_count problem item(s)"
|
|
|
|
# Process each problem item
|
|
while IFS= read -r item; do
|
|
[[ -z "$item" ]] && continue
|
|
|
|
local queue_id title added tracked_state tracked_status problem_type media_id
|
|
|
|
queue_id=$(echo "$item" | jq -r '.id // empty' 2>/dev/null)
|
|
title=$(echo "$item" | jq -r '.title // "Unknown"' 2>/dev/null)
|
|
added=$(echo "$item" | jq -r '.added // empty' 2>/dev/null)
|
|
tracked_state=$(echo "$item" | jq -r '.trackedDownloadState // ""' 2>/dev/null)
|
|
tracked_status=$(echo "$item" | jq -r '.trackedDownloadStatus // ""' 2>/dev/null)
|
|
|
|
# Human-readable problem type
|
|
case "$tracked_state" in
|
|
importFailed) problem_type="import failed" ;;
|
|
importPending) problem_type="import pending/stuck" ;;
|
|
importBlocked) problem_type="import blocked (matched by ID)" ;;
|
|
*)
|
|
[[ "$tracked_status" == "error" ]] && \
|
|
problem_type="error" || problem_type="stalled"
|
|
;;
|
|
esac
|
|
|
|
# Media ID for search trigger
|
|
case "$arr_type" in
|
|
sonarr) media_id=$(echo "$item" | jq -r '.episodeId // .episode.id // empty' 2>/dev/null) ;;
|
|
radarr) media_id=$(echo "$item" | jq -r '.movieId // .movie.id // empty' 2>/dev/null) ;;
|
|
lidarr) media_id=$(echo "$item" | jq -r '.albumId // .album.id // empty' 2>/dev/null) ;;
|
|
esac
|
|
|
|
[[ -z "$queue_id" ]] && continue
|
|
|
|
# Age check — skip items that are too new to have self-resolved
|
|
if ! item_is_old_enough "$added"; then
|
|
log " Skipping (too new < ${ARR_IMPORT_RECOVERY_AGE}hr): $title"
|
|
(( skipped_new++ ))
|
|
(( TOTAL_SKIPPED++ ))
|
|
continue
|
|
fi
|
|
|
|
warn " $ICON_TRASH $problem_type — $title"
|
|
|
|
# Step 1: Blocklist + remove from queue
|
|
if ! blocklist_item "$url" "$api_key" "$api_version" "$queue_id"; then
|
|
warn " Failed to blocklist: $title"
|
|
(( TOTAL_SKIPPED++ ))
|
|
continue
|
|
fi
|
|
log " Blocklisted: $queue_id"
|
|
|
|
# Step 2: Circuit breaker — track consecutive failures per (arr_type, media_id).
|
|
# Some items can never resolve via blind retry (e.g. an album missing 1-2 tracks
|
|
# where no available release matches the existing edition) — without this, the
|
|
# same item gets blocklisted + re-searched forever, every run.
|
|
is_chronic=false
|
|
if [[ -n "$media_id" ]]; then
|
|
fail_key="${arr_type}:${media_id}"
|
|
fail_count=$(( ${FAILURE_COUNTS[$fail_key]:-0} + 1 ))
|
|
FAILURE_COUNTS[$fail_key]="$fail_count"
|
|
if [[ "$fail_count" -gt "$ARR_RECOVERY_MAX_ATTEMPTS" ]]; then
|
|
is_chronic=true
|
|
(( chronic++ ))
|
|
(( TOTAL_CHRONIC++ ))
|
|
warn " Chronic (${fail_count} consecutive failures) — needs manual review: $title"
|
|
[[ "$fail_count" -eq $(( ARR_RECOVERY_MAX_ATTEMPTS + 1 )) ]] && \
|
|
notify "$arr_name item now chronic after ${ARR_RECOVERY_MAX_ATTEMPTS} failed attempts — needs manual review: $title" \
|
|
"Arr Recovery" "warning"
|
|
fi
|
|
fi
|
|
|
|
# Step 3: Trigger new search — skipped for chronic items
|
|
if [[ "$is_chronic" == true ]]; then
|
|
log " Skipping auto re-search (chronic): $title"
|
|
elif [[ -n "$media_id" ]]; then
|
|
if trigger_search "$url" "$api_key" "$api_version" "$arr_type" "$media_id"; then
|
|
log " New search triggered: $title"
|
|
else
|
|
warn " Blocklisted but search trigger failed: $title"
|
|
fi
|
|
else
|
|
warn " Blocklisted but no media ID found — search not triggered: $title"
|
|
fi
|
|
|
|
(( actioned++ ))
|
|
(( TOTAL_ACTIONED++ ))
|
|
|
|
done <<< "$problem_items"
|
|
|
|
if [[ "$actioned" -gt 0 ]]; then
|
|
warn "$arr_name — actioned: $actioned | skipped (too new): $skipped_new | chronic: $chronic"
|
|
else
|
|
log "$arr_name — nothing actioned | skipped (too new): $skipped_new"
|
|
fi
|
|
|
|
ARR_SUMMARIES+=("$arr_name: actioned $actioned | too new $skipped_new | chronic $chronic")
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Process Each Arr ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Arrs Failed/Stalled Recovery — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
|
log "Age threshold: ${ARR_IMPORT_RECOVERY_AGE}hr"
|
|
|
|
START=$(date +%s)
|
|
|
|
# Sonarr — uses aliased vars set by detect_hosts()
|
|
process_arr \
|
|
"Sonarr" \
|
|
"sonarr" \
|
|
"${SONARR_URL:-}" \
|
|
"${SONARR_API_KEY:-}" \
|
|
"v3" \
|
|
"${SONARR_RECOVERY:-true}" \
|
|
"${SONARR_VERSION_MAJOR:-4}" \
|
|
"v3"
|
|
|
|
# Radarr — uses aliased vars set by detect_hosts()
|
|
process_arr \
|
|
"Radarr" \
|
|
"radarr" \
|
|
"${RADARR_URL:-}" \
|
|
"${RADARR_API_KEY:-}" \
|
|
"v3" \
|
|
"${RADARR_RECOVERY:-true}" \
|
|
"${RADARR_VERSION_MAJOR:-6}" \
|
|
"v3"
|
|
|
|
# Lidarr — HOST1 only, LIDARR_URL empty on HOST2 → exits cleanly via "not configured" guard
|
|
process_arr \
|
|
"Lidarr" \
|
|
"lidarr" \
|
|
"${LIDARR_URL:-}" \
|
|
"${LIDARR_API_KEY:-}" \
|
|
"v1" \
|
|
"${LIDARR_RECOVERY:-false}" \
|
|
"${LIDARR_VERSION_MAJOR:-3}" \
|
|
"v1"
|
|
|
|
END=$(date +%s)
|
|
|
|
# Persist updated failure counts — skipped in dry-run so nothing is recorded for a preview
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
mkdir -p "$(dirname "$ARR_RECOVERY_FAILURE_COUNTS")" 2>/dev/null
|
|
: > "$ARR_RECOVERY_FAILURE_COUNTS"
|
|
for key in "${!FAILURE_COUNTS[@]}"; do
|
|
echo "${key}|${FAILURE_COUNTS[$key]}" >> "$ARR_RECOVERY_FAILURE_COUNTS"
|
|
done
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY ARR RECOVERY SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
echo "$ICON_TRASH Actioned: $TOTAL_ACTIONED items blocklisted + searched"
|
|
echo "$ICON_SKIP Skipped: $TOTAL_SKIPPED items (too new)"
|
|
echo "$ICON_WARN Chronic: $TOTAL_CHRONIC items (blocklisted, auto re-search stopped)"
|
|
echo ""
|
|
for summary in "${ARR_SUMMARIES[@]}"; do
|
|
echo " $ICON_SUMMARY $summary"
|
|
done
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no changes made"
|
|
elif [[ "$TOTAL_ACTIONED" -gt 0 ]]; then
|
|
warn "$ICON_DONE Done — $TOTAL_ACTIONED item(s) blocklisted and re-searched"
|
|
notify "Arr recovery on $(hostname) — $TOTAL_ACTIONED item(s) blocklisted and re-searched" \
|
|
"Arr Recovery" "warning"
|
|
else
|
|
echo "$ICON_DONE Done — nothing to recover (all arrs clean)"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
# Write stats for sunday_morning_coffee_report.sh
|
|
if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_RECOVERY_STATS:-}" ]]; then
|
|
echo "$(date '+%Y-%m-%d')|$(date '+%H:%M')|${TOTAL_ACTIONED}|${TOTAL_SKIPPED}|${TOTAL_CHRONIC}" \
|
|
>> "$ARR_RECOVERY_STATS" 2>/dev/null || true
|
|
fi
|
|
|
|
exit 0 |