Move arr stack scripts from Media/ to Arrs_Stack/

Media/ now holds only media-level scripts (cleaner, permissions, play_state_sync).
All arr management scripts (cleanup, discovery, sync, webhooks, release fixer) live in Arrs_Stack/.
This commit is contained in:
Gmer4Lfe
2026-06-27 18:39:33 -04:00
parent 99b2d1879b
commit b4bc9267e9
26 changed files with 88 additions and 88 deletions
+477
View File
@@ -0,0 +1,477 @@
#!/bin/bash
# ==============================================================================================
# ========================= Arrs Failed / Stalled Recovery =====================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Detect and recover failed imports and stalled downloads across Sonarr, Radarr,
# and Lidarr. Blocklists the bad release and triggers a re-search — hands-free
# overnight recovery.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Four problem types detected from the arr queue API:
# importFailed — downloaded but arr couldn't import the file
# importPending — downloaded, stuck waiting to import (will not self-resolve)
# error status — serious failure not covered by the above two states
# stalled — download stuck with no connections or no progress
#
# Never touches items with state "downloading" or "imported" — safe to run anytime.
# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first.
#
# Per problem item (3-step response):
# 1. Blocklist the release — prevents re-grabbing the same bad release
# 2. Remove from queue — cleans up the failed item
# 3. Trigger new search — finds a different release automatically
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Hands-Free Recovery
# The script completes the full recovery cycle autonomously — blocklist, remove,
# re-search. No operator decision required. A failed import at midnight resolves
# itself before morning without any intervention.
#
# Age Gate Before Action
# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped. Arrs have their own
# retry logic — acting immediately would race against it. The age gate gives
# the arr time to self-resolve before this script escalates.
#
# Blocklist First
# The bad release is blocklisted before removal and re-search. Without this,
# the re-search can re-grab the same release that just failed.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# acquire_lock — prevents concurrent runs overlapping
# jq validation — exits if jq not installed (required for JSON parsing)
# API pre-flight — checks each arr is reachable before querying queue
# Version check — check_arr_version() verifies running arr matches master.conf major
# version; exits rather than silently misoperating after upgrade
# Age threshold — skips items newer than ARR_IMPORT_RECOVERY_AGE (default 6hr)
# Silent by default — only problems produce output, clean arrs stay silent
#
# API version mapping (endpoint paths differ from major version labels):
# Sonarr v4 → /api/v3/ (v3 endpoint retained in v4)
# Radarr v6 → /api/v3/ (v3 endpoint retained in v6)
# Lidarr v3 → /api/v1/ (different from Sonarr/Radarr)
#
# ==============================================================================================
# STATE FILES
# ==============================================================================================
#
# ARR_RECOVERY_STATS — stats file written after each run (read by coffee report)
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# host*.conf
#
# HOST*_SONARR_URL / HOST*_SONARR_API_KEY / HOST*_SONARR_RECOVERY
# HOST*_RADARR_URL / HOST*_RADARR_API_KEY / HOST*_RADARR_RECOVERY
# HOST1_LIDARR_URL / HOST1_LIDARR_API_KEY / HOST1_LIDARR_RECOVERY
# All aliased by detect_hosts() — script uses unprefixed names
#
# master.conf
#
# ARR_IMPORT_RECOVERY_AGE — hours before item is eligible for recovery (default: 6)
# SONARR_VERSION_MAJOR — expected Sonarr major version (e.g. 4)
# RADARR_VERSION_MAJOR — expected Radarr major version (e.g. 6)
# LIDARR_VERSION_MAJOR — expected Lidarr major version (e.g. 3)
# ARR_RECOVERY_STATS — stats file path (read by coffee report)
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# arrs_failed_stalled_recovery.sh — normal run
# arrs_failed_stalled_recovery.sh --dry-run — show what would be actioned, no changes
# arrs_failed_stalled_recovery.sh --log — verbose output
# arrs_failed_stalled_recovery.sh --status — show config and exit
#
# Recommended schedule: 0 5 * * * (5am daily)
# Or every 6hr: 0 */6 * * * (matches ARR_IMPORT_RECOVERY_AGE default)
#
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
acquire_lock
# detect_hosts() sets MY_ID and aliases SONARR_*, RADARR_*, LIDARR_* vars
detect_hosts
# jq is required — not optional — for JSON parsing
if ! command -v jq >/dev/null 2>&1; then
error "jq is not installed — required for arr API JSON parsing"
error "Install: apt-get install jq or brew install jq"
notify "arrs_failed_stalled_recovery failed on $(hostname) — jq not installed" \
"Arr Recovery" "warning"
exit 1
fi
log "jq found"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no items will be blocklisted or searched"
# Age threshold in seconds
AGE_THRESHOLD_SECONDS=$(( ARR_IMPORT_RECOVERY_AGE * 3600 ))
log "$ICON_GEAR Config: age-threshold=${ARR_IMPORT_RECOVERY_AGE}hr sonarr-v${SONARR_VERSION_MAJOR} radarr-v${RADARR_VERSION_MAJOR} lidarr-v${LIDARR_VERSION_MAJOR:-?}"
TOTAL_ACTIONED=0
TOTAL_SKIPPED=0
ARR_SUMMARIES=()
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_SYNC Sonarr: ${SONARR_URL:-not configured} (recovery: ${SONARR_RECOVERY:-true})"
echo "$ICON_SYNC Radarr: ${RADARR_URL:-not configured} (recovery: ${RADARR_RECOVERY:-true})"
echo "$ICON_SYNC Lidarr: ${LIDARR_URL:-not configured on this host} (recovery: ${LIDARR_RECOVERY:-false})"
echo "$ICON_TIME Age thresh: ${ARR_IMPORT_RECOVERY_AGE}hr"
echo "$ICON_GEAR Sonarr ver: v${SONARR_VERSION_MAJOR} expected"
echo "$ICON_GEAR Radarr ver: v${RADARR_VERSION_MAJOR} expected"
echo "$ICON_GEAR Lidarr ver: v${LIDARR_VERSION_MAJOR} expected"
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Check if a queue item is older than ARR_IMPORT_RECOVERY_AGE
# Returns 0 (old enough) or 1 (too new — skip)
item_is_old_enough() {
local added="$1"
[[ -z "$added" ]] && return 0 # no date = treat as old enough, safe to act
local added_epoch
added_epoch=$(date -d "$added" +%s 2>/dev/null) || return 0
local age_seconds=$(( $(date +%s) - added_epoch ))
[[ "$age_seconds" -ge "$AGE_THRESHOLD_SECONDS" ]]
}
# Query the arr queue API and return all records
# Args: url, api_key, api_version
get_queue_data() {
local url="$1" api_key="$2" api_version="$3"
curl -sf --max-time 15 \
-H "X-Api-Key: $api_key" \
"${url}/api/${api_version}/queue?page=1&pageSize=200&includeUnknownSeriesItems=true&includeUnknownArtistItems=true" \
2>/dev/null
}
# Blocklist and remove a queue item
# Args: url, api_key, api_version, queue_id
blocklist_item() {
local url="$1" api_key="$2" api_version="$3" queue_id="$4"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would blocklist queue item $queue_id"
return 0
fi
curl -sf --max-time 15 \
-X DELETE \
-H "X-Api-Key: $api_key" \
"${url}/api/${api_version}/queue/${queue_id}?removeFromClient=true&blocklist=true&skipRedownload=false" \
>/dev/null 2>&1
}
# Trigger a new search for the media item
# Args: url, api_key, api_version, arr_type, media_id
trigger_search() {
local url="$1" api_key="$2" api_version="$3" arr_type="$4" media_id="$5"
local command body
case "$arr_type" in
sonarr) command="EpisodeSearch"; body="{\"name\":\"EpisodeSearch\",\"episodeIds\":[$media_id]}" ;;
radarr) command="MoviesSearch"; body="{\"name\":\"MoviesSearch\",\"movieIds\":[$media_id]}" ;;
lidarr) command="AlbumSearch"; body="{\"name\":\"AlbumSearch\",\"albumIds\":[$media_id]}" ;;
esac
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would trigger $command for media ID $media_id"
return 0
fi
curl -sf --max-time 15 \
-X POST \
-H "X-Api-Key: $api_key" \
-H "Content-Type: application/json" \
-d "$body" \
"${url}/api/${api_version}/command" \
>/dev/null 2>&1
}
# ==============================================================================================
# ── PROCESS AN ARR ────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Args: display_name, arr_type, url, api_key, api_version, enabled,
# version_major, version_api_prefix
#
# Exits cleanly if disabled.
# Checks API reachability and version before touching queue.
# Processes each problem item: blocklist + trigger new search.
# Silent when clean — only warns when problems found or actioned.
process_arr() {
local arr_name="$1"
local arr_type="$2"
local url="$3"
local api_key="$4"
local api_version="$5"
local enabled="$6"
local version_major="$7"
local version_api_prefix="$8"
local actioned=0 skipped_new=0
echo ""
echo "━━━ $ICON_SYNC $arr_name ━━━"
# Disabled — skip cleanly
if [[ "$enabled" != "true" ]]; then
log "$arr_name recovery disabled — skipping"
ARR_SUMMARIES+=("$arr_name: disabled")
return
fi
# URL not configured on this host — skip cleanly
if [[ -z "$url" ]]; then
log "$arr_name not configured on $MY_ID — skipping"
ARR_SUMMARIES+=("$arr_name: not configured on $MY_ID")
return
fi
# API reachability
if ! check_api "$url" "$arr_name" 10; then
warn "$arr_name API unreachable — skipping"
ARR_SUMMARIES+=("$arr_name: unreachable")
return
fi
# Version check — exit if API structure may have changed
if ! check_arr_version "$url" "$api_key" "$version_api_prefix" \
"$version_major" "$arr_name"; then
ARR_SUMMARIES+=("$arr_name: version mismatch — skipped")
return
fi
# Fetch queue
local queue_data
queue_data=$(get_queue_data "$url" "$api_key" "$api_version")
if [[ -z "$queue_data" ]]; then
warn "$arr_name — could not retrieve queue data"
ARR_SUMMARIES+=("$arr_name: queue fetch failed")
return
fi
local total_records
total_records=$(echo "$queue_data" | jq '.totalRecords // 0' 2>/dev/null)
log "$arr_name queue: $total_records total items"
# Filter for problem items — never touch downloading or imported
local problem_items
problem_items=$(echo "$queue_data" | jq -c '
.records // [] |
.[] |
select(
.trackedDownloadState != "downloading" and
.trackedDownloadState != "imported" and
(
.trackedDownloadState == "importFailed" or
.trackedDownloadState == "importPending" or
.trackedDownloadStatus == "error" or
(.status == "warning" and (
(.errorMessage // "" | ascii_downcase | contains("stalled")) or
(.statusMessages // [] | .[] | .messages // [] | .[] |
ascii_downcase | contains("stalled"))
))
)
)
' 2>/dev/null)
if [[ -z "$problem_items" ]]; then
echo "$arr_name — clean ✅ no failed imports or stalled downloads"
ARR_SUMMARIES+=("$arr_name: clean ✅")
return
fi
local problem_count
problem_count=$(echo "$problem_items" | wc -l)
warn "$arr_name — found $problem_count problem item(s)"
# Process each problem item
while IFS= read -r item; do
[[ -z "$item" ]] && continue
local queue_id title added tracked_state tracked_status problem_type media_id
queue_id=$(echo "$item" | jq -r '.id // empty' 2>/dev/null)
title=$(echo "$item" | jq -r '.title // "Unknown"' 2>/dev/null)
added=$(echo "$item" | jq -r '.added // empty' 2>/dev/null)
tracked_state=$(echo "$item" | jq -r '.trackedDownloadState // ""' 2>/dev/null)
tracked_status=$(echo "$item" | jq -r '.trackedDownloadStatus // ""' 2>/dev/null)
# Human-readable problem type
case "$tracked_state" in
importFailed) problem_type="import failed" ;;
importPending) problem_type="import pending/stuck" ;;
*)
[[ "$tracked_status" == "error" ]] && \
problem_type="error" || problem_type="stalled"
;;
esac
# Media ID for search trigger
case "$arr_type" in
sonarr) media_id=$(echo "$item" | jq -r '.episodeId // .episode.id // empty' 2>/dev/null) ;;
radarr) media_id=$(echo "$item" | jq -r '.movieId // .movie.id // empty' 2>/dev/null) ;;
lidarr) media_id=$(echo "$item" | jq -r '.albumId // .album.id // empty' 2>/dev/null) ;;
esac
[[ -z "$queue_id" ]] && continue
# Age check — skip items that are too new to have self-resolved
if ! item_is_old_enough "$added"; then
log " Skipping (too new < ${ARR_IMPORT_RECOVERY_AGE}hr): $title"
(( skipped_new++ ))
(( TOTAL_SKIPPED++ ))
continue
fi
warn " $ICON_TRASH $problem_type$title"
# Step 1: Blocklist + remove from queue
if ! blocklist_item "$url" "$api_key" "$api_version" "$queue_id"; then
warn " Failed to blocklist: $title"
(( TOTAL_SKIPPED++ ))
continue
fi
log " Blocklisted: $queue_id"
# Step 2: Trigger new search
if [[ -n "$media_id" ]]; then
if trigger_search "$url" "$api_key" "$api_version" "$arr_type" "$media_id"; then
log " New search triggered: $title"
else
warn " Blocklisted but search trigger failed: $title"
fi
else
warn " Blocklisted but no media ID found — search not triggered: $title"
fi
(( actioned++ ))
(( TOTAL_ACTIONED++ ))
done <<< "$problem_items"
if [[ "$actioned" -gt 0 ]]; then
warn "$arr_name — actioned: $actioned | skipped (too new): $skipped_new"
else
log "$arr_name — nothing actioned | skipped (too new): $skipped_new"
fi
ARR_SUMMARIES+=("$arr_name: actioned $actioned | too new $skipped_new")
}
# ==============================================================================================
# ━━━ Process Each Arr ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Arrs Failed/Stalled Recovery — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
log "Age threshold: ${ARR_IMPORT_RECOVERY_AGE}hr"
START=$(date +%s)
# Sonarr — uses aliased vars set by detect_hosts()
process_arr \
"Sonarr" \
"sonarr" \
"${SONARR_URL:-}" \
"${SONARR_API_KEY:-}" \
"v3" \
"${SONARR_RECOVERY:-true}" \
"${SONARR_VERSION_MAJOR:-4}" \
"v3"
# Radarr — uses aliased vars set by detect_hosts()
process_arr \
"Radarr" \
"radarr" \
"${RADARR_URL:-}" \
"${RADARR_API_KEY:-}" \
"v3" \
"${RADARR_RECOVERY:-true}" \
"${RADARR_VERSION_MAJOR:-6}" \
"v3"
# Lidarr — HOST1 only, LIDARR_URL empty on HOST2 → exits cleanly via "not configured" guard
process_arr \
"Lidarr" \
"lidarr" \
"${LIDARR_URL:-}" \
"${LIDARR_API_KEY:-}" \
"v1" \
"${LIDARR_RECOVERY:-false}" \
"${LIDARR_VERSION_MAJOR:-3}" \
"v1"
END=$(date +%s)
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY ARR RECOVERY SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo "$ICON_TRASH Actioned: $TOTAL_ACTIONED items blocklisted + searched"
echo "$ICON_SKIP Skipped: $TOTAL_SKIPPED items (too new)"
echo ""
for summary in "${ARR_SUMMARIES[@]}"; do
echo " $ICON_SUMMARY $summary"
done
echo ""
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — no changes made"
elif [[ "$TOTAL_ACTIONED" -gt 0 ]]; then
warn "$ICON_DONE Done — $TOTAL_ACTIONED item(s) blocklisted and re-searched"
notify "Arr recovery on $(hostname)$TOTAL_ACTIONED item(s) blocklisted and re-searched" \
"Arr Recovery" "warning"
else
echo "$ICON_DONE Done — nothing to recover (all arrs clean)"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
# Write stats for sunday_morning_coffee_report.sh
if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_RECOVERY_STATS:-}" ]]; then
echo "$(date '+%Y-%m-%d')|$(date '+%H:%M')|${TOTAL_ACTIONED}|${TOTAL_SKIPPED}" \
>> "$ARR_RECOVERY_STATS" 2>/dev/null || true
fi
exit 0