Add circuit breaker to arrs_failed_stalled_recovery.sh
Some items (e.g. an album missing 1-2 tracks where no available release matches the existing edition/track count) can never resolve via blind retry. Without a limit, the same media ID gets blocklisted and re-searched every 4 hours forever — confirmed live on ~19 Lidarr albums cycling identically across five consecutive runs today, each one downloading a fresh release, failing import for the same structural reason, and starting over. Tracks consecutive failures per (arr_type, media_id) in a persisted state file. After ARR_RECOVERY_MAX_ATTEMPTS (default 3) failures, the item is still blocklisted and removed from the queue, but auto re-search stops — notified once when it crosses the threshold, then left for manual review instead of retried forever.
This commit is contained in:
Gmer4Lfe
@@ -45,6 +45,16 @@
|
||||
# The bad release is blocklisted before removal and re-search. Without this,
|
||||
# the re-search can re-grab the same release that just failed.
|
||||
#
|
||||
# Circuit Breaker Per Media Item
|
||||
# Some items can never resolve via blind retry — e.g. an album missing 1-2
|
||||
# tracks where every available release is a different edition that doesn't
|
||||
# match. Without a limit, the same media ID gets blocklisted + re-searched
|
||||
# forever, every run, burning bandwidth and indexer queries for nothing.
|
||||
# After ARR_RECOVERY_MAX_ATTEMPTS consecutive failures for the same
|
||||
# (arr_type, media_id), the item is still blocklisted/cleaned from the queue
|
||||
# but search is no longer auto-triggered — it's flagged chronic and left for
|
||||
# manual review instead.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
@@ -66,7 +76,9 @@
|
||||
# STATE FILES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# ARR_RECOVERY_STATS — stats file written after each run (read by coffee report)
|
||||
# ARR_RECOVERY_STATS — stats file written after each run (read by coffee report)
|
||||
# ARR_RECOVERY_FAILURE_COUNTS — per (arr_type, media_id) consecutive-failure counts,
|
||||
# persists across runs so the circuit breaker survives restarts.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
@@ -82,10 +94,13 @@
|
||||
# master.conf
|
||||
#
|
||||
# ARR_IMPORT_RECOVERY_AGE — hours before item is eligible for recovery (default: 6)
|
||||
# ARR_RECOVERY_MAX_ATTEMPTS — consecutive failures before an item is flagged chronic
|
||||
# and auto re-search stops (default: 3)
|
||||
# SONARR_VERSION_MAJOR — expected Sonarr major version (e.g. 4)
|
||||
# RADARR_VERSION_MAJOR — expected Radarr major version (e.g. 6)
|
||||
# LIDARR_VERSION_MAJOR — expected Lidarr major version (e.g. 3)
|
||||
# ARR_RECOVERY_STATS — stats file path (read by coffee report)
|
||||
# ARR_RECOVERY_FAILURE_COUNTS — failure-count state file path
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
@@ -135,11 +150,23 @@ log "jq found"
|
||||
|
||||
# Age threshold in seconds
|
||||
AGE_THRESHOLD_SECONDS=$(( ARR_IMPORT_RECOVERY_AGE * 3600 ))
|
||||
ARR_RECOVERY_MAX_ATTEMPTS="${ARR_RECOVERY_MAX_ATTEMPTS:-3}"
|
||||
ARR_RECOVERY_FAILURE_COUNTS="${ARR_RECOVERY_FAILURE_COUNTS:-$DATA_DIR/arr_recovery_failure_counts.db}"
|
||||
|
||||
log "$ICON_GEAR Config: age-threshold=${ARR_IMPORT_RECOVERY_AGE}hr sonarr-v${SONARR_VERSION_MAJOR} radarr-v${RADARR_VERSION_MAJOR} lidarr-v${LIDARR_VERSION_MAJOR:-?}"
|
||||
log "$ICON_GEAR Config: age-threshold=${ARR_IMPORT_RECOVERY_AGE}hr max-attempts=${ARR_RECOVERY_MAX_ATTEMPTS} sonarr-v${SONARR_VERSION_MAJOR} radarr-v${RADARR_VERSION_MAJOR} lidarr-v${LIDARR_VERSION_MAJOR:-?}"
|
||||
|
||||
# Load persisted per-item failure counts — key is "arr_type:media_id"
|
||||
declare -A FAILURE_COUNTS
|
||||
if [[ -f "$ARR_RECOVERY_FAILURE_COUNTS" ]]; then
|
||||
while IFS='|' read -r _key _count; do
|
||||
[[ -z "$_key" ]] && continue
|
||||
FAILURE_COUNTS["$_key"]="$_count"
|
||||
done < "$ARR_RECOVERY_FAILURE_COUNTS"
|
||||
fi
|
||||
|
||||
TOTAL_ACTIONED=0
|
||||
TOTAL_SKIPPED=0
|
||||
TOTAL_CHRONIC=0
|
||||
ARR_SUMMARIES=()
|
||||
|
||||
# ==============================================================================================
|
||||
@@ -153,6 +180,7 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo "$ICON_SYNC Radarr: ${RADARR_URL:-not configured} (recovery: ${RADARR_RECOVERY:-true})"
|
||||
echo "$ICON_SYNC Lidarr: ${LIDARR_URL:-not configured on this host} (recovery: ${LIDARR_RECOVERY:-false})"
|
||||
echo "$ICON_TIME Age thresh: ${ARR_IMPORT_RECOVERY_AGE}hr"
|
||||
echo "$ICON_GEAR Max attempts: ${ARR_RECOVERY_MAX_ATTEMPTS:-3} (chronic after this many)"
|
||||
echo "$ICON_GEAR Sonarr ver: v${SONARR_VERSION_MAJOR} expected"
|
||||
echo "$ICON_GEAR Radarr ver: v${RADARR_VERSION_MAJOR} expected"
|
||||
echo "$ICON_GEAR Lidarr ver: v${LIDARR_VERSION_MAJOR} expected"
|
||||
@@ -246,7 +274,8 @@ process_arr() {
|
||||
local version_major="$7"
|
||||
local version_api_prefix="$8"
|
||||
|
||||
local actioned=0 skipped_new=0
|
||||
local actioned=0 skipped_new=0 chronic=0
|
||||
local is_chronic fail_key fail_count
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC $arr_name ━━━"
|
||||
@@ -372,8 +401,30 @@ process_arr() {
|
||||
fi
|
||||
log " Blocklisted: $queue_id"
|
||||
|
||||
# Step 2: Trigger new search
|
||||
# Step 2: Circuit breaker — track consecutive failures per (arr_type, media_id).
|
||||
# Some items can never resolve via blind retry (e.g. an album missing 1-2 tracks
|
||||
# where no available release matches the existing edition) — without this, the
|
||||
# same item gets blocklisted + re-searched forever, every run.
|
||||
is_chronic=false
|
||||
if [[ -n "$media_id" ]]; then
|
||||
fail_key="${arr_type}:${media_id}"
|
||||
fail_count=$(( ${FAILURE_COUNTS[$fail_key]:-0} + 1 ))
|
||||
FAILURE_COUNTS[$fail_key]="$fail_count"
|
||||
if [[ "$fail_count" -gt "$ARR_RECOVERY_MAX_ATTEMPTS" ]]; then
|
||||
is_chronic=true
|
||||
(( chronic++ ))
|
||||
(( TOTAL_CHRONIC++ ))
|
||||
warn " Chronic (${fail_count} consecutive failures) — needs manual review: $title"
|
||||
[[ "$fail_count" -eq $(( ARR_RECOVERY_MAX_ATTEMPTS + 1 )) ]] && \
|
||||
notify "$arr_name item now chronic after ${ARR_RECOVERY_MAX_ATTEMPTS} failed attempts — needs manual review: $title" \
|
||||
"Arr Recovery" "warning"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Step 3: Trigger new search — skipped for chronic items
|
||||
if [[ "$is_chronic" == true ]]; then
|
||||
log " Skipping auto re-search (chronic): $title"
|
||||
elif [[ -n "$media_id" ]]; then
|
||||
if trigger_search "$url" "$api_key" "$api_version" "$arr_type" "$media_id"; then
|
||||
log " New search triggered: $title"
|
||||
else
|
||||
@@ -389,12 +440,12 @@ process_arr() {
|
||||
done <<< "$problem_items"
|
||||
|
||||
if [[ "$actioned" -gt 0 ]]; then
|
||||
warn "$arr_name — actioned: $actioned | skipped (too new): $skipped_new"
|
||||
warn "$arr_name — actioned: $actioned | skipped (too new): $skipped_new | chronic: $chronic"
|
||||
else
|
||||
log "$arr_name — nothing actioned | skipped (too new): $skipped_new"
|
||||
fi
|
||||
|
||||
ARR_SUMMARIES+=("$arr_name: actioned $actioned | too new $skipped_new")
|
||||
ARR_SUMMARIES+=("$arr_name: actioned $actioned | too new $skipped_new | chronic $chronic")
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
@@ -442,6 +493,15 @@ process_arr \
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# Persist updated failure counts — skipped in dry-run so nothing is recorded for a preview
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mkdir -p "$(dirname "$ARR_RECOVERY_FAILURE_COUNTS")" 2>/dev/null
|
||||
: > "$ARR_RECOVERY_FAILURE_COUNTS"
|
||||
for key in "${!FAILURE_COUNTS[@]}"; do
|
||||
echo "${key}|${FAILURE_COUNTS[$key]}" >> "$ARR_RECOVERY_FAILURE_COUNTS"
|
||||
done
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
@@ -451,6 +511,7 @@ echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo "$ICON_TRASH Actioned: $TOTAL_ACTIONED items blocklisted + searched"
|
||||
echo "$ICON_SKIP Skipped: $TOTAL_SKIPPED items (too new)"
|
||||
echo "$ICON_WARN Chronic: $TOTAL_CHRONIC items (blocklisted, auto re-search stopped)"
|
||||
echo ""
|
||||
for summary in "${ARR_SUMMARIES[@]}"; do
|
||||
echo " $ICON_SUMMARY $summary"
|
||||
@@ -470,7 +531,7 @@ echo "━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
# Write stats for sunday_morning_coffee_report.sh
|
||||
if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_RECOVERY_STATS:-}" ]]; then
|
||||
echo "$(date '+%Y-%m-%d')|$(date '+%H:%M')|${TOTAL_ACTIONED}|${TOTAL_SKIPPED}" \
|
||||
echo "$(date '+%Y-%m-%d')|$(date '+%H:%M')|${TOTAL_ACTIONED}|${TOTAL_SKIPPED}|${TOTAL_CHRONIC}" \
|
||||
>> "$ARR_RECOVERY_STATS" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
|
||||
Reference in New Issue
Block a user