diff --git a/Arrs_Stack/arrs_failed_stalled_recovery.sh b/Arrs_Stack/arrs_failed_stalled_recovery.sh index aa286a3..fe5d8ad 100755 --- a/Arrs_Stack/arrs_failed_stalled_recovery.sh +++ b/Arrs_Stack/arrs_failed_stalled_recovery.sh @@ -45,6 +45,16 @@ # The bad release is blocklisted before removal and re-search. Without this, # the re-search can re-grab the same release that just failed. # +# Circuit Breaker Per Media Item +# Some items can never resolve via blind retry — e.g. an album missing 1-2 +# tracks where every available release is a different edition that doesn't +# match. Without a limit, the same media ID gets blocklisted + re-searched +# forever, every run, burning bandwidth and indexer queries for nothing. +# After ARR_RECOVERY_MAX_ATTEMPTS consecutive failures for the same +# (arr_type, media_id), the item is still blocklisted/cleaned from the queue +# but search is no longer auto-triggered — it's flagged chronic and left for +# manual review instead. +# # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== @@ -66,7 +76,9 @@ # STATE FILES # ============================================================================================== # -# ARR_RECOVERY_STATS — stats file written after each run (read by coffee report) +# ARR_RECOVERY_STATS — stats file written after each run (read by coffee report) +# ARR_RECOVERY_FAILURE_COUNTS — per (arr_type, media_id) consecutive-failure counts, +# persists across runs so the circuit breaker survives restarts. # # ============================================================================================== # CONFIGURATION @@ -82,10 +94,13 @@ # master.conf # # ARR_IMPORT_RECOVERY_AGE — hours before item is eligible for recovery (default: 6) +# ARR_RECOVERY_MAX_ATTEMPTS — consecutive failures before an item is flagged chronic +# and auto re-search stops (default: 3) # SONARR_VERSION_MAJOR — expected Sonarr major version (e.g. 4) # RADARR_VERSION_MAJOR — expected Radarr major version (e.g. 6) # LIDARR_VERSION_MAJOR — expected Lidarr major version (e.g. 3) # ARR_RECOVERY_STATS — stats file path (read by coffee report) +# ARR_RECOVERY_FAILURE_COUNTS — failure-count state file path # # ============================================================================================== # RUNTIME MODES @@ -135,11 +150,23 @@ log "jq found" # Age threshold in seconds AGE_THRESHOLD_SECONDS=$(( ARR_IMPORT_RECOVERY_AGE * 3600 )) +ARR_RECOVERY_MAX_ATTEMPTS="${ARR_RECOVERY_MAX_ATTEMPTS:-3}" +ARR_RECOVERY_FAILURE_COUNTS="${ARR_RECOVERY_FAILURE_COUNTS:-$DATA_DIR/arr_recovery_failure_counts.db}" -log "$ICON_GEAR Config: age-threshold=${ARR_IMPORT_RECOVERY_AGE}hr sonarr-v${SONARR_VERSION_MAJOR} radarr-v${RADARR_VERSION_MAJOR} lidarr-v${LIDARR_VERSION_MAJOR:-?}" +log "$ICON_GEAR Config: age-threshold=${ARR_IMPORT_RECOVERY_AGE}hr max-attempts=${ARR_RECOVERY_MAX_ATTEMPTS} sonarr-v${SONARR_VERSION_MAJOR} radarr-v${RADARR_VERSION_MAJOR} lidarr-v${LIDARR_VERSION_MAJOR:-?}" + +# Load persisted per-item failure counts — key is "arr_type:media_id" +declare -A FAILURE_COUNTS +if [[ -f "$ARR_RECOVERY_FAILURE_COUNTS" ]]; then + while IFS='|' read -r _key _count; do + [[ -z "$_key" ]] && continue + FAILURE_COUNTS["$_key"]="$_count" + done < "$ARR_RECOVERY_FAILURE_COUNTS" +fi TOTAL_ACTIONED=0 TOTAL_SKIPPED=0 +TOTAL_CHRONIC=0 ARR_SUMMARIES=() # ============================================================================================== @@ -153,6 +180,7 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "$ICON_SYNC Radarr: ${RADARR_URL:-not configured} (recovery: ${RADARR_RECOVERY:-true})" echo "$ICON_SYNC Lidarr: ${LIDARR_URL:-not configured on this host} (recovery: ${LIDARR_RECOVERY:-false})" echo "$ICON_TIME Age thresh: ${ARR_IMPORT_RECOVERY_AGE}hr" + echo "$ICON_GEAR Max attempts: ${ARR_RECOVERY_MAX_ATTEMPTS:-3} (chronic after this many)" echo "$ICON_GEAR Sonarr ver: v${SONARR_VERSION_MAJOR} expected" echo "$ICON_GEAR Radarr ver: v${RADARR_VERSION_MAJOR} expected" echo "$ICON_GEAR Lidarr ver: v${LIDARR_VERSION_MAJOR} expected" @@ -246,7 +274,8 @@ process_arr() { local version_major="$7" local version_api_prefix="$8" - local actioned=0 skipped_new=0 + local actioned=0 skipped_new=0 chronic=0 + local is_chronic fail_key fail_count echo "" echo "━━━ $ICON_SYNC $arr_name ━━━" @@ -372,8 +401,30 @@ process_arr() { fi log " Blocklisted: $queue_id" - # Step 2: Trigger new search + # Step 2: Circuit breaker — track consecutive failures per (arr_type, media_id). + # Some items can never resolve via blind retry (e.g. an album missing 1-2 tracks + # where no available release matches the existing edition) — without this, the + # same item gets blocklisted + re-searched forever, every run. + is_chronic=false if [[ -n "$media_id" ]]; then + fail_key="${arr_type}:${media_id}" + fail_count=$(( ${FAILURE_COUNTS[$fail_key]:-0} + 1 )) + FAILURE_COUNTS[$fail_key]="$fail_count" + if [[ "$fail_count" -gt "$ARR_RECOVERY_MAX_ATTEMPTS" ]]; then + is_chronic=true + (( chronic++ )) + (( TOTAL_CHRONIC++ )) + warn " Chronic (${fail_count} consecutive failures) — needs manual review: $title" + [[ "$fail_count" -eq $(( ARR_RECOVERY_MAX_ATTEMPTS + 1 )) ]] && \ + notify "$arr_name item now chronic after ${ARR_RECOVERY_MAX_ATTEMPTS} failed attempts — needs manual review: $title" \ + "Arr Recovery" "warning" + fi + fi + + # Step 3: Trigger new search — skipped for chronic items + if [[ "$is_chronic" == true ]]; then + log " Skipping auto re-search (chronic): $title" + elif [[ -n "$media_id" ]]; then if trigger_search "$url" "$api_key" "$api_version" "$arr_type" "$media_id"; then log " New search triggered: $title" else @@ -389,12 +440,12 @@ process_arr() { done <<< "$problem_items" if [[ "$actioned" -gt 0 ]]; then - warn "$arr_name — actioned: $actioned | skipped (too new): $skipped_new" + warn "$arr_name — actioned: $actioned | skipped (too new): $skipped_new | chronic: $chronic" else log "$arr_name — nothing actioned | skipped (too new): $skipped_new" fi - ARR_SUMMARIES+=("$arr_name: actioned $actioned | too new $skipped_new") + ARR_SUMMARIES+=("$arr_name: actioned $actioned | too new $skipped_new | chronic $chronic") } # ============================================================================================== @@ -442,6 +493,15 @@ process_arr \ END=$(date +%s) +# Persist updated failure counts — skipped in dry-run so nothing is recorded for a preview +if [[ "$DRY_RUN" == false ]]; then + mkdir -p "$(dirname "$ARR_RECOVERY_FAILURE_COUNTS")" 2>/dev/null + : > "$ARR_RECOVERY_FAILURE_COUNTS" + for key in "${!FAILURE_COUNTS[@]}"; do + echo "${key}|${FAILURE_COUNTS[$key]}" >> "$ARR_RECOVERY_FAILURE_COUNTS" + done +fi + # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== @@ -451,6 +511,7 @@ echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "$ICON_TRASH Actioned: $TOTAL_ACTIONED items blocklisted + searched" echo "$ICON_SKIP Skipped: $TOTAL_SKIPPED items (too new)" +echo "$ICON_WARN Chronic: $TOTAL_CHRONIC items (blocklisted, auto re-search stopped)" echo "" for summary in "${ARR_SUMMARIES[@]}"; do echo " $ICON_SUMMARY $summary" @@ -470,7 +531,7 @@ echo "━━━━━━━━━━━━━━━━━━━━━━━━ # Write stats for sunday_morning_coffee_report.sh if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_RECOVERY_STATS:-}" ]]; then - echo "$(date '+%Y-%m-%d')|$(date '+%H:%M')|${TOTAL_ACTIONED}|${TOTAL_SKIPPED}" \ + echo "$(date '+%Y-%m-%d')|$(date '+%H:%M')|${TOTAL_ACTIONED}|${TOTAL_SKIPPED}|${TOTAL_CHRONIC}" \ >> "$ARR_RECOVERY_STATS" 2>/dev/null || true fi diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 7983113..43234d2 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -1215,6 +1215,8 @@ # Per-host recovery toggles (HOST1_SONARR_RECOVERY etc.) live in host*.conf. ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this # matches cron interval — items eligible after one missed cycle + ARR_RECOVERY_MAX_ATTEMPTS=3 # consecutive failures before an item is flagged chronic + # and auto re-search stops (still blocklisted/cleaned up) # ============================================================================================== # ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── @@ -1328,6 +1330,7 @@ # All in DATA_DIR — array always running when these are written. ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats + ARR_RECOVERY_FAILURE_COUNTS="$DATA_DIR/arr_recovery_failure_counts.db" # per-item chronic-failure tracking # ━━━ Health Digest ━━━ # Aggregated system health summary — reads existing state files, no new writes.