diff --git a/Docker_Essentials/downloaders_reset.sh b/Docker_Essentials/downloaders_reset.sh new file mode 100644 index 0000000..32469de --- /dev/null +++ b/Docker_Essentials/downloaders_reset.sh @@ -0,0 +1,433 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Downloaders Reset ------------------------------------------ +# ----------------------------------------------------------------------------------------------- +# Daily maintenance reset for all download clients. +# Clears stuck states, purges old history, and prepares downloaders for a clean daily cycle. +# Run before container restarts in daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS. +# +# Downloaders covered: +# slskd — clears stuck/errored searches, dead transfer records, +# purges expired failed imports (albums Lidarr rejected) +# SABnzbd — clears completed and failed history older than retention period, +# removes stalled/paused queue items +# qBittorrent — last chance failsafe delete for torrents older than +# QBIT_FAILSAFE_MIN_DAYS regardless of ratio +# +# Safety: +# Always --dry-run first before scheduling +# slskd transfer cleanup skips any user with InProgress or Queued transfers +# qBittorrent only deletes if torrent age exceeds QBIT_FAILSAFE_MIN_DAYS +# SABnzbd only deletes history older than DOWNLOADER_RETENTION_DAYS +# qBittorrent deleteFiles=false — removes from qBit, leaves files for arrs to manage +# +# Configuration in Master.conf under Docker Essentials — Downloaders Reset. +# Supports --dry-run to preview without making changes. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Downloaders Reset ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +detect_hosts + +acquire_lock "wait" + +START_TIME=$(date +%s) + +# Select host-specific config +if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then + SLSKD_URL="$HOST1_SLSKD_URL" + SLSKD_API_KEY="$HOST1_SLSKD_API_KEY" + SLSKD_FAILED_IMPORTS_DIR="$HOST1_SLSKD_FAILED_IMPORTS_DIR" + SABNZBD_URL="$HOST1_SABNZBD_URL" + SABNZBD_API_KEY="$HOST1_SABNZBD_API_KEY" + QBIT_URL="$HOST1_QBIT_URL" + QBIT_USERNAME="$HOST1_QBIT_USERNAME" + QBIT_PASSWORD="$HOST1_QBIT_PASSWORD" +else + # HOST2 placeholders — fill in when HOST2 is back online + SLSKD_URL="${HOST2_SLSKD_URL:-}" + SLSKD_API_KEY="${HOST2_SLSKD_API_KEY:-}" + SLSKD_FAILED_IMPORTS_DIR="${HOST2_SLSKD_FAILED_IMPORTS_DIR:-}" + SABNZBD_URL="${HOST2_SABNZBD_URL:-}" + SABNZBD_API_KEY="${HOST2_SABNZBD_API_KEY:-}" + QBIT_URL="${HOST2_QBIT_URL:-}" + QBIT_USERNAME="${HOST2_QBIT_USERNAME:-}" + QBIT_PASSWORD="${HOST2_QBIT_PASSWORD:-}" +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" + +CUTOFF=$(( $(date +%s) - (DOWNLOADER_RETENTION_DAYS * 86400) )) +TOTAL_PASS=0 +TOTAL_FAIL=0 + +# ----------------------------------------------------------------------------------------------- +# ━━━ slskd — Stuck Searches ━━━ +# Clears searches in Completed/Errored state left by Soularr crashes +# Prevents 409 Conflict error on next Soularr startup +# ----------------------------------------------------------------------------------------------- +if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then + echo "" + echo "━━━ 🔍 slskd — Stuck Searches ━━━" + + SEARCHES=$(curl -sf --max-time 10 -X GET "$SLSKD_URL/api/v0/searches" \ + -H "X-Api-Key: $SLSKD_API_KEY" 2>/dev/null) + + if [[ -z "$SEARCHES" ]]; then + warn "slskd not reachable — skipping searches" + else + IDS=$(echo "$SEARCHES" | grep -o '"id":"[^"]*","isComplete":true' | \ + grep -o '"id":"[^"]*"' | sed 's/"id":"//;s/"//') + COUNT=$(echo "$IDS" | grep -c . 2>/dev/null || echo 0) + COUNT="${COUNT//[^0-9]/}"; COUNT="${COUNT:-0}" + + if [[ "$COUNT" -eq 0 ]]; then + success "No stuck searches found ✅" + else + info "Found $COUNT stuck search(es)" + SUCCESS=0; FAIL=0 + while IFS= read -r ID; do + [[ -z "$ID" ]] && continue + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would delete search: $ID" + ((SUCCESS++)) + continue + fi + RESULT=$(curl -sf --max-time 10 -o /dev/null -w "%{http_code}" -X DELETE \ + "$SLSKD_URL/api/v0/searches/$ID" \ + -H "X-Api-Key: $SLSKD_API_KEY") + if [[ "$RESULT" == "200" || "$RESULT" == "204" ]]; then + info "$ICON_TRASH Cleared search: $ID" + ((SUCCESS++)) + else + error "Failed: $ID (HTTP $RESULT)" + ((FAIL++)) + fi + done <<< "$IDS" + success "Searches: $SUCCESS cleared, $FAIL failed" + (( TOTAL_FAIL += FAIL )) + (( TOTAL_PASS += SUCCESS )) + fi + fi +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ slskd — Dead Transfer Records ━━━ +# Removes completed/errored/aborted transfer records per user +# Prevents Soularr 404 loop when polling a user whose transfer no longer exists +# NEVER removes transfers that are InProgress or Queued +# ----------------------------------------------------------------------------------------------- +if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then + echo "" + echo "━━━ 🔍 slskd — Dead Transfer Records ━━━" + + TRANSFERS=$(curl -sf --max-time 10 -X GET "$SLSKD_URL/api/v0/transfers/downloads" \ + -H "X-Api-Key: $SLSKD_API_KEY" 2>/dev/null) + + if [[ -z "$TRANSFERS" ]]; then + warn "slskd not reachable — skipping transfers" + else + USERNAMES=$(echo "$TRANSFERS" | grep -o '"username":"[^"]*"' | \ + sed 's/"username":"//;s/"//' | sort -u) + + if [[ -z "$USERNAMES" ]]; then + success "No transfer records found ✅" + else + SUCCESS=0; SKIPPED=0; FAIL=0 + while IFS= read -r USER; do + [[ -z "$USER" ]] && continue + ACTIVE=$(echo "$TRANSFERS" | grep -o "\"username\":\"$USER\"[^}]*\"state\":\"[^\"]*\"" | \ + grep -c "InProgress\|Queued") + if [[ "$ACTIVE" -gt 0 ]]; then + info "$ICON_SKIP Skipping $USER — $ACTIVE active/queued transfer(s)" + ((SKIPPED++)) + continue + fi + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would clear transfers for: $USER" + ((SUCCESS++)) + continue + fi + RESULT=$(curl -sf --max-time 10 -o /dev/null -w "%{http_code}" -X DELETE \ + "$SLSKD_URL/api/v0/transfers/downloads/$USER" \ + -H "X-Api-Key: $SLSKD_API_KEY") + if [[ "$RESULT" == "200" || "$RESULT" == "204" ]]; then + info "$ICON_TRASH Cleared transfers for: $USER" + ((SUCCESS++)) + else + error "Failed to clear: $USER (HTTP $RESULT)" + ((FAIL++)) + fi + done <<< "$USERNAMES" + success "Transfers: $SUCCESS cleared, $SKIPPED skipped (active), $FAIL failed" + (( TOTAL_FAIL += FAIL )) + (( TOTAL_PASS += SUCCESS )) + fi + fi +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ slskd — Purge Expired Failed Imports ━━━ +# Removes albums Soularr downloaded but Lidarr rejected +# Soularr moves these to failed_imports/ and never cleans them up +# ----------------------------------------------------------------------------------------------- +if [[ -n "$SLSKD_FAILED_IMPORTS_DIR" ]]; then + echo "" + echo "━━━ 🔍 slskd — Failed Imports (older than ${DOWNLOADER_RETENTION_DAYS} days) ━━━" + + if [[ ! -d "$SLSKD_FAILED_IMPORTS_DIR" ]]; then + warn "Directory not found: $SLSKD_FAILED_IMPORTS_DIR — skipping" + else + OLD_IMPORTS=$(find "$SLSKD_FAILED_IMPORTS_DIR" \ + -mindepth 1 -maxdepth 1 -mtime +"${DOWNLOADER_RETENTION_DAYS}") + IMPORT_COUNT=$(echo "$OLD_IMPORTS" | grep -c . 2>/dev/null || echo 0) + IMPORT_COUNT="${IMPORT_COUNT//[^0-9]/}"; IMPORT_COUNT="${IMPORT_COUNT:-0}" + + if [[ "$IMPORT_COUNT" -eq 0 ]]; then + success "No expired failed imports found ✅" + else + info "Found $IMPORT_COUNT expired failed import(s)" + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would delete:" + echo "$OLD_IMPORTS" + else + find "$SLSKD_FAILED_IMPORTS_DIR" \ + -mindepth 1 -maxdepth 1 -mtime +"${DOWNLOADER_RETENTION_DAYS}" \ + -exec rm -rf {} \; + success "$ICON_TRASH Purged $IMPORT_COUNT expired failed import(s)" + (( TOTAL_PASS += IMPORT_COUNT )) + fi + fi + fi +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ SABnzbd — Clear Completed History ━━━ +# Removes completed download history older than DOWNLOADER_RETENTION_DAYS +# ----------------------------------------------------------------------------------------------- +if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then + echo "" + echo "━━━ 🔍 SABnzbd — Completed History (older than ${DOWNLOADER_RETENTION_DAYS} days) ━━━" + + HISTORY=$(curl -sf --max-time 15 \ + "$SABNZBD_URL/api?mode=history&output=json&limit=1000&apikey=$SABNZBD_API_KEY" 2>/dev/null) + + if [[ -z "$HISTORY" ]]; then + warn "SABnzbd not reachable — skipping completed history" + else + COMPLETED_IDS=$(echo "$HISTORY" | grep -o '"nzo_id":"[^"]*"' | \ + sed 's/"nzo_id":"//;s/"//') + + if [[ -z "$COMPLETED_IDS" ]]; then + success "No completed history found ✅" + else + DELETED=0; SKIPPED=0 + while IFS= read -r NZO_ID; do + [[ -z "$NZO_ID" ]] && continue + JOB_TIME=$(echo "$HISTORY" | grep -A5 "$NZO_ID" | \ + grep -o '"completed":[0-9]*' | grep -o '[0-9]*') + [[ -z "$JOB_TIME" ]] && continue + [[ "$JOB_TIME" -gt "$CUTOFF" ]] && ((SKIPPED++)) && continue + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would delete completed job: $NZO_ID" + ((DELETED++)) + else + curl -sf --max-time 10 \ + "$SABNZBD_URL/api?mode=history&name=delete&value=$NZO_ID&apikey=$SABNZBD_API_KEY" \ + >/dev/null + info "$ICON_TRASH Deleted: $NZO_ID" + ((DELETED++)) + fi + done <<< "$COMPLETED_IDS" + success "Completed: $DELETED deleted, $SKIPPED within retention" + (( TOTAL_PASS += DELETED )) + fi + fi +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ SABnzbd — Clear Failed History ━━━ +# Removes failed download history older than DOWNLOADER_RETENTION_DAYS +# ----------------------------------------------------------------------------------------------- +if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then + echo "" + echo "━━━ 🔍 SABnzbd — Failed History (older than ${DOWNLOADER_RETENTION_DAYS} days) ━━━" + + FAILED_HIST=$(curl -sf --max-time 15 \ + "$SABNZBD_URL/api?mode=history&output=json&limit=1000&failed_only=1&apikey=$SABNZBD_API_KEY" 2>/dev/null) + + if [[ -z "$FAILED_HIST" ]]; then + warn "SABnzbd not reachable — skipping failed history" + else + FAILED_IDS=$(echo "$FAILED_HIST" | grep -o '"nzo_id":"[^"]*"' | \ + sed 's/"nzo_id":"//;s/"//') + + if [[ -z "$FAILED_IDS" ]]; then + success "No failed history found ✅" + else + DELETED=0; SKIPPED=0 + while IFS= read -r NZO_ID; do + [[ -z "$NZO_ID" ]] && continue + JOB_TIME=$(echo "$FAILED_HIST" | grep -A5 "$NZO_ID" | \ + grep -o '"completed":[0-9]*' | grep -o '[0-9]*') + [[ -z "$JOB_TIME" ]] && continue + [[ "$JOB_TIME" -gt "$CUTOFF" ]] && ((SKIPPED++)) && continue + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would delete failed job: $NZO_ID" + ((DELETED++)) + else + curl -sf --max-time 10 \ + "$SABNZBD_URL/api?mode=history&name=delete&value=$NZO_ID&apikey=$SABNZBD_API_KEY" \ + >/dev/null + info "$ICON_TRASH Deleted: $NZO_ID" + ((DELETED++)) + fi + done <<< "$FAILED_IDS" + success "Failed: $DELETED deleted, $SKIPPED within retention" + (( TOTAL_PASS += DELETED )) + fi + fi +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ SABnzbd — Remove Stalled Queue Items ━━━ +# Removes paused queue items no longer progressing +# Active downloading items are never touched +# ----------------------------------------------------------------------------------------------- +if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then + echo "" + echo "━━━ 🔍 SABnzbd — Stalled Queue Items ━━━" + + QUEUE=$(curl -sf --max-time 10 \ + "$SABNZBD_URL/api?mode=queue&output=json&apikey=$SABNZBD_API_KEY" 2>/dev/null) + + if [[ -z "$QUEUE" ]]; then + warn "SABnzbd not reachable — skipping queue" + else + STALLED_IDS=$(echo "$QUEUE" | grep -o '"nzo_id":"[^"]*"' | \ + sed 's/"nzo_id":"//;s/"//') + + if [[ -z "$STALLED_IDS" ]]; then + success "No stalled queue items found ✅" + else + DELETED=0; SKIPPED=0 + while IFS= read -r NZO_ID; do + [[ -z "$NZO_ID" ]] && continue + STATUS=$(echo "$QUEUE" | grep -A10 "$NZO_ID" | \ + grep -o '"status":"[^"]*"' | sed 's/"status":"//;s/"//') + [[ "$STATUS" != "Paused" ]] && ((SKIPPED++)) && continue + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would remove stalled item: $NZO_ID" + ((DELETED++)) + else + curl -sf --max-time 10 \ + "$SABNZBD_URL/api?mode=queue&name=delete&value=$NZO_ID&apikey=$SABNZBD_API_KEY" \ + >/dev/null + info "$ICON_TRASH Removed stalled: $NZO_ID" + ((DELETED++)) + fi + done <<< "$STALLED_IDS" + success "Queue: $DELETED removed, $SKIPPED active (skipped)" + (( TOTAL_PASS += DELETED )) + fi + fi +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ qBittorrent — Last Chance Failsafe Cleanup ━━━ +# Deletes torrents older than QBIT_FAILSAFE_MIN_DAYS +# deleteFiles=false — removes from qBit, leaves files for arrs to manage +# ----------------------------------------------------------------------------------------------- +if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then + echo "" + echo "━━━ 🔍 qBittorrent — Failsafe (older than ${QBIT_FAILSAFE_MIN_DAYS} days) ━━━" + [[ "$QBIT_FAILSAFE_MIN_RATIO" != "0" ]] && \ + info "Ratio requirement: >= ${QBIT_FAILSAFE_MIN_RATIO}" + + QBIT_COOKIE=$(curl -sf --max-time 10 -c - \ + "$QBIT_URL/api/v2/auth/login" \ + --data "username=$QBIT_USERNAME&password=$QBIT_PASSWORD" 2>/dev/null | \ + grep SID | awk '{print "SID="$NF}') + + if [[ -z "$QBIT_COOKIE" ]]; then + error "Failed to authenticate with qBittorrent" + ((TOTAL_FAIL++)) + else + TORRENTS=$(curl -sf --max-time 15 \ + "$QBIT_URL/api/v2/torrents/info" \ + -H "Cookie: $QBIT_COOKIE" 2>/dev/null) + + NOW=$(date +%s) + DELETED=0; SKIPPED=0 + + while read -r TORRENT; do + [[ -z "$TORRENT" ]] && continue + HASH=$(echo "$TORRENT" | grep -o '"hash":"[^"]*"' | sed 's/"hash":"//;s/"//') + NAME=$(echo "$TORRENT" | grep -o '"name":"[^"]*"' | sed 's/"name":"//;s/"//') + ADDED=$(echo "$TORRENT" | grep -o '"added_on":[0-9]*' | grep -o '[0-9]*') + RATIO=$(echo "$TORRENT" | grep -o '"ratio":[0-9.]*' | grep -o '[0-9.]*') + [[ -z "$HASH" || -z "$ADDED" ]] && continue + + AGE_DAYS=$(( (NOW - ADDED) / 86400 )) + [[ "$AGE_DAYS" -lt "$QBIT_FAILSAFE_MIN_DAYS" ]] && ((SKIPPED++)) && continue + + if [[ "$QBIT_FAILSAFE_MIN_RATIO" != "0" ]]; then + RATIO_INT="${RATIO%.*}" + MIN_RATIO_INT="${QBIT_FAILSAFE_MIN_RATIO%.*}" + [[ "$RATIO_INT" -lt "$MIN_RATIO_INT" ]] && ((SKIPPED++)) && continue + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would delete: $NAME (${AGE_DAYS}d old, ratio: $RATIO)" + ((DELETED++)) + else + curl -sf --max-time 10 -X POST \ + "$QBIT_URL/api/v2/torrents/delete" \ + -H "Cookie: $QBIT_COOKIE" \ + --data "hashes=$HASH&deleteFiles=false" >/dev/null + info "$ICON_TRASH Deleted: $NAME (${AGE_DAYS}d old, ratio: $RATIO)" + ((DELETED++)) + fi + done < <(echo "$TORRENTS" | tr '}' '\n') + + success "qBittorrent: $DELETED deleted, $SKIPPED skipped (under threshold)" + (( TOTAL_PASS += DELETED )) + fi +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━━━ $ICON_SUMMARY DOWNLOADERS RESET SUMMARY ━━━━━" +echo "$ICON_TIME Duration: $(format_duration $(( $(date +%s) - START_TIME )))" +echo "$ICON_SUCCESS Actions: $TOTAL_PASS" +echo "$ICON_ERROR Failures: $TOTAL_FAIL" +echo "" + +if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Status: DRY RUN — no changes made" +elif [[ "$TOTAL_FAIL" -gt 0 ]]; then + echo "$ICON_ERROR Status: $TOTAL_FAIL failure(s) — check logs" + notify "Downloaders reset completed with failures on $(hostname)" "Downloaders Reset" "warning" + exit 1 +else + echo "$ICON_DONE Status: $ICON_SUCCESS DONE" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Master.conf b/Master.conf index 6f3998e..0100930 100644 --- a/Master.conf +++ b/Master.conf @@ -163,7 +163,8 @@ ARRAY_START_SCRIPTS=( "Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts "unRAID_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill "unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning -# "Docker_Essentials/docker_network_connect.sh" # connect containers to extra networks + "unRAID_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted + "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers "unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop "Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop # "Failover/failover.sh" # mutual failover — continuous loop @@ -218,6 +219,22 @@ HOST2_PERSONAL_SHARES=( # /mnt/user/Jayred365-Personal # uncomment after creating encrypted dataset ) +# ━━━ Media Management ━━━ +# Job list run directly by daily_sync_maintenance.sh after the media share sync. +# Runs sequentially — permissions first, then cleaners, then arr cleanup. +# Comment out any job to disable without removing it. +# Each individual script can still be run manually for one-off maintenance. + +MEDIA_MANAGEMENT_JOBS=( + "Media/media_shares_permissions.sh" # apply permissions — runs first + "Media/media_cleaner.sh anime" # remove junk from anime shares + "Media/media_cleaner.sh media" # remove junk from media shares +# "Media/lidarr_cleanup.sh" # remove orphaned music files +# "Media/sonarr_cleanup.sh" # remove orphaned TV files +# "Media/radarr_cleanup.sh" # remove orphaned movie files + "Docker_Essentials/downloaders_reset.sh" # clear stuck states + purge old history +) + # ━━━ Weekly Sync Maintenance ━━━ # weekly_sync_maintenance.sh handles the critical sync built into the script first: # stop containers both sides → pull updates → sync Emby + Critical-Data → restart @@ -244,21 +261,6 @@ WEEKLY_SYNC_JOBS=( CRITICAL_SYNC_UPDATES=true # pull container updates locally CRITICAL_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH -# ━━━ Media Management ━━━ -# Job list run directly by daily_sync_maintenance.sh after the media share sync. -# Runs sequentially — permissions first, then cleaners, then arr cleanup. -# Comment out any job to disable without removing it. -# Each individual script can still be run manually for one-off maintenance. - -MEDIA_MANAGEMENT_JOBS=( - "Media/media_shares_permissions.sh" # apply permissions — runs first - "Media/media_cleaner.sh anime" # remove junk from anime shares - "Media/media_cleaner.sh media" # remove junk from media shares -# "Media/lidarr_cleanup.sh" # remove orphaned music files -# "Media/sonarr_cleanup.sh" # remove orphaned TV files -# "Media/radarr_cleanup.sh" # remove orphaned movie files -) - # ============================================================================================== # ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -592,6 +594,39 @@ FAILOVER_HOST2_WRITEBACK_TIER4=( # ── DOCKER ESSENTIALS ───────────────────────────────────────────────────────────────────────── # ============================================================================================== +# ━━━ Downloaders Reset ━━━ +# Daily maintenance reset for all download clients. +# Called by daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS before container restarts. +# Clears stuck states, purges old history, prepares each downloader for a clean daily cycle. +# +# Retention period — applies to: slskd failed imports, SABnzbd completed and failed history + DOWNLOADER_RETENTION_DAYS=7 + +# ── slskd ── +# Clears stuck/errored searches, dead transfer records, purges expired failed imports +# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected + HOST1_SLSKD_URL="http://localhost:8980" + HOST1_SLSKD_API_KEY="4bF9kL2mNpQrT7vWxYz1A3dEgHjKoRsU" + HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports" + +# ── SABnzbd ── +# Clears completed history, failed history, and stalled paused queue items + HOST1_SABNZBD_URL="http://localhost:8180" + HOST1_SABNZBD_API_KEY="8bfefe41d83b4d50883e32859b55ca9a" + +# ── qBittorrent ── +# Last chance failsafe — deletes torrents older than QBIT_FAILSAFE_MIN_DAYS +# qBittorrent's own rules handle normal cleanup (ratio >= 1.25 OR 45 days inactive) +# This catches anything missed after extended time +# deleteFiles=false — removes torrent from qBit but leaves files on disk +# Radarr/Sonarr manage actual files independently +# QBIT_FAILSAFE_MIN_RATIO=0 disables ratio gate — age is the only condition + HOST1_QBIT_URL="http://localhost:8080" + HOST1_QBIT_USERNAME="admin" + HOST1_QBIT_PASSWORD="changeme" + QBIT_FAILSAFE_MIN_DAYS=180 + QBIT_FAILSAFE_MIN_RATIO=0 # 0 = age only, no ratio requirement + # ━━━ Docker Daily Restart ━━━ # Containers restarted every day by docker_daily_restart.sh via daily_sync_maintenance.sh. # These containers run better with a daily restart — not just "keeping things fresh". @@ -782,6 +817,16 @@ NETWORK_CONNECT_NETWORKS=( # ── UNRAID ESSENTIALS ───────────────────────────────────────────────────────────────────────── # ============================================================================================== +# ━━━ inotify Tuning ━━━ +# Linux inotify limits — applied at every array start by inotify_tuning.sh +# Default unRAID values are very low — with many Docker containers watching files +# (Sonarr, Radarr, Lidarr, NextCloud etc.) you can silently exhaust the limit. +# Symptoms: containers miss file events, downloads not detected, library not updated. +# These settings are lost on reboot — reapplied automatically at array start. + INOTIFY_MAX_INSTANCES=512 # default: 128 — max inotify instances per user + INOTIFY_MAX_WATCHES=524288 # default: 8192 — max files watched per instance + INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — max events queued before dropping + # ━━━ Reboot ━━━ # Seconds of warning broadcast to logged-in users before server_reboot.sh reboots. # Gives users time to save work — 300s = 5 minutes @@ -949,7 +994,7 @@ LIDARR_PROTECTED_PATTERNS=( # NEVER deleted — cover art, metadata, lyrics # Lidarr generates these but doesn't include them in trackFile API # Without this protection cleanup would delete all your artwork -LIDARR_MAX_DELETE_GB=5 # require --i-know-what-im-doing if deletion exceeds this +LIDARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run # protects against API returning partial data on a bad day LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count" @@ -980,7 +1025,7 @@ declare -A HOST2_SONARR_PATH_MAP=( ) SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion -SONARR_MAX_DELETE_GB=10 # require --i-know-what-im-doing if deletion exceeds this +SONARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") SONARR_PROTECTED_PATTERNS=( # Subtitles @@ -1024,7 +1069,7 @@ declare -A HOST2_RADARR_PATH_MAP=( ) RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion -RADARR_MAX_DELETE_GB=15 # require --i-know-what-im-doing if deletion exceeds this +RADARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") RADARR_PROTECTED_PATTERNS=( # Subtitles diff --git a/Tools/downloaders_reset.sh b/Tools/downloaders_reset.sh deleted file mode 100644 index 6655033..0000000 --- a/Tools/downloaders_reset.sh +++ /dev/null @@ -1,420 +0,0 @@ -#!/bin/bash -# ============================================================================================== -# downloaders_reset.sh -# ============================================================================================== -# PURPOSE: Daily maintenance reset for all download clients. -# Clears stuck states, purges old history, and prepares downloaders for a -# clean daily cycle. Run before container restarts in daily maintenance. -# -# DOWNLOADERS COVERED: -# slskd — clears stuck/errored searches, dead transfer records, -# purges expired failed imports -# SABnzbd — clears completed and failed history older than retention period, -# removes stalled/paused queue items -# qBittorrent — last chance failsafe delete for torrents older than -# QBIT_FAILSAFE_MIN_DAYS regardless of ratio -# -# USAGE: -# bash downloaders_reset.sh # live run -# bash downloaders_reset.sh --dry-run # preview only, no changes made -# -# SCHEDULE: Run before downloader restarts in daily_sync_maintenance.sh -# Ensures all downloaders start each day in a clean known state -# -# SAFETY: -# Always --dry-run first before scheduling -# slskd transfer cleanup skips any user with InProgress or Queued transfers -# qBittorrent only deletes if torrent age exceeds QBIT_FAILSAFE_MIN_DAYS -# SABnzbd only deletes history older than RETENTION_DAYS -# -# TO ADD A NEW DOWNLOADER: -# Add URL, API key, and any specific variables in Configuration section -# Add a new section block following the same pattern as existing downloaders -# ============================================================================================== - -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ━━━ Configuration ━━━ -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -# Retention period in days — applies to ALL cleanup sections: -# slskd failed imports, SABnzbd completed history, SABnzbd failed history -RETENTION_DAYS=7 - -# ━━━ slskd ━━━ -SLSKD_URL="http://localhost:8980" -SLSKD_API_KEY="4bF9kL2mNpQrT7vWxYz1A3dEgHjKoRsU" - -# Path to Soularr failed imports folder -# Created by Soularr when Lidarr rejects a downloaded album -SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports" - -# ━━━ SABnzbd ━━━ -SABNZBD_URL="http://localhost:8180" -SABNZBD_API_KEY="8bfefe41d83b4d50883e32859b55ca9a" - -# ━━━ qBittorrent ━━━ -QBIT_URL="http://localhost:8080" -QBIT_USERNAME="admin" -QBIT_PASSWORD="changeme" - -# Last chance failsafe — deletes torrents older than this many days. -# qBittorrent's own rules handle normal cleanup (ratio >= 1.25 OR 45 days -# inactive). This catches anything missed after an extended period. -# QBIT_FAILSAFE_MIN_RATIO=0 disables ratio gate — age is the only condition. -QBIT_FAILSAFE_MIN_DAYS=180 -QBIT_FAILSAFE_MIN_RATIO=0 - -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -DRY_RUN=false -[[ "$1" == "--dry-run" ]] && DRY_RUN=true - -$DRY_RUN && echo "🧪 DRY RUN MODE — no changes will be made" -echo "" - -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ━━━ slskd — Clear Stuck Searches ━━━ -# Clears searches in Completed/Errored state left behind by Soularr crashes. -# Prevents 409 Conflict error on next Soularr startup. -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo "🔍 slskd — Stuck Searches" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -SEARCHES=$(curl -s -X GET "$SLSKD_URL/api/v0/searches" \ - -H "X-Api-Key: $SLSKD_API_KEY") - -IDS=$(echo "$SEARCHES" | grep -o '"id":"[^"]*","isComplete":true' | \ - grep -o '"id":"[^"]*"' | sed 's/"id":"//;s/"//') -COUNT=$(echo "$IDS" | grep -c . 2>/dev/null || echo 0) - -if [[ -z "$IDS" || "$COUNT" -eq 0 ]]; then - echo "✅ No stuck searches found" -else - echo "📋 Found $COUNT stuck search(es)" - if $DRY_RUN; then - echo "🧪 Would delete search IDs:" - echo "$IDS" - else - SUCCESS=0 - FAIL=0 - while IFS= read -r ID; do - [[ -z "$ID" ]] && continue - RESULT=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE \ - "$SLSKD_URL/api/v0/searches/$ID" \ - -H "X-Api-Key: $SLSKD_API_KEY") - if [[ "$RESULT" == "200" || "$RESULT" == "204" ]]; then - echo " 🗑️ Cleared search: $ID" - ((SUCCESS++)) - else - echo " ❌ Failed: $ID (HTTP $RESULT)" - ((FAIL++)) - fi - done <<< "$IDS" - echo "✅ Searches: $SUCCESS cleared, $FAIL failed" - fi -fi - -echo "" - -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ━━━ slskd — Clear Dead Transfer Records ━━━ -# Removes completed/errored/aborted transfer records per user. -# Prevents Soularr 404 loop when polling a user whose transfer -# record no longer exists after a crash or disconnect. -# NEVER removes transfers that are InProgress or Queued. -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo "🔍 slskd — Dead Transfer Records" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -TRANSFERS=$(curl -s -X GET "$SLSKD_URL/api/v0/transfers/downloads" \ - -H "X-Api-Key: $SLSKD_API_KEY") - -USERNAMES=$(echo "$TRANSFERS" | grep -o '"username":"[^"]*"' | \ - sed 's/"username":"//;s/"//' | sort -u) - -if [[ -z "$USERNAMES" ]]; then - echo "✅ No transfer records found" -else - SUCCESS=0 - SKIPPED=0 - FAIL=0 - - while IFS= read -r USER; do - [[ -z "$USER" ]] && continue - - USER_STATES=$(echo "$TRANSFERS" | grep -o \ - "\"username\":\"$USER\"[^}]*\"state\":\"[^\"]*\"" | \ - grep -o '"state":"[^"]*"') - - ACTIVE=$(echo "$USER_STATES" | grep -c "InProgress\|Queued") - - if [[ "$ACTIVE" -gt 0 ]]; then - echo " ⏳ Skipping $USER — $ACTIVE active/queued transfer(s)" - ((SKIPPED++)) - continue - fi - - if $DRY_RUN; then - echo "🧪 Would clear transfers for: $USER" - ((SUCCESS++)) - continue - fi - - RESULT=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE \ - "$SLSKD_URL/api/v0/transfers/downloads/$USER" \ - -H "X-Api-Key: $SLSKD_API_KEY") - - if [[ "$RESULT" == "200" || "$RESULT" == "204" ]]; then - echo " 🗑️ Cleared transfers for: $USER" - ((SUCCESS++)) - else - echo " ❌ Failed to clear: $USER (HTTP $RESULT)" - ((FAIL++)) - fi - done <<< "$USERNAMES" - - echo "✅ Transfers: $SUCCESS cleared, $SKIPPED skipped (active), $FAIL failed" -fi - -echo "" - -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ━━━ slskd — Purge Expired Failed Imports ━━━ -# Removes albums Soularr downloaded but Lidarr rejected. -# Soularr moves these to failed_imports/ and never cleans them up. -# Retention controlled by RETENTION_DAYS variable. -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo "🔍 slskd — Failed Imports (older than ${RETENTION_DAYS} days)" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -if [[ ! -d "$SLSKD_FAILED_IMPORTS_DIR" ]]; then - echo "⚠️ Directory not found: $SLSKD_FAILED_IMPORTS_DIR" -else - OLD_IMPORTS=$(find "$SLSKD_FAILED_IMPORTS_DIR" \ - -mindepth 1 -maxdepth 1 -mtime +${RETENTION_DAYS}) - IMPORT_COUNT=$(echo "$OLD_IMPORTS" | grep -c . 2>/dev/null || echo 0) - - if [[ -z "$OLD_IMPORTS" || "$IMPORT_COUNT" -eq 0 ]]; then - echo "✅ No expired failed imports found" - else - echo "📋 Found $IMPORT_COUNT expired failed import(s)" - if $DRY_RUN; then - echo "🧪 Would delete:" - echo "$OLD_IMPORTS" - else - find "$SLSKD_FAILED_IMPORTS_DIR" \ - -mindepth 1 -maxdepth 1 -mtime +${RETENTION_DAYS} \ - -exec rm -rf {} \; - echo "🗑️ Purged $IMPORT_COUNT expired failed import(s)" - fi - fi -fi - -echo "" - -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ━━━ SABnzbd — Clear Completed History ━━━ -# Removes completed download history older than RETENTION_DAYS. -# Keeps history clean without losing recent job records. -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo "🔍 SABnzbd — Completed History (older than ${RETENTION_DAYS} days)" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -CUTOFF=$(( $(date +%s) - (RETENTION_DAYS * 86400) )) - -HISTORY=$(curl -s \ - "$SABNZBD_URL/api?mode=history&output=json&limit=1000&apikey=$SABNZBD_API_KEY") - -COMPLETED_IDS=$(echo "$HISTORY" | grep -o '"nzo_id":"[^"]*"' | \ - sed 's/"nzo_id":"//;s/"//') - -if [[ -z "$COMPLETED_IDS" ]]; then - echo "✅ No completed history found" -else - DELETED=0 - SKIPPED=0 - while IFS= read -r NZO_ID; do - [[ -z "$NZO_ID" ]] && continue - JOB_TIME=$(echo "$HISTORY" | grep -A5 "$NZO_ID" | \ - grep -o '"completed":[0-9]*' | grep -o '[0-9]*') - [[ -z "$JOB_TIME" ]] && continue - [[ "$JOB_TIME" -gt "$CUTOFF" ]] && ((SKIPPED++)) && continue - if $DRY_RUN; then - echo "🧪 Would delete completed job: $NZO_ID" - ((DELETED++)) - else - curl -s \ - "$SABNZBD_URL/api?mode=history&name=delete&value=$NZO_ID&apikey=$SABNZBD_API_KEY" \ - > /dev/null - echo " 🗑️ Deleted: $NZO_ID" - ((DELETED++)) - fi - done <<< "$COMPLETED_IDS" - echo "✅ Completed: $DELETED deleted, $SKIPPED within retention period" -fi - -echo "" - -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ━━━ SABnzbd — Clear Failed History ━━━ -# Removes failed download history older than RETENTION_DAYS. -# Failed jobs older than retention period serve no purpose. -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo "🔍 SABnzbd — Failed History (older than ${RETENTION_DAYS} days)" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -FAILED=$(curl -s \ - "$SABNZBD_URL/api?mode=history&output=json&limit=1000&failed_only=1&apikey=$SABNZBD_API_KEY") - -FAILED_IDS=$(echo "$FAILED" | grep -o '"nzo_id":"[^"]*"' | \ - sed 's/"nzo_id":"//;s/"//') - -if [[ -z "$FAILED_IDS" ]]; then - echo "✅ No failed history found" -else - DELETED=0 - SKIPPED=0 - while IFS= read -r NZO_ID; do - [[ -z "$NZO_ID" ]] && continue - JOB_TIME=$(echo "$FAILED" | grep -A5 "$NZO_ID" | \ - grep -o '"completed":[0-9]*' | grep -o '[0-9]*') - [[ -z "$JOB_TIME" ]] && continue - [[ "$JOB_TIME" -gt "$CUTOFF" ]] && ((SKIPPED++)) && continue - if $DRY_RUN; then - echo "🧪 Would delete failed job: $NZO_ID" - ((DELETED++)) - else - curl -s \ - "$SABNZBD_URL/api?mode=history&name=delete&value=$NZO_ID&apikey=$SABNZBD_API_KEY" \ - > /dev/null - echo " 🗑️ Deleted: $NZO_ID" - ((DELETED++)) - fi - done <<< "$FAILED_IDS" - echo "✅ Failed: $DELETED deleted, $SKIPPED within retention period" -fi - -echo "" - -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ━━━ SABnzbd — Remove Stalled Queue Items ━━━ -# Removes paused queue items that are no longer progressing. -# Active downloading items are never touched. -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo "🔍 SABnzbd — Stalled Queue Items" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -QUEUE=$(curl -s \ - "$SABNZBD_URL/api?mode=queue&output=json&apikey=$SABNZBD_API_KEY") - -STALLED_IDS=$(echo "$QUEUE" | grep -o '"nzo_id":"[^"]*"' | \ - sed 's/"nzo_id":"//;s/"//') - -if [[ -z "$STALLED_IDS" ]]; then - echo "✅ No stalled queue items found" -else - DELETED=0 - SKIPPED=0 - while IFS= read -r NZO_ID; do - [[ -z "$NZO_ID" ]] && continue - STATUS=$(echo "$QUEUE" | grep -A10 "$NZO_ID" | \ - grep -o '"status":"[^"]*"' | sed 's/"status":"//;s/"//') - [[ "$STATUS" != "Paused" ]] && ((SKIPPED++)) && continue - if $DRY_RUN; then - echo "🧪 Would remove stalled item: $NZO_ID" - ((DELETED++)) - else - curl -s \ - "$SABNZBD_URL/api?mode=queue&name=delete&value=$NZO_ID&apikey=$SABNZBD_API_KEY" \ - > /dev/null - echo " 🗑️ Removed: $NZO_ID" - ((DELETED++)) - fi - done <<< "$STALLED_IDS" - echo "✅ Queue: $DELETED removed, $SKIPPED active (skipped)" -fi - -echo "" - -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ━━━ qBittorrent — Last Chance Failsafe Cleanup ━━━ -# Deletes torrents older than QBIT_FAILSAFE_MIN_DAYS. -# qBittorrent's own rules handle normal cleanup (ratio >= 1.25 OR 45 days -# inactive). This catches anything missed after extended time. -# QBIT_FAILSAFE_MIN_RATIO=0 disables ratio gate — age is the only condition. -# deleteFiles=false — removes torrent from qBit but leaves files on disk. -# Radarr/Sonarr manage the actual files independently. -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo "🔍 qBittorrent — Failsafe Cleanup" -echo " Age > ${QBIT_FAILSAFE_MIN_DAYS} days" -[[ "$QBIT_FAILSAFE_MIN_RATIO" != "0" ]] && \ - echo " Ratio >= ${QBIT_FAILSAFE_MIN_RATIO}" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -QBIT_COOKIE=$(curl -s -c - \ - "$QBIT_URL/api/v2/auth/login" \ - --data "username=$QBIT_USERNAME&password=$QBIT_PASSWORD" | \ - grep SID | awk '{print "SID="$NF}') - -if [[ -z "$QBIT_COOKIE" ]]; then - echo "❌ Failed to authenticate with qBittorrent" -else - TORRENTS=$(curl -s \ - "$QBIT_URL/api/v2/torrents/info" \ - -H "Cookie: $QBIT_COOKIE") - - NOW=$(date +%s) - DELETED=0 - SKIPPED=0 - - # Each torrent is one JSON object — split on },{ to process individually - echo "$TORRENTS" | tr '}' '\n' | while read -r TORRENT; do - [[ -z "$TORRENT" ]] && continue - - HASH=$(echo "$TORRENT" | grep -o '"hash":"[^"]*"' | \ - sed 's/"hash":"//;s/"//') - NAME=$(echo "$TORRENT" | grep -o '"name":"[^"]*"' | \ - sed 's/"name":"//;s/"//') - ADDED=$(echo "$TORRENT" | grep -o '"added_on":[0-9]*' | \ - grep -o '[0-9]*') - RATIO=$(echo "$TORRENT" | grep -o '"ratio":[0-9.]*' | \ - grep -o '[0-9.]*') - - [[ -z "$HASH" || -z "$ADDED" ]] && continue - - # Age check — integer only, no bc needed - AGE_DAYS=$(( (NOW - ADDED) / 86400 )) - [[ "$AGE_DAYS" -lt "$QBIT_FAILSAFE_MIN_DAYS" ]] && continue - - # Ratio check — integer comparison only, strip decimal - if [[ "$QBIT_FAILSAFE_MIN_RATIO" != "0" ]]; then - RATIO_INT=${RATIO%.*} - MIN_RATIO_INT=${QBIT_FAILSAFE_MIN_RATIO%.*} - [[ "$RATIO_INT" -lt "$MIN_RATIO_INT" ]] && continue - fi - - if $DRY_RUN; then - echo "🧪 Would delete: $NAME (${AGE_DAYS} days old, ratio: $RATIO)" - else - curl -s -X POST \ - "$QBIT_URL/api/v2/torrents/delete" \ - -H "Cookie: $QBIT_COOKIE" \ - --data "hashes=$HASH&deleteFiles=false" - echo " 🗑️ Deleted: $NAME (${AGE_DAYS} days old, ratio: $RATIO)" - fi - done - - echo "✅ qBittorrent: complete" -fi - -echo "" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo "✅ Downloaders reset complete" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/unRAID_Essentials/inotify_tuning.sh b/unRAID_Essentials/inotify_tuning.sh new file mode 100644 index 0000000..5d24b74 --- /dev/null +++ b/unRAID_Essentials/inotify_tuning.sh @@ -0,0 +1,126 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- inotify Tuning -------------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Increases Linux inotify limits to prevent "too many open files" and inotify exhaustion. +# Run once at array start via ARRAY_START_SCRIPTS in Master.conf. +# +# Why this matters: +# Each Docker container that watches files (Sonarr, Radarr, Lidarr, NextCloud etc.) +# consumes inotify instances and watches. unRAID defaults are very low — with many +# containers running you can exhaust the limit silently, causing containers to miss +# file events (new downloads not detected, library not updated etc.) +# +# max_user_instances = max number of inotify instances per user (default: 128) +# max_user_watches = max number of files/dirs watched per instance (default: 8192) +# max_queued_events = max events queued before dropping (default: 16384) +# +# These settings are lost on reboot — this script reapplies them at every array start. +# All values configurable in Master.conf under unRAID Essentials. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR inotify Tuning ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +# ----------------------------------------------------------------------------------------------- +# ━━━ Current Values ━━━ +# ----------------------------------------------------------------------------------------------- +CURRENT_INSTANCES=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo "?") +CURRENT_WATCHES=$(sysctl -n fs.inotify.max_user_watches 2>/dev/null || echo "?") +CURRENT_EVENTS=$(sysctl -n fs.inotify.max_queued_events 2>/dev/null || echo "?") + +info "Current: instances=$CURRENT_INSTANCES watches=$CURRENT_WATCHES queued=$CURRENT_EVENTS" +info "Target: instances=${INOTIFY_MAX_INSTANCES} watches=${INOTIFY_MAX_WATCHES} queued=${INOTIFY_MAX_QUEUED_EVENTS}" + +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY INOTIFY STATUS ━━━━━" + echo " max_user_instances: $CURRENT_INSTANCES (target: ${INOTIFY_MAX_INSTANCES})" + echo " max_user_watches: $CURRENT_WATCHES (target: ${INOTIFY_MAX_WATCHES})" + echo " max_queued_events: $CURRENT_EVENTS (target: ${INOTIFY_MAX_QUEUED_EVENTS})" + echo "" + echo " Active inotify instances in use:" + find /proc/*/fd -lname "anon_inode:inotify" 2>/dev/null | \ + awk -F/ '{print $3}' | sort -u | while read -r pid; do + cmd=$(cat /proc/$pid/comm 2>/dev/null || echo "?") + echo " PID $pid ($cmd)" + done | head -20 + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ Apply Settings ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +CHANGED=0 +FAILED=0 + +apply_sysctl() { + local key="$1" + local value="$2" + local current + current=$(sysctl -n "$key" 2>/dev/null || echo 0) + + if [[ "$current" -eq "$value" ]]; then + success "$key = $value (already set)" + return + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would set $key = $value (currently $current)" + return + fi + + if sysctl -w "${key}=${value}" >/dev/null 2>&1; then + success "$key = $value (was $current)" + ((CHANGED++)) + else + error "Failed to set $key = $value" + ((FAILED++)) + fi +} + +apply_sysctl "fs.inotify.max_user_instances" "$INOTIFY_MAX_INSTANCES" +apply_sysctl "fs.inotify.max_user_watches" "$INOTIFY_MAX_WATCHES" +apply_sysctl "fs.inotify.max_queued_events" "$INOTIFY_MAX_QUEUED_EVENTS" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━━━ $ICON_SUMMARY INOTIFY TUNING SUMMARY ━━━━━" +echo " max_user_instances: $(sysctl -n fs.inotify.max_user_instances 2>/dev/null)" +echo " max_user_watches: $(sysctl -n fs.inotify.max_user_watches 2>/dev/null)" +echo " max_queued_events: $(sysctl -n fs.inotify.max_queued_events 2>/dev/null)" +echo "" + +if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Status: DRY RUN — no changes made" +elif [[ "$FAILED" -gt 0 ]]; then + echo "$ICON_ERROR Status: $FAILED setting(s) failed" + notify "inotify tuning failed on $(hostname) — $FAILED setting(s) could not be applied" "inotify Tuning" "warning" + exit 1 +elif [[ "$CHANGED" -gt 0 ]]; then + echo "$ICON_DONE Status: $ICON_SUCCESS $CHANGED setting(s) applied" +else + echo "$ICON_DONE Status: $ICON_SUCCESS All settings already correct" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/user_script_plug-in.sh b/user_script_plug-in.sh index 1797709..437480b 100644 --- a/user_script_plug-in.sh +++ b/user_script_plug-in.sh @@ -28,6 +28,10 @@ # ZFS memory snapshot moved to Monitors/ — informational only # Directory tree updated to reflect full ecosystem # v1.9 — transcode_management.sh added to Orchestrators/ +# v2.0 — Major restructure: array_start.sh single entry point, daily/weekly orchestrators, +# sunday_morning_coffee_report.sh, continuous_scripts_status.sh, +# downloaders_reset.sh, docker_network_connect ensure+connect combined, +# inotify_tuning.sh, arr path translation, version checking, nuclear mode flags # replaces separate transcode_manager and transcode_cleanup cron entries # Tools/ folder expanded with all utility scripts # Docker watchdog — two-tier monitoring, startup grace, dependency ordering @@ -83,7 +87,8 @@ # │ ├── docker_watchdog.sh # Two-tier container health monitor — self-healing # │ ├── docker_daily_restart.sh # Restarts configured containers daily # │ ├── docker_weekly_restart.sh # Restarts configured containers weekly -# │ ├── docker_network_connect.sh # Connects containers to extra networks on boot +# │ ├── docker_network_connect.sh # Ensures custom networks exist + connects containers +# │ ├── downloaders_reset.sh # Daily reset for slskd, SABnzbd, qBittorrent # │ └── README-Docker_Essentials.md # │ # ├── Media/ @@ -113,6 +118,7 @@ # └── unRAID_Essentials/ # ├── clear_logs.sh # Clears unRAID system and Docker log files # ├── docker_syslog_filter.sh # Filters Docker veth noise from syslog +# ├── inotify_tuning.sh # Bumps inotify limits — prevents container file event exhaustion # ├── mover_stop.sh # Safely stops the unRAID mover # ├── php_fpm_max_children.sh # Sets PHP-FPM max children value # ├── rsync_stop.sh # Stops all rsync processes on both servers @@ -154,6 +160,17 @@ #/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh --dry-run #/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh # +# ━━━ Monitors ━━━ +#/mnt/user/appdata/unraid_scripts/Monitors/continuous_scripts_status.sh +# ^^ run manually anytime — live status of system_watchdog, docker_watchdog, failover +#/mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh +#/mnt/user/appdata/unraid_scripts/Monitors/bandwidth_monitor.sh --report +#/mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh +#/mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh +#/mnt/user/appdata/unraid_scripts/Monitors/smart_health.sh +#/mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh +#/mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh +# # ━━━ Orchestrators ━━━ # Orchestrators are the main scripts — they handle most if not all needed scripts # in a specific order of operations. Each orchestrator owns a domain and runs @@ -223,6 +240,8 @@ #/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_daily_restart.sh #/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh #/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh +#/mnt/user/appdata/unraid_scripts/Docker_Essentials/downloaders_reset.sh --dry-run --log +#/mnt/user/appdata/unraid_scripts/Docker_Essentials/downloaders_reset.sh # # ━━━ Media ━━━ # media_management.sh is absorbed into daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS. @@ -243,17 +262,6 @@ #/mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh # ^^ schedule: 0 5 * * * (5am daily) — always dry-run first # -# ━━━ Monitors ━━━ -#/mnt/user/appdata/unraid_scripts/Monitors/continuous_scripts_status.sh -# ^^ run manually anytime — live status of system_watchdog, docker_watchdog, failover -#/mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh -#/mnt/user/appdata/unraid_scripts/Monitors/bandwidth_monitor.sh --report -#/mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh -#/mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh -#/mnt/user/appdata/unraid_scripts/Monitors/smart_health.sh -#/mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh -#/mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh -# # ━━━ Transcodes ━━━ # transcode_management.sh runs cleanup then manager — schedule that, not the individuals. # ramdisk_setup.sh runs once at array start. @@ -327,7 +335,7 @@ # ramdisk_setup.sh — creates ramdisk before Emby starts (one-shot) # docker_syslog_filter.sh — suppress veth noise before logs fill (one-shot) # php_fpm_max_children.sh — WebGUI performance tuning (one-shot) -# docker_network_connect.sh — connect containers to extra networks (one-shot) +# docker_network_connect.sh — ensure custom networks exist + connect containers (one-shot) # system_watchdog.sh — system health monitor (continuous) # docker_watchdog.sh — container health monitor (continuous) # failover.sh — mutual failover (continuous)