From 69189bbf187519aadd428343351d3b23f8a4cbdf Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Fri, 3 Jul 2026 17:35:30 -0400 Subject: [PATCH] Fix dead-variable and exit-code bugs found in codebase-wide audit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same audit as the orchestrator standardization pass (2a062e5), extended to every remaining script. Found the same class of bug independently recurring: ramdisk_stop.sh checked $LOG (nothing assigns it, should be $ENABLE_LOGGING), partnership_onboard.sh checked $LOG_MODE (same issue), emby_session_report.sh checked $TRANSCODE_PCT which was never computed so the high-transcode alert could never fire, and storage_migrate.sh never called detect_hosts() so $MY_ID was empty, silently breaking the post-migration host*.conf update. partnership_manager.sh used `local` at top-level script scope (invalid outside a function) and had two master.conf path references missing "Configurations/". Along the way: several scripts (share_setup.sh, conf_sync.sh, downloaders_reset.sh, transcode_cleanup.sh, transcode_manager.sh, remote_arr_cache_writer.sh, upgrade_webhook_handler.sh) had no explicit trailing exit code, so they always reported success regardless of real failures. play_state_sync.sh was missing the partnership gate its own header documented, so remote play-state sync ran even with PARTNERSHIP_ENABLED=false; it also always exited 0 on sync errors. arr_profile_enforcer.sh and webhook_setup.sh hand-rolled their own flag parsing instead of common.sh's parse_args, so --log silently did nothing on either. system_watchdog.sh was itself an un-standardized mini-orchestrator — converted to the shared run_orch_child()/JOB_PASS/JOB_FAIL pattern, added the missing failure notification, and fixed dry-run to pass --dry-run down to children instead of skipping them outright. Also fixed a stale webgui_watchdog.sh path in master.conf.template that would break system_watchdog.sh on any fresh install. Closed a sibling-drift gap: radarr_cleanup.sh and sonarr_cleanup.sh were missing lidarr_cleanup.sh's tracked-count percentage-drop safety gate and its "not configured on this host, skip cleanly" guard — both now match Lidarr's 7-gate model. --- Arrs_Stack/radarr_cleanup.sh | 45 ++++++++++++++- Arrs_Stack/sonarr_cleanup.sh | 45 ++++++++++++++- Arrs_Stack/upgrade_webhook_handler.sh | 8 +++ Configurations/master.conf | 6 ++ Deployment/master.conf.template | 8 ++- Docker_Essentials/downloaders_reset.sh | 4 +- Media/play_state_sync.sh | 15 ++++- Monitors/emby_session_report.sh | 7 ++- Partnership/partnership_manager.sh | 6 +- Partnership/partnership_onboard.sh | 4 +- Partnership/share_setup.sh | 2 + .../unraid/Tools/remote_arr_cache_writer.sh | 3 + Plugin/unraid/Tools/storage_migrate.sh | 2 + System_Essentials/conf_sync.sh | 2 + Tools/arr_profile_enforcer.sh | 6 +- Tools/ramdisk_stop.sh | 2 +- Tools/webhook_setup.sh | 8 ++- Transcodes/transcode_cleanup.sh | 27 ++++++--- Transcodes/transcode_manager.sh | 11 +++- Watchdogs/resource_watchdog.sh | 7 ++- Watchdogs/system_watchdog.sh | 55 +++++-------------- 21 files changed, 197 insertions(+), 76 deletions(-) diff --git a/Arrs_Stack/radarr_cleanup.sh b/Arrs_Stack/radarr_cleanup.sh index d79f7ff..c5a3672 100755 --- a/Arrs_Stack/radarr_cleanup.sh +++ b/Arrs_Stack/radarr_cleanup.sh @@ -53,13 +53,14 @@ # OPERATIONAL SAFEGUARDS # ============================================================================================== # -# Six gates — ALL must pass before any file is touched: +# Seven gates — ALL must pass before any file is touched: # 1. Container running and not starting/unhealthy # 2. API reachable # 3. API version matches RADARR_VERSION_MAJOR in master.conf # 4. Movie count > 0 # 5. Tracked file count > 0 -# 6. Deletion size < RADARR_MAX_DELETE_GB — or --i-know-what-im-doing required +# 6. Tracked count >= RADARR_MIN_TRACKED_PCT % of last known count +# 7. Deletion size < RADARR_MAX_DELETE_GB — or --i-know-what-im-doing required # # acquire_lock "wait" — large scans take time, wait for previous run to finish # jq + curl validation — exits if either tool missing @@ -69,6 +70,14 @@ # Silent by default — orphans/junk warn(), clean library logs silently # # ============================================================================================== +# STATE FILES +# ============================================================================================== +# +# RADARR_TRACKED_COUNT_FILE — persistent baseline for the tracked % safety check (gate 6) +# Updated after each successful run. Protects against misconfigured root path +# returning an empty API response and deleting the entire library. +# +# ============================================================================================== # CONFIGURATION # ============================================================================================== # @@ -82,6 +91,8 @@ # # RADARR_ORPHAN_AGE — days before untracked file eligible for deletion # RADARR_MAX_DELETE_GB — require --i-know-what-im-doing above this +# RADARR_MIN_TRACKED_PCT — abort if tracked count drops below this % of last run +# RADARR_TRACKED_COUNT_FILE — persistent baseline file path # RADARR_EXTENSIONS — video file extensions for orphan classification # RADARR_PROTECTED_PATTERNS — file patterns never deleted # RADARR_VERSION_MAJOR — expected Radarr major version for API safety check @@ -174,6 +185,12 @@ fi # detect_hosts() sets MY_ID and aliases RADARR_URL, RADARR_API_KEY, RADARR_MOVIES_ROOT detect_hosts +# Skip if Radarr is not configured on this host +if [[ -z "${RADARR_URL:-}" ]] || [[ -z "${RADARR_API_KEY:-}" ]]; then + info "Radarr not configured on $MY_ID ($LOCAL_SERVER_NAME) — skipping" + exit 0 +fi + DOCKER_TIMEOUT=15 RADARR_CONTAINER="Radarr" @@ -213,6 +230,7 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "$ICON_GEAR Movies root: $RADARR_MOVIES_ROOT" echo "$ICON_TIME Orphan age: ${RADARR_ORPHAN_AGE} days" echo "$ICON_GEAR Max delete: ${RADARR_MAX_DELETE_GB}GB (requires --i-know-what-im-doing)" + echo "$ICON_GEAR Min tracked %: ${RADARR_MIN_TRACKED_PCT}%" echo "$ICON_GEAR Radarr ver: v${RADARR_VERSION_MAJOR} expected" echo "$ICON_GEAR Extensions: ${RADARR_EXTENSIONS[*]}" echo "$ICON_GEAR Protected patterns: ${RADARR_PROTECTED_PATTERNS[*]}" @@ -448,6 +466,27 @@ fi info "$MOVIE_COUNT movies | $TRACKED_COUNT tracked movie files" +# Safety Layer 6 — percentage drop vs last known count +if [[ -f "$RADARR_TRACKED_COUNT_FILE" ]]; then + LAST_COUNT=$(cat "$RADARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0) + if [[ "$LAST_COUNT" -gt 0 ]]; then + PCT=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $LAST_COUNT) * 100}") + if [[ "$PCT" -lt "$RADARR_MIN_TRACKED_PCT" ]]; then + error "Tracked count dropped to ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT)" + error "Suggests API issue — aborting to prevent mass deletion" + error "If expected (large library removal) delete: $RADARR_TRACKED_COUNT_FILE" + notify "Radarr cleanup aborted on $(hostname) — tracked count dropped to ${PCT}%" \ + "Radarr Cleanup" "warning" + exit 1 + fi + info "Tracked count: ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT) ✅" + fi +else + info "No previous count on record — first run, saving baseline" +fi + +echo "$TRACKED_COUNT" > "$RADARR_TRACKED_COUNT_FILE" + # ============================================================================================== # ━━━ Scan Movies Root ━━━ # ============================================================================================== @@ -512,7 +551,7 @@ TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES )) TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) # ============================================================================================== -# ━━━ Safety Layer 6 — Deletion Size Threshold ━━━ +# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━ # ============================================================================================== if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}") diff --git a/Arrs_Stack/sonarr_cleanup.sh b/Arrs_Stack/sonarr_cleanup.sh index f835937..da20ec3 100755 --- a/Arrs_Stack/sonarr_cleanup.sh +++ b/Arrs_Stack/sonarr_cleanup.sh @@ -53,13 +53,14 @@ # OPERATIONAL SAFEGUARDS # ============================================================================================== # -# Six gates — ALL must pass before any file is touched: +# Seven gates — ALL must pass before any file is touched: # 1. Container running and not starting/unhealthy # 2. API reachable # 3. API version matches SONARR_VERSION_MAJOR in master.conf # 4. Series count > 0 # 5. Tracked file count > 0 -# 6. Deletion size < SONARR_MAX_DELETE_GB — or --i-know-what-im-doing required +# 6. Tracked count >= SONARR_MIN_TRACKED_PCT % of last known count +# 7. Deletion size < SONARR_MAX_DELETE_GB — or --i-know-what-im-doing required # # acquire_lock "wait" — large scans take time, wait for previous run to finish # jq + curl validation — exits if either tool missing @@ -69,6 +70,14 @@ # Silent by default — orphans/junk warn(), clean library logs silently # # ============================================================================================== +# STATE FILES +# ============================================================================================== +# +# SONARR_TRACKED_COUNT_FILE — persistent baseline for the tracked % safety check (gate 6) +# Updated after each successful run. Protects against misconfigured root path +# returning an empty API response and deleting the entire library. +# +# ============================================================================================== # CONFIGURATION # ============================================================================================== # @@ -82,6 +91,8 @@ # # SONARR_ORPHAN_AGE — days before untracked file eligible for deletion # SONARR_MAX_DELETE_GB — require --i-know-what-im-doing above this +# SONARR_MIN_TRACKED_PCT — abort if tracked count drops below this % of last run +# SONARR_TRACKED_COUNT_FILE — persistent baseline file path # SONARR_EXTENSIONS — video file extensions for orphan classification # SONARR_PROTECTED_PATTERNS — file patterns never deleted # SONARR_VERSION_MAJOR — expected Sonarr major version for API safety check @@ -174,6 +185,12 @@ fi # detect_hosts() sets MY_ID and aliases SONARR_URL, SONARR_API_KEY, SONARR_TV_ROOT detect_hosts +# Skip if Sonarr is not configured on this host +if [[ -z "${SONARR_URL:-}" ]] || [[ -z "${SONARR_API_KEY:-}" ]]; then + info "Sonarr not configured on $MY_ID ($LOCAL_SERVER_NAME) — skipping" + exit 0 +fi + DOCKER_TIMEOUT=15 SONARR_CONTAINER="Sonarr" @@ -213,6 +230,7 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "$ICON_GEAR TV root: $SONARR_TV_ROOT" echo "$ICON_TIME Orphan age: ${SONARR_ORPHAN_AGE} days" echo "$ICON_GEAR Max delete: ${SONARR_MAX_DELETE_GB}GB (requires --i-know-what-im-doing)" + echo "$ICON_GEAR Min tracked %: ${SONARR_MIN_TRACKED_PCT}%" echo "$ICON_GEAR Sonarr ver: v${SONARR_VERSION_MAJOR} expected" echo "$ICON_GEAR Extensions: ${SONARR_EXTENSIONS[*]}" echo "$ICON_GEAR Protected patterns: ${SONARR_PROTECTED_PATTERNS[*]}" @@ -447,6 +465,27 @@ fi info "$SERIES_COUNT series | $TRACKED_COUNT tracked episode files" +# Safety Layer 6 — percentage drop vs last known count +if [[ -f "$SONARR_TRACKED_COUNT_FILE" ]]; then + LAST_COUNT=$(cat "$SONARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0) + if [[ "$LAST_COUNT" -gt 0 ]]; then + PCT=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $LAST_COUNT) * 100}") + if [[ "$PCT" -lt "$SONARR_MIN_TRACKED_PCT" ]]; then + error "Tracked count dropped to ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT)" + error "Suggests API issue — aborting to prevent mass deletion" + error "If expected (large library removal) delete: $SONARR_TRACKED_COUNT_FILE" + notify "Sonarr cleanup aborted on $(hostname) — tracked count dropped to ${PCT}%" \ + "Sonarr Cleanup" "warning" + exit 1 + fi + info "Tracked count: ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT) ✅" + fi +else + info "No previous count on record — first run, saving baseline" +fi + +echo "$TRACKED_COUNT" > "$SONARR_TRACKED_COUNT_FILE" + # ============================================================================================== # ━━━ Scan TV Root ━━━ # ============================================================================================== @@ -511,7 +550,7 @@ TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES )) TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) # ============================================================================================== -# ━━━ Safety Layer 6 — Deletion Size Threshold ━━━ +# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━ # ============================================================================================== if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}") diff --git a/Arrs_Stack/upgrade_webhook_handler.sh b/Arrs_Stack/upgrade_webhook_handler.sh index 72c935e..ad042a8 100755 --- a/Arrs_Stack/upgrade_webhook_handler.sh +++ b/Arrs_Stack/upgrade_webhook_handler.sh @@ -123,11 +123,14 @@ echo "[$(date '+%H:%M:%S')] Upgrade push: ${ARR_TYPE} — ${ITEM_NAME}" echo " Path: $ITEM_PATH" echo " Targets: ${REMOTE_NODES[*]}" +NODE_FAIL=0 + # ── Push and rescan each remote ─────────────────────────────────────────────────────────────── for node_id in "${REMOTE_NODES[@]}"; do node_name="${!node_id}" node_ip=$(resolve_tailscale_ip "$node_name") || { echo " [${node_name}] Cannot resolve Tailscale IP — skipping" + (( NODE_FAIL++ )) continue } @@ -142,6 +145,7 @@ for node_id in "${REMOTE_NODES[@]}"; do if [[ "$rsync_exit" -ne 0 ]]; then echo " [${node_name}] rsync failed (exit ${rsync_exit}) — skipping rescan" + (( NODE_FAIL++ )) continue fi echo " [${node_name}] rsync done (${transferred:-0} bytes)" @@ -177,7 +181,11 @@ REMOTE echo " [${node_name}] ${RESCAN_CMD} triggered ✅" else echo " [${node_name}] ${RESCAN_CMD} failed (HTTP ${http_code:-timeout})" + (( NODE_FAIL++ )) fi done echo "[$(date '+%H:%M:%S')] Done — ${ITEM_NAME}" + +[[ "$NODE_FAIL" -gt 0 ]] && exit 1 +exit 0 diff --git a/Configurations/master.conf b/Configurations/master.conf index 2bbc8ce..52ee483 100644 --- a/Configurations/master.conf +++ b/Configurations/master.conf @@ -1145,6 +1145,9 @@ # Sonarr shared settings SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion SONARR_MAX_DELETE_GB=10 # require --i-know-what-im-doing if deletion exceeds this + SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run + # protects against API returning partial data on a bad day + SONARR_TRACKED_COUNT_FILE="$DATA_DIR/sonarr_tracked.count" SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") SONARR_PROTECTED_PATTERNS=( # Subtitles @@ -1168,6 +1171,9 @@ # Radarr shared settings RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion RADARR_MAX_DELETE_GB=15 # require --i-know-what-im-doing if deletion exceeds this + RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run + # protects against API returning partial data on a bad day + RADARR_TRACKED_COUNT_FILE="$DATA_DIR/radarr_tracked.count" RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") RADARR_PROTECTED_PATTERNS=( # Subtitles diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index a365a2a..17a6822 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -345,7 +345,7 @@ # Called by watchdog_orchestrator.sh — not scheduled directly. SYSTEM_WATCHDOG_SCRIPTS=( "Watchdogs/System/storage_watchdog.sh" # pool growth + runaway log detection - "Watchdogs/System/webgui_watchdog.sh" # WebGUI availability — nginx → php-fpm → emhttp + "Plugin/unraid/Watchdogs/System/webgui_watchdog.sh" # WebGUI availability — nginx → php-fpm → emhttp "Watchdogs/System/network_watchdog.sh" # internet, DDNS, Tailscale, NPM proxy "Watchdogs/System/conf_cache_watchdog.sh" # maintain persistent conf backup while partner is offline ) @@ -1136,6 +1136,9 @@ # Sonarr shared settings SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion SONARR_MAX_DELETE_GB=10 # require --i-know-what-im-doing if deletion exceeds this + SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run + # protects against API returning partial data on a bad day + SONARR_TRACKED_COUNT_FILE="$DATA_DIR/sonarr_tracked.count" SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") SONARR_PROTECTED_PATTERNS=( # Subtitles @@ -1157,6 +1160,9 @@ # Radarr shared settings RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion RADARR_MAX_DELETE_GB=15 # require --i-know-what-im-doing if deletion exceeds this + RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run + # protects against API returning partial data on a bad day + RADARR_TRACKED_COUNT_FILE="$DATA_DIR/radarr_tracked.count" RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") RADARR_PROTECTED_PATTERNS=( # Subtitles diff --git a/Docker_Essentials/downloaders_reset.sh b/Docker_Essentials/downloaders_reset.sh index 2c1c168..b3af203 100755 --- a/Docker_Essentials/downloaders_reset.sh +++ b/Docker_Essentials/downloaders_reset.sh @@ -643,8 +643,10 @@ if [[ "$DRY_RUN" == true ]]; then elif [[ "$TOTAL_FAIL" -gt 0 ]]; then echo "$ICON_ERROR Status: $TOTAL_FAIL failure(s) — check logs" notify "Downloaders reset completed with failures on $(hostname)" "Downloaders Reset" "warning" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" exit 1 else echo "$ICON_DONE Status: $ICON_SUCCESS DONE" fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +exit 0 \ No newline at end of file diff --git a/Media/play_state_sync.sh b/Media/play_state_sync.sh index 073b852..f91f4e0 100755 --- a/Media/play_state_sync.sh +++ b/Media/play_state_sync.sh @@ -167,6 +167,12 @@ for _varname in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do continue fi + # Skip remote hosts when partnership is inactive + if [[ "$_is_me" == false && "${PARTNERSHIP_ENABLED:-false}" != "true" ]]; then + log "$_host_hostname — partnership inactive (PARTNERSHIP_ENABLED=false), skipping" + continue + fi + # Resolve Tailscale IP for remote hosts _ts_ip="" if [[ "$_is_me" == false ]]; then @@ -780,10 +786,15 @@ echo "$ICON_DONE Synced: $TOTAL_SYNCED" echo "" echo " Favorites — synced: $FAV_TOTAL_SYNCED skipped: $FAV_TOTAL_SKIPPED errors: $FAV_TOTAL_ERRORS" +TOTAL_ALL_ERRORS=$(( TOTAL_ERRORS + FAV_TOTAL_ERRORS )) + if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes written" -elif [[ "$TOTAL_ERRORS" -eq 0 ]]; then + exit 0 +elif [[ "$TOTAL_ALL_ERRORS" -eq 0 ]]; then success "Done ✅" + exit 0 else - warn "Done with $TOTAL_ERRORS error(s)" + warn "Done with $TOTAL_ALL_ERRORS error(s)" + exit 1 fi diff --git a/Monitors/emby_session_report.sh b/Monitors/emby_session_report.sh index b921448..9f5f97f 100755 --- a/Monitors/emby_session_report.sh +++ b/Monitors/emby_session_report.sh @@ -218,6 +218,9 @@ TRANSCODE_NOW=$(echo "$SESSIONS" | \ 2>/dev/null || echo 0) DIRECT_NOW=$(( ACTIVE_COUNT - TRANSCODE_NOW )) +TRANSCODE_PCT=0 +[[ "$ACTIVE_COUNT" -gt 0 ]] && TRANSCODE_PCT=$(( TRANSCODE_NOW * 100 / ACTIVE_COUNT )) + echo " $ICON_EMBY Active streams: $ACTIVE_COUNT" echo " $ICON_EMBY Direct play: $DIRECT_NOW" echo " $ICON_EMBY Transcoding: $TRANSCODE_NOW" @@ -333,7 +336,7 @@ END=$(date +%s) echo "━━━━━ $ICON_SUMMARY EMBY REPORT SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_EMBY Server: $SERVER_NAME (v$SERVER_VERSION)" -echo "$ICON_EMBY Active: $ACTIVE_COUNT streams ($DIRECT_NOW direct / $TRANSCODE_NOW transcode)" +echo "$ICON_EMBY Active: $ACTIVE_COUNT streams ($DIRECT_NOW direct / $TRANSCODE_NOW transcode, ${TRANSCODE_PCT}%)" echo "$ICON_EMBY Library: $MOVIE_COUNT movies $EPISODE_COUNT episodes $SONG_COUNT songs" echo "$ICON_EMBY Period: $TOTAL_PLAYS play events in last ${EMBY_REPORT_DAYS} days" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" @@ -341,7 +344,7 @@ echo "━━━━━━━━━━━━━━━━━━━━━━━━ # Only notify on issues — high transcode rate may indicate config problem if [[ "$DRY_RUN" == false ]]; then - if [[ "$TOTAL_PLAYS" -gt 10 && "${TRANSCODE_PCT:-0}" -gt 80 ]]; then + if [[ "$TOTAL_PLAYS" -gt 10 && "$TRANSCODE_PCT" -gt 80 ]]; then notify "Emby report on $(hostname) — high transcode rate: ${TRANSCODE_PCT}% of $TOTAL_PLAYS plays — check direct play config" \ "Emby Report" "warning" fi diff --git a/Partnership/partnership_manager.sh b/Partnership/partnership_manager.sh index a944570..29113db 100755 --- a/Partnership/partnership_manager.sh +++ b/Partnership/partnership_manager.sh @@ -764,7 +764,7 @@ provision_emby_admin() { error "Emby username '${username}' is already taken on the shared instance" error "Options:" error " 1. Sign in with that account — it may already be yours" - error " 2. Set a different name in ${user_var} and re-run --onboard" + error " 2. Set a different name in HOST${MIRROR_ID: -1}_PARTNERSHIP_EMBY_ADMIN_USER and re-run --onboard" return 1 fi @@ -916,7 +916,7 @@ do_final_sync() { # Safe master.conf modification with error handling update_master_conf() { local key="$1" value="$2" - local conf="$SCRIPT_DIR/../master.conf" + local conf="$CONF_DIR/master.conf" if [[ ! -f "$conf" ]]; then error "master.conf not found at $conf" return 1 @@ -1162,7 +1162,7 @@ if [[ "$MODE" == "onboard" ]]; then echo "" echo "Enabling partnership in master.conf..." if [[ "$DRY_RUN" == false ]]; then - local _master_conf="${SCRIPTS_ROOT:-$(dirname "$SCRIPT_DIR")}/master.conf" + _master_conf="$CONF_DIR/master.conf" if grep -q "^[[:space:]]*PARTNERSHIP_ENABLED=" "$_master_conf" 2>/dev/null; then sed -i "s|^[[:space:]]*PARTNERSHIP_ENABLED=.*|PARTNERSHIP_ENABLED=true|" "$_master_conf" else diff --git a/Partnership/partnership_onboard.sh b/Partnership/partnership_onboard.sh index 1cf8955..1b421bb 100755 --- a/Partnership/partnership_onboard.sh +++ b/Partnership/partnership_onboard.sh @@ -240,7 +240,7 @@ AM_MIRROR=false EXTRA_FLAGS=() [[ "$DRY_RUN" == true ]] && EXTRA_FLAGS+=("--dry-run") -[[ "$LOG_MODE" == true ]] && EXTRA_FLAGS+=("--log") +[[ "$ENABLE_LOGGING" == true ]] && EXTRA_FLAGS+=("--log") START=$(date +%s) @@ -718,7 +718,7 @@ else _rsync_script="$SCRIPTS_ROOT/Rsync/rsync.sh" _seed_flags=(--seed) [[ "$DRY_RUN" == true ]] && _seed_flags+=(--dry-run) - [[ "$LOG_MODE" == true ]] && _seed_flags+=(--log) + [[ "$ENABLE_LOGGING" == true ]] && _seed_flags+=(--log) for _share in "${DAILY_SYNC_SHARES[@]}"; do echo " Seeding: $_share" if bash "$_rsync_script" "$_share" "${_seed_flags[@]}"; then diff --git a/Partnership/share_setup.sh b/Partnership/share_setup.sh index 9b67642..99f7f22 100755 --- a/Partnership/share_setup.sh +++ b/Partnership/share_setup.sh @@ -175,3 +175,5 @@ done echo "" echo " Shares: $CREATED created, $SKIPPED already existed on $MIRROR" [[ "$DRY_RUN" == true ]] && echo " (dry-run — nothing written)" + +exit 0 diff --git a/Plugin/unraid/Tools/remote_arr_cache_writer.sh b/Plugin/unraid/Tools/remote_arr_cache_writer.sh index 06a57d7..892d290 100755 --- a/Plugin/unraid/Tools/remote_arr_cache_writer.sh +++ b/Plugin/unraid/Tools/remote_arr_cache_writer.sh @@ -162,3 +162,6 @@ echo "" echo "━━━━━ $ICON_SUMMARY Cache Writer Summary ━━━━━" echo " $FETCH_OK updated · $FETCH_FAIL failed · $FETCH_SKIP skipped" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ "$FETCH_FAIL" -gt 0 ]] && exit 1 +exit 0 diff --git a/Plugin/unraid/Tools/storage_migrate.sh b/Plugin/unraid/Tools/storage_migrate.sh index ef53be2..5521d55 100755 --- a/Plugin/unraid/Tools/storage_migrate.sh +++ b/Plugin/unraid/Tools/storage_migrate.sh @@ -69,10 +69,12 @@ source "$SCRIPT_DIR/../../../load_config.sh" parse_args "$@" acquire_lock +detect_hosts VV_CFG="/boot/config/plugins/varaverk/varaverk.cfg" INTERNAL_DIR="/boot/config/plugins/varaverk" FLASH_DIR="/mnt/user/appdata/Varaverk" +CONF_FILE="$CONF_DIR/${MY_ID,,}.conf" # ────────────────────────────────────────────────────────────────────────────── # Parse --to= from raw args (parse_args doesn't handle this flag) diff --git a/System_Essentials/conf_sync.sh b/System_Essentials/conf_sync.sh index d94ebc5..c192360 100755 --- a/System_Essentials/conf_sync.sh +++ b/System_Essentials/conf_sync.sh @@ -200,3 +200,5 @@ if [[ "$FAILED" -gt 0 ]]; then "Conf Sync" "warning" exit 1 fi + +exit 0 diff --git a/Tools/arr_profile_enforcer.sh b/Tools/arr_profile_enforcer.sh index cf4f230..4abdbef 100755 --- a/Tools/arr_profile_enforcer.sh +++ b/Tools/arr_profile_enforcer.sh @@ -51,15 +51,16 @@ # arr_profile_enforcer.sh --radarr-only # Run only Radarr enforcement. # +# arr_profile_enforcer.sh --log +# Verbose output. +# # ============================================================================================== -DRY_RUN=false RUN_SONARR=true RUN_RADARR=true for arg in "$@"; do case "$arg" in - --dry-run) DRY_RUN=true ;; --sonarr-only) RUN_RADARR=false ;; --radarr-only) RUN_SONARR=false ;; esac @@ -67,6 +68,7 @@ done SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" +parse_args "$@" detect_hosts KIDS_PROFILE_NAME="${ARR_KIDS_PROFILE_NAME:-Kids shows}" diff --git a/Tools/ramdisk_stop.sh b/Tools/ramdisk_stop.sh index c8d5a4d..96cc8f7 100755 --- a/Tools/ramdisk_stop.sh +++ b/Tools/ramdisk_stop.sh @@ -247,7 +247,7 @@ if [[ "$FILE_COUNT" -gt 0 ]]; then warn "⚠️ $FILE_COUNT file(s) still on ramdisk — will be lost on unmount" warn "Active transcode sessions should be stopped before unmounting" warn "Proceeding regardless (this is expected for maintenance)" - if [[ "$LOG" == true ]]; then + if [[ "$ENABLE_LOGGING" == true ]]; then find "$RAMDISK_PATH" -type f 2>/dev/null | while read -r f; do log " $f" done diff --git a/Tools/webhook_setup.sh b/Tools/webhook_setup.sh index 79543fe..6e9a66c 100755 --- a/Tools/webhook_setup.sh +++ b/Tools/webhook_setup.sh @@ -40,6 +40,7 @@ # Secret auto-gen — WEBHOOK_SECRET generated if empty; never left blank # --local-only — used internally for SSH; prevents infinite recursion # --dry-run mode — shows what would be registered without making API calls +# acquire_lock — prevents concurrent registration runs # # ============================================================================================== # RUNTIME MODES @@ -54,17 +55,18 @@ # webhook_setup.sh --dry-run # Show what would be registered without making any changes. # +# webhook_setup.sh --log +# Verbose output. +# # ============================================================================================== set -uo pipefail LOCAL_ONLY=false -DRY_RUN=false for arg in "$@"; do case "$arg" in --local-only) LOCAL_ONLY=true ;; - --dry-run) DRY_RUN=true ;; esac done @@ -73,6 +75,8 @@ ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" MASTER_CONF="$ECOSYSTEM_ROOT/Configurations/master.conf" source "$ECOSYSTEM_ROOT/load_config.sh" +parse_args "$@" +acquire_lock detect_hosts WEBHOOK_NAME="Varaverk Upgrade" diff --git a/Transcodes/transcode_cleanup.sh b/Transcodes/transcode_cleanup.sh index 4f52dd4..3d10095 100755 --- a/Transcodes/transcode_cleanup.sh +++ b/Transcodes/transcode_cleanup.sh @@ -168,15 +168,15 @@ fi # ============================================================================================== # Scans a location and removes eligible files. # Calls lsof ONCE per location — builds in-memory OPEN_FILES_MAP for O(1) lookup. -# Returns via LOCATION_REMOVED, LOCATION_FREED, LOCATION_SKIPPED, LOCATION_ACTIVE +# Returns via LOCATION_REMOVED, LOCATION_FREED, LOCATION_SKIPPED, LOCATION_ACTIVE, LOCATION_FAILED cleanup_location() { local location="$1" label="$2" max_age="$3" - local files_removed=0 bytes_freed=0 files_skipped=0 files_active=0 files_streaming=0 files_too_young=0 + local files_removed=0 bytes_freed=0 files_skipped=0 files_active=0 files_streaming=0 files_too_young=0 files_failed=0 if [[ ! -d "$location" ]]; then log "$label does not exist — skipping" - LOCATION_REMOVED=0 LOCATION_FREED="0B" LOCATION_SKIPPED=0 LOCATION_ACTIVE=0 LOCATION_STREAMING=0 LOCATION_TOO_YOUNG=0 + LOCATION_REMOVED=0 LOCATION_FREED="0B" LOCATION_SKIPPED=0 LOCATION_ACTIVE=0 LOCATION_STREAMING=0 LOCATION_TOO_YOUNG=0 LOCATION_FAILED=0 return fi @@ -224,7 +224,7 @@ cleanup_location() { log "Deleted: $file" else warn "Could not delete: $file" - (( files_skipped++ )) + (( files_failed++ )) fi fi @@ -249,7 +249,7 @@ cleanup_location() { freed_human="0B" fi - log "$label — removed $files_removed ($freed_human) | active(fresh): $files_too_young | streaming(lsof): $files_streaming | protected(old+open): $files_active | skipped: $files_skipped" + log "$label — removed $files_removed ($freed_human) | active(fresh): $files_too_young | streaming(lsof): $files_streaming | protected(old+open): $files_active | skipped: $files_skipped | failed: $files_failed" LOCATION_REMOVED=$files_removed LOCATION_FREED=$freed_human @@ -257,13 +257,14 @@ cleanup_location() { LOCATION_ACTIVE=$files_active LOCATION_STREAMING=$files_streaming LOCATION_TOO_YOUNG=$files_too_young + LOCATION_FAILED=$files_failed } # ============================================================================================== # ━━━ Transcode Cleanup ━━━ # ============================================================================================== START=$(date +%s) -TOTAL_REMOVED=0 TOTAL_SKIPPED=0 TOTAL_ACTIVE=0 TOTAL_STREAMING=0 TOTAL_TOO_YOUNG=0 +TOTAL_REMOVED=0 TOTAL_SKIPPED=0 TOTAL_ACTIVE=0 TOTAL_STREAMING=0 TOTAL_TOO_YOUNG=0 TOTAL_FAILED=0 RAMDISK_FREED="0B" SSD_FREED="0B" # Cleanup ramdisk @@ -274,6 +275,7 @@ if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE )) TOTAL_STREAMING=$(( TOTAL_STREAMING + LOCATION_STREAMING )) TOTAL_TOO_YOUNG=$(( TOTAL_TOO_YOUNG + LOCATION_TOO_YOUNG )) + TOTAL_FAILED=$(( TOTAL_FAILED + LOCATION_FAILED )) RAMDISK_FREED=$LOCATION_FREED else log "Ramdisk not mounted — skipping ramdisk cleanup" @@ -287,6 +289,7 @@ if [[ -d "$TRANSCODE_SSD" ]]; then TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE )) TOTAL_STREAMING=$(( TOTAL_STREAMING + LOCATION_STREAMING )) TOTAL_TOO_YOUNG=$(( TOTAL_TOO_YOUNG + LOCATION_TOO_YOUNG )) + TOTAL_FAILED=$(( TOTAL_FAILED + LOCATION_FAILED )) SSD_FREED=$LOCATION_FREED else log "SSD fallback not found — skipping SSD cleanup" @@ -319,11 +322,19 @@ echo "$ICON_RUNNING Active: $TOTAL_TOO_YOUNG files (< ${TRANSCODE_MAX_AGE} echo "$ICON_RUNNING Streaming: $TOTAL_STREAMING files (open file handle — long-running transcode)" echo "$ICON_SHIELD Protected: $TOTAL_ACTIVE aged files saved by open-file check" echo "$ICON_TRASH Skipped: $TOTAL_SKIPPED files" +[[ "$TOTAL_FAILED" -gt 0 ]] && echo "$ICON_ERROR Failed: $TOTAL_FAILED files could not be deleted" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no files deleted" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +elif [[ "$TOTAL_FAILED" -gt 0 ]]; then + warn "Status: $TOTAL_FAILED file(s) failed to delete" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + exit 1 else echo "$ICON_DONE Status: done ✅" -fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi \ No newline at end of file diff --git a/Transcodes/transcode_manager.sh b/Transcodes/transcode_manager.sh index c58550e..871414d 100755 --- a/Transcodes/transcode_manager.sh +++ b/Transcodes/transcode_manager.sh @@ -298,6 +298,7 @@ RAMDISK_HEALTHY=true SSD_HEALTHY=true EMBY_RUNNING=true SOMETHING_HAPPENED=false # controls whether summary is printed +ERROR_OCCURRED=false # controls exit code # ── Check 1 — Emby running ─────────────────────────────────────────────────────────────────── if [[ "$TRANSCODE_CHECK_EMBY" == true ]]; then @@ -345,6 +346,7 @@ if ! mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then error "Ramdisk not mounted at $RAMDISK_PATH" RAMDISK_HEALTHY=false SOMETHING_HAPPENED=true + ERROR_OCCURRED=true if [[ "$DRY_RUN" == false ]]; then warn "Flipping symlink to SSD — ramdisk unavailable" flip_symlink "$TRANSCODE_SSD" "ramdisk disappeared" @@ -544,6 +546,7 @@ case "$TRANSCODE_MANAGER_MODE" in notify "Transcode ramdisk mode failed on $(hostname) ($MY_ID) — ramdisk not mounted" \ "Transcode Manager" "warning" SOMETHING_HAPPENED=true + ERROR_OCCURRED=true else if [[ "$CURRENT_TARGET" != "$RAMDISK_PATH" ]]; then flip_symlink "$RAMDISK_PATH" "ramdisk mode" @@ -565,6 +568,7 @@ case "$TRANSCODE_MANAGER_MODE" in if [[ "$SSD_HEALTHY" == false ]]; then error "SSD mode selected but SSD path is not available" SOMETHING_HAPPENED=true + ERROR_OCCURRED=true else if [[ "$CURRENT_TARGET" != "$TRANSCODE_SSD" ]]; then flip_symlink "$TRANSCODE_SSD" "ssd mode" @@ -591,11 +595,13 @@ case "$TRANSCODE_MANAGER_MODE" in notify "Transcode ramdisk full on $(hostname) ($MY_ID) and SSD unavailable" \ "Transcode Manager" "warning" SOMETHING_HAPPENED=true + ERROR_OCCURRED=true else SSD_FREE_GB=$(get_ssd_free_gb) if (( $(awk "BEGIN {print ($SSD_FREE_GB < $RAMDISK_SSD_MIN_GB) ? 1 : 0}") )); then warn "SSD only ${SSD_FREE_GB}GB free — below ${RAMDISK_SSD_MIN_GB}GB minimum, not flipping" SOMETHING_HAPPENED=true + ERROR_OCCURRED=true else warn "Ramdisk ${RAMDISK_USED_GB}GB — above ${RAMDISK_WARN_GB}GB, flipping to SSD" flip_symlink "$TRANSCODE_SSD" "threshold exceeded" @@ -659,4 +665,7 @@ if [[ "$SOMETHING_HAPPENED" == true ]]; then echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" else echo "Transcode manager — all healthy — $MY_ID ($(format_duration $(( END - START ))))" -fi \ No newline at end of file +fi + +[[ "$ERROR_OCCURRED" == true ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/Watchdogs/resource_watchdog.sh b/Watchdogs/resource_watchdog.sh index 40cba09..7846f35 100755 --- a/Watchdogs/resource_watchdog.sh +++ b/Watchdogs/resource_watchdog.sh @@ -133,6 +133,10 @@ detect_hosts DOCKER_TIMEOUT=15 +TOTAL_CORES=$(nproc) +RW_LOAD_SOFT_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RW_LOAD_SOFT_MULTIPLIER:-2.0}}") +RW_LOAD_MEDIUM_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RW_LOAD_MEDIUM_MULTIPLIER:-3.0}}") + log "$ICON_GEAR Config: soft=RAM<${RW_RAM_SOFT_GB}GB/load≥${RW_LOAD_SOFT_THRESH} medium=RAM<${RW_RAM_MEDIUM_GB}GB/load≥${RW_LOAD_MEDIUM_THRESH} hard=RAM<${RW_RAM_HARD_GB}GB recover=RAM≥${RW_RAM_RECOVER_GB}GB cycles=${RW_RECOVER_CYCLES}" log "$ICON_CONTAINERS Pause at medium: ${RW_PAUSE_CONTAINERS[*]:-none} Stop at hard: ${RW_STOP_CONTAINERS[*]:-none}" @@ -195,13 +199,10 @@ STOPPED_LIST=$(rm_state_get "rm_stopped_containers"); STOPPED_LIST=${STOPPED_LIS # ============================================================================================== # ━━━ Pressure Calculation ━━━ # ============================================================================================== -TOTAL_CORES=$(nproc) MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) MEM_GB=$(( MEM_KB / 1024 / 1024 )) LOAD=$(awk '{print $1}' /proc/loadavg) LOAD_INT=$(printf "%.0f" "$LOAD") -RW_LOAD_SOFT_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RW_LOAD_SOFT_MULTIPLIER:-2.0}}") -RW_LOAD_MEDIUM_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RW_LOAD_MEDIUM_MULTIPLIER:-3.0}}") TARGET_LEVEL=0 TARGET_REASON="" diff --git a/Watchdogs/system_watchdog.sh b/Watchdogs/system_watchdog.sh index 9b50025..1addb2c 100755 --- a/Watchdogs/system_watchdog.sh +++ b/Watchdogs/system_watchdog.sh @@ -56,13 +56,13 @@ # Run all system component watchdogs. # # system_watchdog.sh --dry-run -# Preview without running anything. +# Passes --dry-run to each sub-script — no changes made. # # system_watchdog.sh --status # Show configured scripts and exit. # # system_watchdog.sh --log -# Verbose output. +# Passes --log to each sub-script for verbose output. # # ============================================================================================== @@ -86,7 +86,7 @@ acquire_lock detect_hosts -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no watchdog scripts will be executed" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts" # ============================================================================================== # ━━━ Status ━━━ @@ -119,46 +119,12 @@ fi echo "━━━ $ICON_SHIELD System Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━" START=$(date +%s) -PASSED=() -FAILED=() -STEP=0 +JOB_PASS=() +JOB_FAIL=() for entry in "${SYSTEM_WATCHDOG_SCRIPTS[@]}"; do [[ -z "$entry" ]] && continue - (( STEP++ )) - - read -r -a parts <<< "$entry" - script_path="$ECOSYSTEM_ROOT/${parts[0]}" - script_name=$(basename "${parts[0]}") - - if [[ ! -f "$script_path" ]]; then - error "$script_name — not found at $script_path" - FAILED+=("$script_name") - continue - fi - - if [[ ! -x "$script_path" ]]; then - chmod +x "$script_path" || { - error "$script_name — chmod +x failed" - FAILED+=("$script_name") - continue - } - fi - - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would run: $script_name" - PASSED+=("$script_name") - continue - fi - - _ss=$(date +%s) - if bash "$script_path"; then - log "$script_name — done in $(format_duration $(( $(date +%s) - _ss ))) ✅" - PASSED+=("$script_name") - else - warn "$script_name — exit non-zero in $(format_duration $(( $(date +%s) - _ss ))) (issues found or fixed) — continuing" - FAILED+=("$script_name") - fi + run_orch_child "$entry" done END=$(date +%s) @@ -166,7 +132,12 @@ END=$(date +%s) # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== -log "System watchdog — $STEP script(s) — $(format_duration $(( END - START )))" +log "System watchdog — ${#JOB_PASS[@]}/${#SYSTEM_WATCHDOG_SCRIPTS[@]} passed — $(format_duration $(( END - START )))" + +if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then + notify "System watchdog failed on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \ + "System Watchdog" "warning" + exit 1 +fi -[[ ${#FAILED[@]} -gt 0 ]] && exit 1 exit 0