diff --git a/Arrs_Stack/arr_sync.sh b/Arrs_Stack/arr_sync.sh index 59c02d3..eca7859 100755 --- a/Arrs_Stack/arr_sync.sh +++ b/Arrs_Stack/arr_sync.sh @@ -206,13 +206,7 @@ declare -A _ID_TYPE=([lidarr]="string" [sonarr]="int" [radarr]="int") ARR_TYPES=(lidarr sonarr radarr) # ── Remote node discovery ────────────────────────────────────────────────────────────────────── -REMOTE_NODES=() -for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do - [[ "$_hv" == "$MY_ID" ]] && continue - [[ -z "${!_hv:-}" ]] && continue - REMOTE_NODES+=("$_hv") -done -unset _hv +discover_remote_nodes if [[ "${#REMOTE_NODES[@]}" -eq 0 ]]; then warn "No remote nodes defined in master.conf — nothing to sync" @@ -478,7 +472,7 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "" echo " Arr Port URL" for _arr in "${ARR_TYPES[@]}"; do - local _url="" _key="" + _url="" case "$_arr" in lidarr) _url="${LIDARR_URL:-not configured}" ;; sonarr) _url="${SONARR_URL:-not configured}" ;; @@ -992,8 +986,8 @@ _sync_arr() { fi fi - local _n_local=${#to_add_local[@]:-}; _n_local=${_n_local:-0} - local _n_remote=${#to_add_remote[@]:-}; _n_remote=${_n_remote:-0} + local _n_local=${#to_add_local[@]} + local _n_remote=${#to_add_remote[@]} log " $node_name: +${_n_local} local | +${_n_remote} remote | $total_skipped blocklisted" unset remote_ids to_add_local to_add_remote _n_local _n_remote diff --git a/Arrs_Stack/lidarr_cleanup.sh b/Arrs_Stack/lidarr_cleanup.sh index cd2edcf..87dd6bc 100755 --- a/Arrs_Stack/lidarr_cleanup.sh +++ b/Arrs_Stack/lidarr_cleanup.sh @@ -60,7 +60,7 @@ # # acquire_lock "wait" — large scans take time, wait for previous run to finish # jq + curl validation — exits if either tool missing -# DOCKER_TIMEOUT — container checks protected against daemon hangs +# ARR_DOCKER_TIMEOUT — container checks protected against daemon hangs (script-local, not common.sh's DOCKER_TIMEOUT) # Duplicate detection — temp file of tracked paths, grep before delete # platform_require_cmd — notify script validated before use # Silent by default — orphans/junk warn(), clean library logs silently @@ -120,37 +120,12 @@ source "$SCRIPT_DIR/../load_config.sh" # ── Special flag pre-processing ─────────────────────────────────────────────────────────────── # Filter --i-know-what-im-doing and --skip-age-check before parse_args # to avoid unknown flag errors — both are handled separately below. -I_KNOW=false -SKIP_AGE_CHECK=false -FILTERED_ARGS=() -for arg in "$@"; do - case "$arg" in - --i-know-what-im-doing) I_KNOW=true ;; - --skip-age-check) SKIP_AGE_CHECK=true ;; - *) FILTERED_ARGS+=("$arg") ;; - esac -done +parse_destructive_flags "$@" parse_args "${FILTERED_ARGS[@]}" # ── Nuclear mode warning ────────────────────────────────────────────────────────────────────── -if [[ "$I_KNOW" == true ]] && [[ "$SKIP_AGE_CHECK" == true ]] && [[ "$DRY_RUN" != true ]]; then - echo "" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo "⚠️ WARNING — NUCLEAR MODE ACTIVE" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo " Flags: --i-know-what-im-doing --skip-age-check" - echo " Age check: BYPASSED — deletes on first pass" - echo " Size threshold: BYPASSED — no GB limit" - echo " Data recovery: NOT POSSIBLE after deletion" - echo "" - echo " Review --dry-run output before proceeding." - echo " You have 10 seconds to cancel (Ctrl+C)..." - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - sleep 10 - echo " Proceeding..." - echo "" -fi +nuclear_mode_warning # ============================================================================================== # ━━━ Setup ━━━ @@ -194,15 +169,11 @@ if [[ -z "${LIDARR_URL:-}" ]] || [[ -z "${LIDARR_API_KEY:-}" ]]; then exit 0 fi -DOCKER_TIMEOUT=15 +ARR_DOCKER_TIMEOUT=15 LIDARR_CONTAINER="Lidarr" # container name on HOST1 # Build path map from MY_ID's Lidarr path map -declare -A ARR_PATH_MAP -local_path_map_var="${MY_ID}_LIDARR_PATH_MAP" -eval "for key in \"\${!${local_path_map_var}[@]}\"; do - ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\" -done" +build_arr_path_map "LIDARR" # Validate required vars — detect_hosts() should have set these require_var LIDARR_URL @@ -251,83 +222,13 @@ fi echo "" echo "━━━ $ICON_SHIELD Safety Checks ━━━" -CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ - '{{.State.Running}}' "$LIDARR_CONTAINER" 2>/dev/null) -if [[ "$CONTAINER_RUNNING" != "true" ]]; then - error "$LIDARR_CONTAINER is not running — aborting" - notify "Lidarr cleanup aborted on $(hostname) — container not running" \ - "Lidarr Cleanup" "warning" - exit 1 -fi - -CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ - '{{.State.Health.Status}}' "$LIDARR_CONTAINER" 2>/dev/null) -case "$CONTAINER_HEALTH" in - healthy) info "$LIDARR_CONTAINER is healthy" ;; - "") info "$LIDARR_CONTAINER has no health check — proceeding" ;; - starting) - error "$LIDARR_CONTAINER is still starting — aborting" - notify "Lidarr cleanup aborted on $(hostname) — container still starting" \ - "Lidarr Cleanup" "warning" - exit 1 ;; - unhealthy) - error "$LIDARR_CONTAINER is unhealthy — aborting" - notify "Lidarr cleanup aborted on $(hostname) — container unhealthy" \ - "Lidarr Cleanup" "warning" - exit 1 ;; - *) warn "$LIDARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;; -esac - -info "Safety layer 1 passed — container healthy" +check_container_health "$LIDARR_CONTAINER" "$ARR_DOCKER_TIMEOUT" "Lidarr Cleanup" # ============================================================================================== # ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── # ============================================================================================== -# Lidarr API call with HTTP status check -# Usage: lidarr_api "artist" | lidarr_api "trackFile?artistId=123" -lidarr_api() { - local endpoint="$1" - local response http_code body - - response=$(curl -sf \ - --max-time 30 \ - -H "X-Api-Key: $LIDARR_API_KEY" \ - -w "\n%{http_code}" \ - "${LIDARR_URL}/api/v1/${endpoint}" 2>/dev/null) - - http_code=$(echo "$response" | tail -1) - body=$(echo "$response" | head -n -1) - - if [[ "$http_code" != "200" ]]; then - error "Lidarr API HTTP $http_code for: $endpoint" - return 1 - fi - echo "$body" -} - -# Check if a file extension is a tracked music format -is_music_file() { - local ext="${1##*.}" - ext="${ext,,}" - for valid_ext in "${LIDARR_EXTENSIONS[@]}"; do - [[ "$ext" == "$valid_ext" ]] && return 0 - done - return 1 -} - -# Check if a file matches any protected pattern -is_protected_file() { - local filename - filename=$(basename "$1") - for pattern in "${LIDARR_PROTECTED_PATTERNS[@]}"; do - # shellcheck disable=SC2254 - case "$filename" in - $pattern) return 0 ;; - esac - done - return 1 -} +# check_container_health(), arr_api(), has_extension(), matches_pattern_list() — common.sh # ============================================================================================== # ━━━ Pre-flight: Lidarr Import Scan ━━━ @@ -353,36 +254,7 @@ else SCAN_PAYLOAD='{"name": "DownloadedAlbumsScan"}' fi -SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \ - -H "X-Api-Key: $LIDARR_API_KEY" \ - -H "Content-Type: application/json" \ - -d "$SCAN_PAYLOAD" \ - "${LIDARR_URL}/api/v1/command" 2>/dev/null) - -SCAN_CMD_ID=$(echo "$SCAN_RESPONSE" | jq -r '.id // empty' 2>/dev/null) - -if [[ -z "$SCAN_CMD_ID" ]]; then - warn "Could not trigger import scan — proceeding without pre-flight" -else - info "Import scan queued (command ID: $SCAN_CMD_ID) — waiting for completion..." - POLL_TIMEOUT=${LIDARR_IMPORT_SCAN_TIMEOUT:-600} - POLLED=0 - while [[ "$POLLED" -lt "$POLL_TIMEOUT" ]]; do - SCAN_STATUS=$(curl -sf --max-time 10 \ - -H "X-Api-Key: $LIDARR_API_KEY" \ - "${LIDARR_URL}/api/v1/command/${SCAN_CMD_ID}" 2>/dev/null | \ - jq -r '.status // empty' 2>/dev/null) - case "$SCAN_STATUS" in - completed) info "Import scan complete ✅"; break ;; - failed) warn "Import scan reported failed — proceeding anyway"; break ;; - esac - sleep 10 - (( POLLED += 10 )) - [[ $(( POLLED % 60 )) -eq 0 ]] && log " Still scanning... (${POLLED}s elapsed)" - done - [[ "$POLLED" -ge "$POLL_TIMEOUT" ]] && \ - warn "Import scan timed out after ${POLL_TIMEOUT}s — proceeding anyway" -fi +trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$SCAN_PAYLOAD" "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}" # ============================================================================================== # ━━━ Fetch Lidarr Tracked Files ━━━ @@ -402,7 +274,7 @@ check_arr_version "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$LIDARR_VERSION_MAJOR" " info "Querying Lidarr API..." # Fetch all artists -ARTIST_RESPONSE=$(lidarr_api "artist") || { +ARTIST_RESPONSE=$(arr_api "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "artist" "Lidarr") || { error "Failed to fetch artists from Lidarr" notify "Lidarr cleanup failed on $(hostname) — could not fetch artists" \ "Lidarr Cleanup" "warning" @@ -427,7 +299,7 @@ TRACKED_FILE="$TMP_DIR/tracked_paths.txt" while IFS= read -r artist_id; do [[ -z "$artist_id" ]] && continue - ARTIST_TRACKS=$(lidarr_api "trackFile?artistId=${artist_id}" 2>/dev/null) + ARTIST_TRACKS=$(arr_api "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "trackFile?artistId=${artist_id}" "Lidarr" 2>/dev/null) if [[ -n "$ARTIST_TRACKS" ]]; then while IFS= read -r api_path; do [[ -z "$api_path" ]] && continue @@ -459,25 +331,7 @@ fi info "$ARTIST_COUNT artists | $TRACKED_COUNT tracked files" # Safety Layer 6 — percentage drop vs last known count -if [[ -f "$LIDARR_TRACKED_COUNT_FILE" ]]; then - LAST_COUNT=$(cat "$LIDARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0) - if [[ "$LAST_COUNT" -gt 0 ]]; then - PCT=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $LAST_COUNT) * 100}") - if [[ "$PCT" -lt "$LIDARR_MIN_TRACKED_PCT" ]]; then - error "Tracked count dropped to ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT)" - error "Suggests API issue — aborting to prevent mass deletion" - error "If expected (large library removal) delete: $LIDARR_TRACKED_COUNT_FILE" - notify "Lidarr cleanup aborted on $(hostname) — tracked count dropped to ${PCT}%" \ - "Lidarr Cleanup" "warning" - exit 1 - fi - info "Tracked count: ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT) ✅" - fi -else - info "No previous count on record — first run, saving baseline" -fi - -echo "$TRACKED_COUNT" > "$LIDARR_TRACKED_COUNT_FILE" +check_tracked_count_floor "$TRACKED_COUNT" "$LIDARR_TRACKED_COUNT_FILE" "$LIDARR_MIN_TRACKED_PCT" "Lidarr Cleanup" # ============================================================================================== # ━━━ Scan Music Root ━━━ @@ -496,7 +350,6 @@ JUNK_BYTES=0 AGE_SECONDS=$(( LIDARR_ORPHAN_AGE * 86400 )) NOW=$(date +%s) -MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $LIDARR_MAX_DELETE_GB * 1073741824}") while IFS= read -r filepath; do [[ -z "$filepath" ]] && continue @@ -508,7 +361,7 @@ while IFS= read -r filepath; do fi # Protected — never delete - if is_protected_file "$filepath"; then + if matches_pattern_list "$filepath" "${LIDARR_PROTECTED_PATTERNS[@]}"; then log "$ICON_PROTECTED PROTECTED: $filepath" (( PROTECTED_COUNT++ )) continue @@ -516,7 +369,7 @@ while IFS= read -r filepath; do FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0) - if is_music_file "$filepath"; then + if has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"; then FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0) FILE_AGE=$(( NOW - FILE_MTIME )) @@ -543,21 +396,7 @@ TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) # ============================================================================================== # ━━━ Safety Layer 7 — Deletion Size Threshold ━━━ # ============================================================================================== -if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then - TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}") - if [[ "$I_KNOW" != true ]]; then - echo "" - error "Deletion would exceed ${LIDARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted" - error "Review ORPHAN lines above carefully before proceeding" - error "Rerun with: --i-know-what-im-doing" - error "To also bypass age check: add --skip-age-check" - notify "Lidarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \ - "Lidarr Cleanup" "warning" - exit 1 - else - warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing" - fi -fi +check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$LIDARR_MAX_DELETE_GB" "Lidarr Cleanup" # ── Execute Deletions ───────────────────────────────────────────────────────────────────────── # All safety layers passed — delete orphans and junk @@ -565,12 +404,12 @@ if [[ "$DRY_RUN" == false ]]; then while IFS= read -r filepath; do [[ -z "$filepath" ]] && continue [[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue - is_protected_file "$filepath" && continue + matches_pattern_list "$filepath" "${LIDARR_PROTECTED_PATTERNS[@]}" && continue FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0) FILE_AGE=$(( NOW - FILE_MTIME )) - if is_music_file "$filepath"; then + if has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"; then [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \ [[ "$SKIP_AGE_CHECK" != true ]] && continue fi @@ -586,16 +425,7 @@ fi END=$(date +%s) -format_bytes() { - local bytes=$1 - if (( bytes > 1073741824 )); then - awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}" - elif (( bytes > 1048576 )); then - awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}" - else - echo "${bytes}B" - fi -} +# format_bytes() — provided by common.sh ORPHAN_HUMAN=$(format_bytes "$ORPHAN_BYTES") JUNK_HUMAN=$(format_bytes "$JUNK_BYTES") diff --git a/Arrs_Stack/lidarr_release_fixer.sh b/Arrs_Stack/lidarr_release_fixer.sh index dbb064e..47deee9 100755 --- a/Arrs_Stack/lidarr_release_fixer.sh +++ b/Arrs_Stack/lidarr_release_fixer.sh @@ -283,17 +283,6 @@ read_file_mbid() { esac } -translate_container_path() { - local cpath="$1" - for cp in "${!ARR_PATH_MAP[@]}"; do - if [[ "$cpath" == "${cp}"* ]]; then - echo "${ARR_PATH_MAP[$cp]}${cpath#$cp}" - return - fi - done - echo "$cpath" -} - # ── Safety checks ───────────────────────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_SHIELD Safety Checks ━━━" @@ -322,7 +311,7 @@ while IFS= read -r artist; do aid=$(echo "$artist" | jq -r '.id') apath=$(echo "$artist" | jq -r '.path // empty') aname=$(echo "$artist" | jq -r '.artistName // empty') - [[ -n "$apath" ]] && ARTIST_PATH_CACHE[$aid]=$(translate_container_path "$apath") + [[ -n "$apath" ]] && ARTIST_PATH_CACHE[$aid]=$(translate_path "$apath") [[ -n "$aname" ]] && ARTIST_NAME_CACHE[$aid]="$aname" done < <(echo "$ALL_ARTISTS" | jq -c '.[]' 2>/dev/null) unset ALL_ARTISTS diff --git a/Arrs_Stack/playback_aware_radarr_discovery.sh b/Arrs_Stack/playback_aware_radarr_discovery.sh index 2b7f4a0..62fef91 100755 --- a/Arrs_Stack/playback_aware_radarr_discovery.sh +++ b/Arrs_Stack/playback_aware_radarr_discovery.sh @@ -134,7 +134,6 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" -source "$SCRIPT_DIR/../Kernel/decision_engine.sh" parse_args "$@" @@ -185,7 +184,7 @@ log "$ICON_GEAR Config: threshold=${THRESHOLD} lookback=${LOOKBACK_DAYS}d max-se [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no movies will be added to Radarr" -_fmt_rating() { local v="${1:-0}"; echo "${v::-1}.${v: -1}" 2>/dev/null || echo "$v"; } +# _fmt_rating() — provided by common.sh # ============================================================================================== # ━━━ Status ━━━ @@ -275,35 +274,7 @@ _freq_score() { fi } -# vote_avg_int = vote_average × 10 as integer (e.g. 7.8 → 78) -_rating_score_s2() { - local v="$1" - if (( v >= 80 )); then echo 40 - elif (( v >= 75 )); then echo 32 - elif (( v >= 70 )); then echo 25 - elif (( v >= 65 )); then echo 18 - elif (( v >= 60 )); then echo 12 - else echo 5 - fi -} - -_votes_score() { - local c="$1" - if (( c >= 10000 )); then echo 20 - elif (( c >= 5000 )); then echo 15 - elif (( c >= 1000 )); then echo 10 - elif (( c >= 200 )); then echo 5 - else echo 2 - fi -} - -_breadth_score() { - local seeds="$1" - if (( seeds >= 3 )); then echo 40 - elif (( seeds == 2 )); then echo 25 - else echo 10 - fi -} +# _rating_score_s2(), _votes_score(), _breadth_score() — provided by common.sh # ============================================================================================== # ━━━ Fetch Emby Libraries + Recently Watched Movies ━━━ diff --git a/Arrs_Stack/playback_aware_sonarr_discovery.sh b/Arrs_Stack/playback_aware_sonarr_discovery.sh index efd9e15..b18667f 100755 --- a/Arrs_Stack/playback_aware_sonarr_discovery.sh +++ b/Arrs_Stack/playback_aware_sonarr_discovery.sh @@ -148,7 +148,6 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" -source "$SCRIPT_DIR/../Kernel/decision_engine.sh" parse_args "$@" @@ -205,7 +204,7 @@ log "$ICON_GEAR Config: threshold=${THRESHOLD} lookback=${LOOKBACK_DAYS}d max-se [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no shows will be added to Sonarr" -_fmt_rating() { local v="${1:-0}"; echo "${v::-1}.${v: -1}" 2>/dev/null || echo "$v"; } +# _fmt_rating() — provided by common.sh # ============================================================================================== # ━━━ Status ━━━ @@ -315,37 +314,7 @@ _volume_score() { fi } -# Stage 2: TMDB vote_average × 10 (0-40) -_rating_score_s2() { - local v="$1" - if (( v >= 80 )); then echo 40 - elif (( v >= 75 )); then echo 32 - elif (( v >= 70 )); then echo 25 - elif (( v >= 65 )); then echo 18 - elif (( v >= 60 )); then echo 12 - else echo 5 - fi -} - -# Stage 2: vote count (0-20) -_votes_score() { - local c="$1" - if (( c >= 10000 )); then echo 20 - elif (( c >= 5000 )); then echo 15 - elif (( c >= 1000 )); then echo 10 - elif (( c >= 200 )); then echo 5 - else echo 2 - fi -} - -# Stage 2: seed breadth (0-40) -_breadth_score() { - local seeds="$1" - if (( seeds >= 3 )); then echo 40 - elif (( seeds == 2 )); then echo 25 - else echo 10 - fi -} +# _rating_score_s2(), _votes_score(), _breadth_score() — provided by common.sh # ============================================================================================== # ━━━ Fetch Emby Series Library ━━━ @@ -636,9 +605,6 @@ done < <(echo "$SONARR_SERIES_JSON" | jq -r '.[] | log "${#SONARR_TVDB[@]} series in Sonarr | ${#EMBY_TMDB_IDS[@]} series in Emby" -_in_sonarr() { [[ "${SONARR_TVDB["$1"]+x}" || "${SONARR_TMDB["$2"]+x}" ]]; } -_in_emby() { [[ "${EMBY_TMDB_IDS["$1"]+x}" || "${EMBY_TVDB_IDS["$2"]+x}" ]]; } - # ============================================================================================== # ━━━ Score Stage 2 Candidates ━━━ # ============================================================================================== diff --git a/Arrs_Stack/radarr_cleanup.sh b/Arrs_Stack/radarr_cleanup.sh index c5a3672..13ca7ec 100755 --- a/Arrs_Stack/radarr_cleanup.sh +++ b/Arrs_Stack/radarr_cleanup.sh @@ -64,7 +64,7 @@ # # acquire_lock "wait" — large scans take time, wait for previous run to finish # jq + curl validation — exits if either tool missing -# DOCKER_TIMEOUT — container checks protected against daemon hangs +# ARR_DOCKER_TIMEOUT — container checks protected against daemon hangs (script-local, not common.sh's DOCKER_TIMEOUT) # notify_emby_scan() — triggers Emby clean after deletion # platform_require_cmd — notify script validated before use # Silent by default — orphans/junk warn(), clean library logs silently @@ -120,37 +120,12 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" # ── Special flag pre-processing ─────────────────────────────────────────────────────────────── -I_KNOW=false -SKIP_AGE_CHECK=false -FILTERED_ARGS=() -for arg in "$@"; do - case "$arg" in - --i-know-what-im-doing) I_KNOW=true ;; - --skip-age-check) SKIP_AGE_CHECK=true ;; - *) FILTERED_ARGS+=("$arg") ;; - esac -done +parse_destructive_flags "$@" parse_args "${FILTERED_ARGS[@]}" # ── Nuclear mode warning ────────────────────────────────────────────────────────────────────── -if [[ "$I_KNOW" == true ]] && [[ "$SKIP_AGE_CHECK" == true ]] && [[ "$DRY_RUN" != true ]]; then - echo "" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo "⚠️ WARNING — NUCLEAR MODE ACTIVE" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo " Flags: --i-know-what-im-doing --skip-age-check" - echo " Age check: BYPASSED — deletes on first pass" - echo " Size threshold: BYPASSED — no GB limit" - echo " Data recovery: NOT POSSIBLE after deletion" - echo "" - echo " Review --dry-run output before proceeding." - echo " You have 10 seconds to cancel (Ctrl+C)..." - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - sleep 10 - echo " Proceeding..." - echo "" -fi +nuclear_mode_warning # ============================================================================================== # ━━━ Setup ━━━ @@ -191,15 +166,11 @@ if [[ -z "${RADARR_URL:-}" ]] || [[ -z "${RADARR_API_KEY:-}" ]]; then exit 0 fi -DOCKER_TIMEOUT=15 +ARR_DOCKER_TIMEOUT=15 RADARR_CONTAINER="Radarr" # Build path map from MY_ID's Radarr path map -declare -A ARR_PATH_MAP -local_path_map_var="${MY_ID}_RADARR_PATH_MAP" -eval "for key in \"\${!${local_path_map_var}[@]}\"; do - ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\" -done" +build_arr_path_map "RADARR" require_var RADARR_URL require_var RADARR_API_KEY @@ -247,90 +218,13 @@ fi echo "" echo "━━━ $ICON_SHIELD Safety Checks ━━━" -CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ - '{{.State.Running}}' "$RADARR_CONTAINER" 2>/dev/null) -if [[ "$CONTAINER_RUNNING" != "true" ]]; then - error "$RADARR_CONTAINER is not running — aborting" - notify "Radarr cleanup aborted on $(hostname) — container not running" \ - "Radarr Cleanup" "warning" - exit 1 -fi - -CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ - '{{.State.Health.Status}}' "$RADARR_CONTAINER" 2>/dev/null) -case "$CONTAINER_HEALTH" in - healthy) info "$RADARR_CONTAINER is healthy" ;; - "") info "$RADARR_CONTAINER has no health check — proceeding" ;; - starting) - error "$RADARR_CONTAINER is still starting — aborting" - notify "Radarr cleanup aborted on $(hostname) — container still starting" \ - "Radarr Cleanup" "warning" - exit 1 ;; - unhealthy) - error "$RADARR_CONTAINER is unhealthy — aborting" - notify "Radarr cleanup aborted on $(hostname) — container unhealthy" \ - "Radarr Cleanup" "warning" - exit 1 ;; - *) warn "$RADARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;; -esac - -info "Safety layer 1 passed — container healthy" +check_container_health "$RADARR_CONTAINER" "$ARR_DOCKER_TIMEOUT" "Radarr Cleanup" # ============================================================================================== # ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── # ============================================================================================== -radarr_api() { - local endpoint="$1" - local response http_code body - - response=$(curl -sf \ - --max-time 30 \ - -H "X-Api-Key: $RADARR_API_KEY" \ - -w "\n%{http_code}" \ - "${RADARR_URL}/api/v3/${endpoint}" 2>/dev/null) - - http_code=$(echo "$response" | tail -1) - body=$(echo "$response" | head -n -1) - - if [[ "$http_code" != "200" ]]; then - error "Radarr API HTTP $http_code for: $endpoint" - return 1 - fi - echo "$body" -} - -is_video_file() { - local ext="${1##*.}" - ext="${ext,,}" - for valid_ext in "${RADARR_EXTENSIONS[@]}"; do - [[ "$ext" == "$valid_ext" ]] && return 0 - done - return 1 -} - -is_protected_file() { - local filename - filename=$(basename "$1") - for pattern in "${RADARR_PROTECTED_PATTERNS[@]}"; do - # shellcheck disable=SC2254 - case "$filename" in - $pattern) return 0 ;; - esac - done - return 1 -} - -format_bytes() { - local bytes=$1 - if (( bytes > 1073741824 )); then - awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}" - elif (( bytes > 1048576 )); then - awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}" - else - echo "${bytes}B" - fi -} +# check_container_health(), arr_api(), has_extension(), matches_pattern_list(), format_bytes() — common.sh # ============================================================================================== # ━━━ Pre-flight: Radarr Import Scan ━━━ @@ -340,7 +234,7 @@ echo "━━━ $ICON_SYNC Pre-flight: Radarr Import Scan ━━━" # Fetch root folders from Radarr API and translate container paths to host paths mapfile -t SCAN_ROOTS < <( - radarr_api "rootfolder" | \ + arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "rootfolder" "Radarr" | \ jq -r '.[].path' 2>/dev/null | \ while IFS= read -r cp; do translate_path "$cp"; done ) @@ -357,36 +251,7 @@ info "Scan targets (${#SCAN_ROOTS[@]}): ${SCAN_ROOTS[*]}" info "Triggering ProcessMonitoredDownloads pre-flight" SCAN_PAYLOAD='{"name": "ProcessMonitoredDownloads"}' -SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \ - -H "X-Api-Key: $RADARR_API_KEY" \ - -H "Content-Type: application/json" \ - -d "$SCAN_PAYLOAD" \ - "${RADARR_URL}/api/v3/command" 2>/dev/null) - -SCAN_CMD_ID=$(echo "$SCAN_RESPONSE" | jq -r '.id // empty' 2>/dev/null) - -if [[ -z "$SCAN_CMD_ID" ]]; then - warn "Could not trigger import scan — proceeding without pre-flight" -else - info "Import scan queued (command ID: $SCAN_CMD_ID) — waiting for completion..." - POLL_TIMEOUT=${RADARR_IMPORT_SCAN_TIMEOUT:-600} - POLLED=0 - while [[ "$POLLED" -lt "$POLL_TIMEOUT" ]]; do - SCAN_STATUS=$(curl -sf --max-time 10 \ - -H "X-Api-Key: $RADARR_API_KEY" \ - "${RADARR_URL}/api/v3/command/${SCAN_CMD_ID}" 2>/dev/null | \ - jq -r '.status // empty' 2>/dev/null) - case "$SCAN_STATUS" in - completed) info "Import scan complete ✅"; break ;; - failed) warn "Import scan reported failed — proceeding anyway"; break ;; - esac - sleep 10 - (( POLLED += 10 )) - [[ $(( POLLED % 60 )) -eq 0 ]] && log " Still scanning... (${POLLED}s elapsed)" - done - [[ "$POLLED" -ge "$POLL_TIMEOUT" ]] && \ - warn "Import scan timed out after ${POLL_TIMEOUT}s — proceeding anyway" -fi +trigger_and_await_command "$RADARR_URL" "$RADARR_API_KEY" "v3" "$SCAN_PAYLOAD" "${RADARR_IMPORT_SCAN_TIMEOUT:-600}" # ============================================================================================== # ━━━ Fetch Radarr Tracked Files ━━━ @@ -406,7 +271,7 @@ check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" " info "Querying Radarr API..." # Fetch all movies -MOVIES_RESPONSE=$(radarr_api "movie") || { +MOVIES_RESPONSE=$(arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "movie" "Radarr") || { error "Failed to fetch movies from Radarr" notify "Radarr cleanup failed on $(hostname) — could not fetch movies" \ "Radarr Cleanup" "warning" @@ -435,7 +300,7 @@ while IFS= read -r movie_id; do (( MOVIE_INDEX++ )) [[ $(( MOVIE_INDEX % 100 )) -eq 0 ]] && \ log "Fetching files: $MOVIE_INDEX/$MOVIE_COUNT movies..." - MOVIE_FILES=$(radarr_api "moviefile?movieId=${movie_id}" 2>/dev/null) + MOVIE_FILES=$(arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "moviefile?movieId=${movie_id}" "Radarr" 2>/dev/null) if [[ -n "$MOVIE_FILES" ]]; then while IFS= read -r api_path; do [[ -z "$api_path" ]] && continue @@ -467,25 +332,7 @@ fi info "$MOVIE_COUNT movies | $TRACKED_COUNT tracked movie files" # Safety Layer 6 — percentage drop vs last known count -if [[ -f "$RADARR_TRACKED_COUNT_FILE" ]]; then - LAST_COUNT=$(cat "$RADARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0) - if [[ "$LAST_COUNT" -gt 0 ]]; then - PCT=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $LAST_COUNT) * 100}") - if [[ "$PCT" -lt "$RADARR_MIN_TRACKED_PCT" ]]; then - error "Tracked count dropped to ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT)" - error "Suggests API issue — aborting to prevent mass deletion" - error "If expected (large library removal) delete: $RADARR_TRACKED_COUNT_FILE" - notify "Radarr cleanup aborted on $(hostname) — tracked count dropped to ${PCT}%" \ - "Radarr Cleanup" "warning" - exit 1 - fi - info "Tracked count: ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT) ✅" - fi -else - info "No previous count on record — first run, saving baseline" -fi - -echo "$TRACKED_COUNT" > "$RADARR_TRACKED_COUNT_FILE" +check_tracked_count_floor "$TRACKED_COUNT" "$RADARR_TRACKED_COUNT_FILE" "$RADARR_MIN_TRACKED_PCT" "Radarr Cleanup" # ============================================================================================== # ━━━ Scan Movies Root ━━━ @@ -504,7 +351,6 @@ JUNK_BYTES=0 AGE_SECONDS=$(( RADARR_ORPHAN_AGE * 86400 )) NOW=$(date +%s) -MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $RADARR_MAX_DELETE_GB * 1073741824}") while IFS= read -r filepath; do [[ -z "$filepath" ]] && continue @@ -514,7 +360,7 @@ while IFS= read -r filepath; do continue fi - if is_protected_file "$filepath"; then + if matches_pattern_list "$filepath" "${RADARR_PROTECTED_PATTERNS[@]}"; then log "$ICON_PROTECTED PROTECTED: $filepath" (( PROTECTED_COUNT++ )) continue @@ -522,7 +368,7 @@ while IFS= read -r filepath; do FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0) - if is_video_file "$filepath"; then + if has_extension "$filepath" "${RADARR_EXTENSIONS[@]}"; then FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0) FILE_AGE=$(( NOW - FILE_MTIME )) @@ -553,33 +399,19 @@ TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) # ============================================================================================== # ━━━ Safety Layer 7 — Deletion Size Threshold ━━━ # ============================================================================================== -if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then - TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}") - if [[ "$I_KNOW" != true ]]; then - echo "" - error "Deletion would exceed ${RADARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted" - error "Review ORPHAN lines above carefully before proceeding" - error "Rerun with: --i-know-what-im-doing" - error "To also bypass age check: add --skip-age-check" - notify "Radarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \ - "Radarr Cleanup" "warning" - exit 1 - else - warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing" - fi -fi +check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$RADARR_MAX_DELETE_GB" "Radarr Cleanup" # ── Execute Deletions ───────────────────────────────────────────────────────────────────────── if [[ "$DRY_RUN" == false ]]; then while IFS= read -r filepath; do [[ -z "$filepath" ]] && continue [[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue - is_protected_file "$filepath" && continue + matches_pattern_list "$filepath" "${RADARR_PROTECTED_PATTERNS[@]}" && continue FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0) FILE_AGE=$(( NOW - FILE_MTIME )) - if is_video_file "$filepath"; then + if has_extension "$filepath" "${RADARR_EXTENSIONS[@]}"; then [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \ [[ "$SKIP_AGE_CHECK" != true ]] && continue fi diff --git a/Arrs_Stack/sonarr_cleanup.sh b/Arrs_Stack/sonarr_cleanup.sh index da20ec3..79623b0 100755 --- a/Arrs_Stack/sonarr_cleanup.sh +++ b/Arrs_Stack/sonarr_cleanup.sh @@ -64,7 +64,7 @@ # # acquire_lock "wait" — large scans take time, wait for previous run to finish # jq + curl validation — exits if either tool missing -# DOCKER_TIMEOUT — container checks protected against daemon hangs +# ARR_DOCKER_TIMEOUT — container checks protected against daemon hangs (script-local, not common.sh's DOCKER_TIMEOUT) # notify_emby_scan() — triggers Emby clean after deletion # platform_require_cmd — notify script validated before use # Silent by default — orphans/junk warn(), clean library logs silently @@ -120,37 +120,12 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" # ── Special flag pre-processing ─────────────────────────────────────────────────────────────── -I_KNOW=false -SKIP_AGE_CHECK=false -FILTERED_ARGS=() -for arg in "$@"; do - case "$arg" in - --i-know-what-im-doing) I_KNOW=true ;; - --skip-age-check) SKIP_AGE_CHECK=true ;; - *) FILTERED_ARGS+=("$arg") ;; - esac -done +parse_destructive_flags "$@" parse_args "${FILTERED_ARGS[@]}" # ── Nuclear mode warning ────────────────────────────────────────────────────────────────────── -if [[ "$I_KNOW" == true ]] && [[ "$SKIP_AGE_CHECK" == true ]] && [[ "$DRY_RUN" != true ]]; then - echo "" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo "⚠️ WARNING — NUCLEAR MODE ACTIVE" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo " Flags: --i-know-what-im-doing --skip-age-check" - echo " Age check: BYPASSED — deletes on first pass" - echo " Size threshold: BYPASSED — no GB limit" - echo " Data recovery: NOT POSSIBLE after deletion" - echo "" - echo " Review --dry-run output before proceeding." - echo " You have 10 seconds to cancel (Ctrl+C)..." - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - sleep 10 - echo " Proceeding..." - echo "" -fi +nuclear_mode_warning # ============================================================================================== # ━━━ Setup ━━━ @@ -191,15 +166,11 @@ if [[ -z "${SONARR_URL:-}" ]] || [[ -z "${SONARR_API_KEY:-}" ]]; then exit 0 fi -DOCKER_TIMEOUT=15 +ARR_DOCKER_TIMEOUT=15 SONARR_CONTAINER="Sonarr" # Build path map from MY_ID's Sonarr path map -declare -A ARR_PATH_MAP -local_path_map_var="${MY_ID}_SONARR_PATH_MAP" -eval "for key in \"\${!${local_path_map_var}[@]}\"; do - ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\" -done" +build_arr_path_map "SONARR" require_var SONARR_URL require_var SONARR_API_KEY @@ -247,90 +218,13 @@ fi echo "" echo "━━━ $ICON_SHIELD Safety Checks ━━━" -CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ - '{{.State.Running}}' "$SONARR_CONTAINER" 2>/dev/null) -if [[ "$CONTAINER_RUNNING" != "true" ]]; then - error "$SONARR_CONTAINER is not running — aborting" - notify "Sonarr cleanup aborted on $(hostname) — container not running" \ - "Sonarr Cleanup" "warning" - exit 1 -fi - -CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ - '{{.State.Health.Status}}' "$SONARR_CONTAINER" 2>/dev/null) -case "$CONTAINER_HEALTH" in - healthy) info "$SONARR_CONTAINER is healthy" ;; - "") info "$SONARR_CONTAINER has no health check — proceeding" ;; - starting) - error "$SONARR_CONTAINER is still starting — aborting" - notify "Sonarr cleanup aborted on $(hostname) — container still starting" \ - "Sonarr Cleanup" "warning" - exit 1 ;; - unhealthy) - error "$SONARR_CONTAINER is unhealthy — aborting" - notify "Sonarr cleanup aborted on $(hostname) — container unhealthy" \ - "Sonarr Cleanup" "warning" - exit 1 ;; - *) warn "$SONARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;; -esac - -info "Safety layer 1 passed — container healthy" +check_container_health "$SONARR_CONTAINER" "$ARR_DOCKER_TIMEOUT" "Sonarr Cleanup" # ============================================================================================== # ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── # ============================================================================================== -sonarr_api() { - local endpoint="$1" - local response http_code body - - response=$(curl -sf \ - --max-time 30 \ - -H "X-Api-Key: $SONARR_API_KEY" \ - -w "\n%{http_code}" \ - "${SONARR_URL}/api/v3/${endpoint}" 2>/dev/null) - - http_code=$(echo "$response" | tail -1) - body=$(echo "$response" | head -n -1) - - if [[ "$http_code" != "200" ]]; then - error "Sonarr API HTTP $http_code for: $endpoint" - return 1 - fi - echo "$body" -} - -is_video_file() { - local ext="${1##*.}" - ext="${ext,,}" - for valid_ext in "${SONARR_EXTENSIONS[@]}"; do - [[ "$ext" == "$valid_ext" ]] && return 0 - done - return 1 -} - -is_protected_file() { - local filename - filename=$(basename "$1") - for pattern in "${SONARR_PROTECTED_PATTERNS[@]}"; do - # shellcheck disable=SC2254 - case "$filename" in - $pattern) return 0 ;; - esac - done - return 1 -} - -format_bytes() { - local bytes=$1 - if (( bytes > 1073741824 )); then - awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}" - elif (( bytes > 1048576 )); then - awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}" - else - echo "${bytes}B" - fi -} +# check_container_health(), arr_api(), has_extension(), matches_pattern_list(), format_bytes() — common.sh # ============================================================================================== # ━━━ Pre-flight: Sonarr Import Scan ━━━ @@ -340,7 +234,7 @@ echo "━━━ $ICON_SYNC Pre-flight: Sonarr Import Scan ━━━" # Fetch root folders from Sonarr API and translate container paths to host paths mapfile -t SCAN_ROOTS < <( - sonarr_api "rootfolder" | \ + arr_api "$SONARR_URL" "$SONARR_API_KEY" "v3" "rootfolder" "Sonarr" | \ jq -r '.[].path' 2>/dev/null | \ while IFS= read -r cp; do translate_path "$cp"; done ) @@ -357,36 +251,7 @@ info "Scan targets (${#SCAN_ROOTS[@]}): ${SCAN_ROOTS[*]}" info "Triggering ProcessMonitoredDownloads pre-flight" SCAN_PAYLOAD='{"name": "ProcessMonitoredDownloads"}' -SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \ - -H "X-Api-Key: $SONARR_API_KEY" \ - -H "Content-Type: application/json" \ - -d "$SCAN_PAYLOAD" \ - "${SONARR_URL}/api/v3/command" 2>/dev/null) - -SCAN_CMD_ID=$(echo "$SCAN_RESPONSE" | jq -r '.id // empty' 2>/dev/null) - -if [[ -z "$SCAN_CMD_ID" ]]; then - warn "Could not trigger import scan — proceeding without pre-flight" -else - info "Import scan queued (command ID: $SCAN_CMD_ID) — waiting for completion..." - POLL_TIMEOUT=${SONARR_IMPORT_SCAN_TIMEOUT:-600} - POLLED=0 - while [[ "$POLLED" -lt "$POLL_TIMEOUT" ]]; do - SCAN_STATUS=$(curl -sf --max-time 10 \ - -H "X-Api-Key: $SONARR_API_KEY" \ - "${SONARR_URL}/api/v3/command/${SCAN_CMD_ID}" 2>/dev/null | \ - jq -r '.status // empty' 2>/dev/null) - case "$SCAN_STATUS" in - completed) info "Import scan complete ✅"; break ;; - failed) warn "Import scan reported failed — proceeding anyway"; break ;; - esac - sleep 10 - (( POLLED += 10 )) - [[ $(( POLLED % 60 )) -eq 0 ]] && log " Still scanning... (${POLLED}s elapsed)" - done - [[ "$POLLED" -ge "$POLL_TIMEOUT" ]] && \ - warn "Import scan timed out after ${POLL_TIMEOUT}s — proceeding anyway" -fi +trigger_and_await_command "$SONARR_URL" "$SONARR_API_KEY" "v3" "$SCAN_PAYLOAD" "${SONARR_IMPORT_SCAN_TIMEOUT:-600}" # ============================================================================================== # ━━━ Fetch Sonarr Tracked Files ━━━ @@ -406,7 +271,7 @@ check_arr_version "$SONARR_URL" "$SONARR_API_KEY" "v3" "$SONARR_VERSION_MAJOR" " info "Querying Sonarr API..." # Fetch all series -SERIES_RESPONSE=$(sonarr_api "series") || { +SERIES_RESPONSE=$(arr_api "$SONARR_URL" "$SONARR_API_KEY" "v3" "series" "Sonarr") || { error "Failed to fetch series from Sonarr" notify "Sonarr cleanup failed on $(hostname) — could not fetch series" \ "Sonarr Cleanup" "warning" @@ -435,7 +300,7 @@ while IFS= read -r series_id; do (( SERIES_INDEX++ )) [[ $(( SERIES_INDEX % 50 )) -eq 0 ]] && \ log "Fetching files: $SERIES_INDEX/$SERIES_COUNT series..." - SERIES_FILES=$(sonarr_api "episodefile?seriesId=${series_id}" 2>/dev/null) + SERIES_FILES=$(arr_api "$SONARR_URL" "$SONARR_API_KEY" "v3" "episodefile?seriesId=${series_id}" "Sonarr" 2>/dev/null) if [[ -n "$SERIES_FILES" ]]; then while IFS= read -r api_path; do [[ -z "$api_path" ]] && continue @@ -466,25 +331,7 @@ fi info "$SERIES_COUNT series | $TRACKED_COUNT tracked episode files" # Safety Layer 6 — percentage drop vs last known count -if [[ -f "$SONARR_TRACKED_COUNT_FILE" ]]; then - LAST_COUNT=$(cat "$SONARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0) - if [[ "$LAST_COUNT" -gt 0 ]]; then - PCT=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $LAST_COUNT) * 100}") - if [[ "$PCT" -lt "$SONARR_MIN_TRACKED_PCT" ]]; then - error "Tracked count dropped to ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT)" - error "Suggests API issue — aborting to prevent mass deletion" - error "If expected (large library removal) delete: $SONARR_TRACKED_COUNT_FILE" - notify "Sonarr cleanup aborted on $(hostname) — tracked count dropped to ${PCT}%" \ - "Sonarr Cleanup" "warning" - exit 1 - fi - info "Tracked count: ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT) ✅" - fi -else - info "No previous count on record — first run, saving baseline" -fi - -echo "$TRACKED_COUNT" > "$SONARR_TRACKED_COUNT_FILE" +check_tracked_count_floor "$TRACKED_COUNT" "$SONARR_TRACKED_COUNT_FILE" "$SONARR_MIN_TRACKED_PCT" "Sonarr Cleanup" # ============================================================================================== # ━━━ Scan TV Root ━━━ @@ -503,7 +350,6 @@ JUNK_BYTES=0 AGE_SECONDS=$(( SONARR_ORPHAN_AGE * 86400 )) NOW=$(date +%s) -MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $SONARR_MAX_DELETE_GB * 1073741824}") while IFS= read -r filepath; do [[ -z "$filepath" ]] && continue @@ -513,7 +359,7 @@ while IFS= read -r filepath; do continue fi - if is_protected_file "$filepath"; then + if matches_pattern_list "$filepath" "${SONARR_PROTECTED_PATTERNS[@]}"; then log "$ICON_PROTECTED PROTECTED: $filepath" (( PROTECTED_COUNT++ )) continue @@ -521,7 +367,7 @@ while IFS= read -r filepath; do FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0) - if is_video_file "$filepath"; then + if has_extension "$filepath" "${SONARR_EXTENSIONS[@]}"; then FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0) FILE_AGE=$(( NOW - FILE_MTIME )) @@ -552,33 +398,19 @@ TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) # ============================================================================================== # ━━━ Safety Layer 7 — Deletion Size Threshold ━━━ # ============================================================================================== -if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then - TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}") - if [[ "$I_KNOW" != true ]]; then - echo "" - error "Deletion would exceed ${SONARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted" - error "Review ORPHAN lines above carefully before proceeding" - error "Rerun with: --i-know-what-im-doing" - error "To also bypass age check: add --skip-age-check" - notify "Sonarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \ - "Sonarr Cleanup" "warning" - exit 1 - else - warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing" - fi -fi +check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$SONARR_MAX_DELETE_GB" "Sonarr Cleanup" # ── Execute Deletions ───────────────────────────────────────────────────────────────────────── if [[ "$DRY_RUN" == false ]]; then while IFS= read -r filepath; do [[ -z "$filepath" ]] && continue [[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue - is_protected_file "$filepath" && continue + matches_pattern_list "$filepath" "${SONARR_PROTECTED_PATTERNS[@]}" && continue FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0) FILE_AGE=$(( NOW - FILE_MTIME )) - if is_video_file "$filepath"; then + if has_extension "$filepath" "${SONARR_EXTENSIONS[@]}"; then [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \ [[ "$SKIP_AGE_CHECK" != true ]] && continue fi diff --git a/Arrs_Stack/upgrade_webhook_handler.sh b/Arrs_Stack/upgrade_webhook_handler.sh index ad042a8..7ce84a0 100755 --- a/Arrs_Stack/upgrade_webhook_handler.sh +++ b/Arrs_Stack/upgrade_webhook_handler.sh @@ -103,13 +103,7 @@ case "$ARR_TYPE" in esac # ── Remote node list ────────────────────────────────────────────────────────────────────────── -declare -a REMOTE_NODES=() -for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do - [[ "$_hv" == "$MY_ID" ]] && continue - [[ -z "${!_hv:-}" ]] && continue - REMOTE_NODES+=("$_hv") -done -unset _hv +discover_remote_nodes if [[ "${#REMOTE_NODES[@]}" -eq 0 ]]; then echo "No remote nodes configured — nothing to push" diff --git a/Configurations/master.conf b/Configurations/master.conf index 870a2fe..a5e7db7 100644 --- a/Configurations/master.conf +++ b/Configurations/master.conf @@ -802,6 +802,9 @@ # Crash loop threshold WATCHDOG_CRASH_LIMIT=5 # RestartCount above this = critical crash loop +# Required-container strikes — consecutive down-checks before docker_watchdog.sh attempts a restart + WATCHDOG_REQUIRED_STRIKE_LIMIT=2 + # Startup grace period — skip restarts while system is still booting WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures @@ -891,6 +894,9 @@ # Seconds of warning broadcast to logged-in users before server_reboot.sh reboots. # Gives users time to save work — 300s = 5 minutes. REBOOT_SLEEP=300 +# Seconds to wait for graceful VM shutdown (ACPI signal via virsh) before libvirt +# stops it anyway — reboot takes priority. + REBOOT_VM_WAIT=30 # ━━━ Mover ━━━ # Seconds to wait before mover_stop.sh sends SIGTERM to the mover process. @@ -1085,6 +1091,7 @@ LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run # protects against API returning partial data on a bad day LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count" + LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") LIDARR_PROTECTED_PATTERNS=( # Metadata @@ -1153,6 +1160,7 @@ SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run # protects against API returning partial data on a bad day SONARR_TRACKED_COUNT_FILE="$DATA_DIR/sonarr_tracked.count" + SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") SONARR_PROTECTED_PATTERNS=( # Subtitles @@ -1179,6 +1187,7 @@ RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run # protects against API returning partial data on a bad day RADARR_TRACKED_COUNT_FILE="$DATA_DIR/radarr_tracked.count" + RADARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") RADARR_PROTECTED_PATTERNS=( # Subtitles diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 98fa6ce..f46c34b 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -797,6 +797,9 @@ # Crash loop threshold WATCHDOG_CRASH_LIMIT=5 # RestartCount above this = critical crash loop +# Required-container strikes — consecutive down-checks before docker_watchdog.sh attempts a restart + WATCHDOG_REQUIRED_STRIKE_LIMIT=2 + # Startup grace period — skip restarts while system is still booting WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures @@ -886,6 +889,9 @@ # Seconds of warning broadcast to logged-in users before server_reboot.sh reboots. # Gives users time to save work — 300s = 5 minutes. REBOOT_SLEEP=300 +# Seconds to wait for graceful VM shutdown (ACPI signal via virsh) before libvirt +# stops it anyway — reboot takes priority. + REBOOT_VM_WAIT=30 # ━━━ Mover ━━━ # Seconds to wait before mover_stop.sh sends SIGTERM to the mover process. @@ -1080,6 +1086,7 @@ LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run # protects against API returning partial data on a bad day LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count" + LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") LIDARR_PROTECTED_PATTERNS=( # Metadata @@ -1144,6 +1151,7 @@ SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run # protects against API returning partial data on a bad day SONARR_TRACKED_COUNT_FILE="$DATA_DIR/sonarr_tracked.count" + SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") SONARR_PROTECTED_PATTERNS=( # Subtitles @@ -1168,6 +1176,7 @@ RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run # protects against API returning partial data on a bad day RADARR_TRACKED_COUNT_FILE="$DATA_DIR/radarr_tracked.count" + RADARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") RADARR_PROTECTED_PATTERNS=( # Subtitles diff --git a/Docker_Essentials/docker_container_stop.sh b/Docker_Essentials/docker_container_stop.sh index 90ffd26..5c9d518 100755 --- a/Docker_Essentials/docker_container_stop.sh +++ b/Docker_Essentials/docker_container_stop.sh @@ -186,9 +186,7 @@ FAILED=() for container in "${RUNNING[@]}"; do [[ -z "$container" ]] && continue - local c_start c_start=$(date +%s) - local c_image c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown") log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━" diff --git a/Docker_Essentials/docker_daily_restart.sh b/Docker_Essentials/docker_daily_restart.sh index eeca1cd..b02ca6f 100755 --- a/Docker_Essentials/docker_daily_restart.sh +++ b/Docker_Essentials/docker_daily_restart.sh @@ -159,71 +159,7 @@ fi # docker_cmd, verify_running, retry_docker — defined in common.sh -# Builds a dependency-safe restart order from DAILY_RESTART_CONTAINERS. -# Containers that are dependencies of others restart first. -# Returns ordered list in ORDERED_RESTART array. -build_restart_order() { - ORDERED_RESTART=() - local remaining=("${DAILY_RESTART_CONTAINERS[@]}") - local placed=() - - # First pass — add dependency containers that appear in our list - for container in "${remaining[@]}"; do - [[ -z "$container" ]] && continue - local is_dependency=false - # Check if this container is a dependency of any other in our list - for dep_string in "${WATCHDOG_DEPENDENCIES[@]}"; do - if [[ "$dep_string" == *"$container"* ]]; then - is_dependency=true - break - fi - done - # Also check associative array format - for dependent in "${!WATCHDOG_DEPENDENCIES[@]}"; do - if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then - is_dependency=true - break - fi - done - if [[ "$is_dependency" == true ]]; then - # Check not already placed - local already=false - for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done - if [[ "$already" == false ]]; then - ORDERED_RESTART+=("$container") - placed+=("$container") - fi - fi - done - - # Second pass — add remaining containers (dependents and independents) - for container in "${remaining[@]}"; do - [[ -z "$container" ]] && continue - local already=false - for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done - if [[ "$already" == false ]]; then - ORDERED_RESTART+=("$container") - placed+=("$container") - fi - done - -} - -# Checks if a container is a dependent of the previously restarted container. -# If so, waits CONTAINER_DELAY before restarting to allow dependency to settle. -# Usage: check_dependency_delay "$container" "$last_restarted" -check_dependency_delay() { - local container="$1" - local last="$2" - [[ -z "$last" ]] && return - - local deps="${WATCHDOG_DEPENDENCIES[$container]:-}" - if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then - echo " Waiting ${CONTAINER_DELAY}s — $container depends on $last..." - sleep "$CONTAINER_DELAY" - fi -} - +# build_restart_order() / check_dependency_delay() — provided by common.sh # ============================================================================================== # ━━━ Daily Restart ━━━ @@ -241,8 +177,7 @@ RESTARTED=() SKIPPED=() # Build dependency-safe restart order -build_restart_order -log "$ICON_GEAR Restart order: ${ORDERED_RESTART[*]}" +build_restart_order DAILY_RESTART_CONTAINERS LAST_RESTARTED="" diff --git a/Docker_Essentials/docker_weekly_restart.sh b/Docker_Essentials/docker_weekly_restart.sh index a025f27..b5d5c1b 100755 --- a/Docker_Essentials/docker_weekly_restart.sh +++ b/Docker_Essentials/docker_weekly_restart.sh @@ -158,58 +158,7 @@ fi # docker_cmd, retry_docker, verify_running — defined in common.sh -# Builds a dependency-safe restart order from WEEKLY_RESTART_CONTAINERS. -# Containers that are dependencies of others restart first. -build_restart_order() { - ORDERED_RESTART=() - local remaining=("${WEEKLY_RESTART_CONTAINERS[@]}") - local placed=() - - # First pass — add dependency containers that appear in our list - for container in "${remaining[@]}"; do - [[ -z "$container" ]] && continue - local is_dependency=false - for dependent in "${!WATCHDOG_DEPENDENCIES[@]}"; do - if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then - is_dependency=true - break - fi - done - if [[ "$is_dependency" == true ]]; then - local already=false - for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done - if [[ "$already" == false ]]; then - ORDERED_RESTART+=("$container") - placed+=("$container") - fi - fi - done - - # Second pass — add remaining containers (dependents and independents) - for container in "${remaining[@]}"; do - [[ -z "$container" ]] && continue - local already=false - for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done - if [[ "$already" == false ]]; then - ORDERED_RESTART+=("$container") - placed+=("$container") - fi - done - - log "Restart order: ${ORDERED_RESTART[*]}" -} - -# Waits CONTAINER_DELAY if this container depends on the last restarted one. -check_dependency_delay() { - local container="$1" - local last="$2" - [[ -z "$last" ]] && return - local deps="${WATCHDOG_DEPENDENCIES[$container]:-}" - if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then - log "Waiting ${CONTAINER_DELAY}s — $container depends on $last..." - sleep "$CONTAINER_DELAY" - fi -} +# build_restart_order() / check_dependency_delay() — provided by common.sh # ============================================================================================== # ━━━ Weekly Restart ━━━ @@ -226,15 +175,13 @@ RESTARTED=() SKIPPED=() # Build dependency-safe restart order -build_restart_order +build_restart_order WEEKLY_RESTART_CONTAINERS LAST_RESTARTED="" for container in "${ORDERED_RESTART[@]}"; do [[ -z "$container" ]] && continue - local c_start c_start=$(date +%s) - local c_image c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown") log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━" diff --git a/Docker_Essentials/downloaders_reset.sh b/Docker_Essentials/downloaders_reset.sh index b3af203..f3e4e51 100755 --- a/Docker_Essentials/downloaders_reset.sh +++ b/Docker_Essentials/downloaders_reset.sh @@ -602,11 +602,13 @@ if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then # Age check — must be old enough [[ "$AGE_DAYS" -lt "$QBIT_FAILSAFE_MIN_DAYS" ]] && ((SKIPPED++)) && continue - # Ratio check — if configured + # Ratio check — if configured. awk handles the fractional comparison; + # bash's [[ -lt ]] only does integers and would treat e.g. 1.4 and 1.5 + # as equal once truncated. Missing/empty ratio defaults to 0 (protected). if [[ "$QBIT_FAILSAFE_MIN_RATIO" != "0" ]]; then - RATIO_INT="${RATIO%.*}" - MIN_RATIO_INT="${QBIT_FAILSAFE_MIN_RATIO%.*}" - [[ "$RATIO_INT" -lt "$MIN_RATIO_INT" ]] && ((SKIPPED++)) && continue + if (( $(awk "BEGIN {print (${RATIO:-0} < $QBIT_FAILSAFE_MIN_RATIO) ? 1 : 0}") )); then + ((SKIPPED++)) && continue + fi fi if [[ "$DRY_RUN" == true ]]; then diff --git a/Fallback/fallback.sh b/Fallback/fallback.sh index 97bfad0..d4ac7e2 100755 --- a/Fallback/fallback.sh +++ b/Fallback/fallback.sh @@ -1000,12 +1000,19 @@ while [[ "$FALLBACK_RUNNING" == true ]]; do fi elif [[ "$INTERNET_UP" == false ]]; then - # Lost internet during fallback — enter DARK + # Lost internet during fallback — enter DARK — same actions as NO_INTERNET echo "" echo "━━━ $ICON_NET Entering DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━" warn "Lost internet during fallback — entering DARK state" state_set state "DARK" local_ddns_stop + + # Stop containers configured to stop on internet loss + read -r -a stop_on_no_net <<< "$(get_stop_on_no_net)" + for container in "${stop_on_no_net[@]}"; do + [[ -n "$container" ]] && local_stop "$container" + done + notify "DARK state on $(hostname) — lost internet during fallback" \ "Fallback" "warning" diff --git a/Kernel/decision_engine.sh b/Kernel/decision_engine.sh index 396bb20..eebd511 100755 --- a/Kernel/decision_engine.sh +++ b/Kernel/decision_engine.sh @@ -65,6 +65,7 @@ score_candidate() { local popularity_score="${2:-0}" local recency_score="${3:-0}" local quality_score="${4:-0}" + local TOTAL_SCORE TOTAL_SCORE=$(( \ user_score + \ diff --git a/Monitors/cert_monitor.sh b/Monitors/cert_monitor.sh index 3ebcbe8..b3aa4c4 100755 --- a/Monitors/cert_monitor.sh +++ b/Monitors/cert_monitor.sh @@ -159,43 +159,24 @@ fi check_cert() { local domain="$1" local port="${2:-443}" - _CERT_DAYS="" - _CERT_EXPIRY="" - local expiry_str - expiry_str=$(echo | timeout "$CERT_TIMEOUT" openssl s_client \ - -connect "${domain}:${port}" \ - -servername "$domain" \ - 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) - - if [[ -z "$expiry_str" ]]; then - error "$ICON_CERT $domain — could not retrieve certificate (unreachable or no TLS)" + if ! check_cert_expiry "$domain" "$port" "$CERT_TIMEOUT"; then + if [[ -n "$_CERT_EXPIRY_RAW" ]]; then + error "$ICON_CERT $domain — could not parse expiry date: $_CERT_EXPIRY_RAW" + else + error "$ICON_CERT $domain — could not retrieve certificate (unreachable or no TLS)" + fi return 3 fi - local expiry_epoch - expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null) - - if [[ -z "$expiry_epoch" ]]; then - error "$ICON_CERT $domain — could not parse expiry date: $expiry_str" - return 3 - fi - - local now days_remaining expiry_display - now=$(date +%s) - days_remaining=$(( (expiry_epoch - now) / 86400 )) - expiry_display=$(date -d "$expiry_str" '+%Y-%m-%d' 2>/dev/null) - _CERT_DAYS=$days_remaining - _CERT_EXPIRY=$expiry_display - - if [[ "$days_remaining" -le "$CERT_CRIT_DAYS" ]]; then - error "$ICON_CERT $domain — CRITICAL: ${days_remaining} days remaining (expires $expiry_display)" + if [[ "$_CERT_DAYS" -le "$CERT_CRIT_DAYS" ]]; then + error "$ICON_CERT $domain — CRITICAL: ${_CERT_DAYS} days remaining (expires $_CERT_EXPIRY)" return 2 - elif [[ "$days_remaining" -le "$CERT_WARN_DAYS" ]]; then - warn "$ICON_CERT $domain — WARNING: ${days_remaining} days remaining (expires $expiry_display)" + elif [[ "$_CERT_DAYS" -le "$CERT_WARN_DAYS" ]]; then + warn "$ICON_CERT $domain — WARNING: ${_CERT_DAYS} days remaining (expires $_CERT_EXPIRY)" return 1 else - log "$ICON_CERT $domain — OK: ${days_remaining} days remaining (expires $expiry_display)" + log "$ICON_CERT $domain — OK: ${_CERT_DAYS} days remaining (expires $_CERT_EXPIRY)" return 0 fi } @@ -285,7 +266,7 @@ fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # ── Write JSON status cache ─────────────────────────────────────────────────── -_CERT_CACHE_FILE="$SCRIPTS_DIR/State_Files/cert_status.json" +_CERT_CACHE_FILE="$STATE_DIR/cert_status.json" { printf '{"checked_at":%d,"host":"%s","warn_days":%d,"crit_days":%d,"dry_run":%s,"domains":[\n' \ "$(date +%s)" "$MY_ID" "$CERT_WARN_DAYS" "$CERT_CRIT_DAYS" \ diff --git a/Monitors/emby_session_report.sh b/Monitors/emby_session_report.sh index 9f5f97f..3670e21 100755 --- a/Monitors/emby_session_report.sh +++ b/Monitors/emby_session_report.sh @@ -316,7 +316,7 @@ echo "" echo "━━━ $ICON_RAM Transcode Status ━━━" if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ') - RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}") + RAMDISK_USED_GB=$(kb_to_gb "$RAMDISK_USED_KB") SYMLINK=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown") echo " $ICON_RAM Ramdisk usage: ${RAMDISK_USED_GB}GB / ${RAMDISK_SIZE:-8G}" echo " $ICON_LINK Symlink target: $SYMLINK" diff --git a/Monitors/smart_health.sh b/Monitors/smart_health.sh index d79afaa..c700597 100755 --- a/Monitors/smart_health.sh +++ b/Monitors/smart_health.sh @@ -156,11 +156,7 @@ if [[ "$SHOW_STATUS" == true ]]; then for drive in /dev/sd? /dev/nvme?; do [[ ! -e "$drive" ]] && continue drive_name=$(basename "$drive") - ignored=false - for ignore in "${SMART_IGNORE_DRIVES[@]}"; do - [[ "$drive_name" == "$ignore" ]] && ignored=true && break - done - if [[ "$ignored" == true ]]; then + if is_in_list "$drive_name" "${SMART_IGNORE_DRIVES[@]}"; then echo " $ICON_WARN $drive — ignored" else echo " $ICON_SMART $drive — would check" @@ -201,18 +197,7 @@ get_drive_temp() { echo "${temp:-}" } -# Detect if a drive is SSD/NVMe (rotational=0) -is_ssd() { - local drive="$1" - local dev_name - dev_name=$(basename "$drive" | sed 's/nvme[0-9]/nvme0/') - local rotational="/sys/block/$(basename "$drive")/queue/rotational" - [[ -f "$rotational" ]] && [[ "$(cat "$rotational" 2>/dev/null)" == "0" ]] && return 0 - # NVMe is always SSD - [[ "$drive" == *nvme* ]] && return 0 - return 1 -} - +# is_ssd() — provided by common.sh # ============================================================================================== # ━━━ SMART Health Check ━━━ # ============================================================================================== @@ -232,12 +217,7 @@ for drive in /dev/sd? /dev/nvme?; do drive_name=$(basename "$drive") # Check ignore list - ignored=false - for ignore in "${SMART_IGNORE_DRIVES[@]}"; do - [[ "$drive_name" == "$ignore" ]] && ignored=true && break - done - - if [[ "$ignored" == true ]]; then + if is_in_list "$drive_name" "${SMART_IGNORE_DRIVES[@]}"; then log "$drive_name — ignored (SMART_IGNORE_DRIVES)" DRIVES_SKIP+=("$drive_name") continue diff --git a/Monitors/weekly_health_digest.sh b/Monitors/weekly_health_digest.sh index d5e3a24..0be2a5a 100755 --- a/Monitors/weekly_health_digest.sh +++ b/Monitors/weekly_health_digest.sh @@ -270,7 +270,7 @@ fi # ── Transcode Ramdisk ───────────────────────────────────────────────────────────────────────── if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ') - RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}") + RAMDISK_USED_GB=$(kb_to_gb "$RAMDISK_USED_KB") SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown") DIGEST_LINES+=("$ICON_RAM Transcodes: ${RAMDISK_USED_GB}GB used → $SYMLINK_TARGET") @@ -309,7 +309,7 @@ if [[ -f "${BANDWIDTH_LOG:-}" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then YESTERDAY=$(date -d "yesterday" '+%Y-%m-%d') YESTERDAY_BYTES=$(awk -F'|' -v d="$YESTERDAY" '$1==d{sum+=$6} END{print sum+0}' \ "$BANDWIDTH_LOG") - YESTERDAY_GB=$(awk "BEGIN {printf \"%.2f\", ${YESTERDAY_BYTES:-0} / 1073741824}") + YESTERDAY_GB=$(bytes_to_gb "$YESTERDAY_BYTES") YESTERDAY_LARGE=$(awk -F'|' -v d="$YESTERDAY" '$1==d && $7=="LARGE"' \ "$BANDWIDTH_LOG" | wc -l) @@ -329,12 +329,8 @@ if [[ ${#CERT_MONITOR_DOMAINS[@]} -gt 0 ]] && command -v openssl >/dev/null 2>&1 CERT_ISSUES=() for domain in "${CERT_MONITOR_DOMAINS[@]}"; do [[ -z "$domain" ]] && continue - expiry_str=$(echo | timeout "${CERT_TIMEOUT:-10}" openssl s_client \ - -connect "${domain}:443" -servername "$domain" \ - 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) - if [[ -n "$expiry_str" ]]; then - expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null) - days_remaining=$(( (expiry_epoch - $(date +%s)) / 86400 )) + if check_cert_expiry "$domain" 443 "${CERT_TIMEOUT:-10}"; then + days_remaining="$_CERT_DAYS" if [[ "$days_remaining" -le "${CERT_CRIT_DAYS:-7}" ]]; then CERT_ISSUES+=("$domain: ${days_remaining}d CRITICAL") SHOULD_SEND=true diff --git a/Monitors/zfs_memory_snapshot.sh b/Monitors/zfs_memory_snapshot.sh index e5b5054..1fd2c8d 100755 --- a/Monitors/zfs_memory_snapshot.sh +++ b/Monitors/zfs_memory_snapshot.sh @@ -261,9 +261,9 @@ else ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) ARC_META_USED=$(awk '/^arc_meta_used / {print $3}' /proc/spl/kstat/zfs/arcstats) - ARC_MAX_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_MAX / 1073741824}") - ARC_CUR_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE / 1073741824}") - ARC_META_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_META_USED / 1073741824}") + ARC_MAX_GB=$(bytes_to_gb "$ARC_MAX" 1) + ARC_CUR_GB=$(bytes_to_gb "$ARC_SIZE" 1) + ARC_META_GB=$(bytes_to_gb "$ARC_META_USED" 1) ARC_PCT=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE * 100 / $ARC_MAX}") ARC_PCT_INT=$(printf "%.0f" "$ARC_PCT") @@ -287,8 +287,8 @@ else META_MISSES=$(awk '/^demand_metadata_misses / {print $3}' \ /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0) - MRU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MRU_GHOST / 1073741824}") - MFU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MFU_GHOST / 1073741824}") + MRU_GB=$(bytes_to_gb "$META_MRU_GHOST") + MFU_GB=$(bytes_to_gb "$META_MFU_GHOST") echo " $ICON_ZFS MRU Ghost: ${MRU_GB}GB" echo " $ICON_ZFS MFU Ghost: ${MFU_GB}GB" diff --git a/Orchestrators/transcode_management.sh b/Orchestrators/transcode_management.sh index 78a6542..6a77911 100755 --- a/Orchestrators/transcode_management.sh +++ b/Orchestrators/transcode_management.sh @@ -88,8 +88,9 @@ # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -source "$SCRIPT_DIR/../load_config.sh" +source "$ECOSYSTEM_ROOT/load_config.sh" parse_args "$@" @@ -158,17 +159,6 @@ if [[ "$SHOW_STATUS" == true ]]; then exit 0 fi -# ============================================================================================== -# ━━━ Validate Child Scripts ━━━ -# ============================================================================================== -for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do - _script_path="$SCRIPT_DIR/../$_entry" - if [[ ! -f "$_script_path" ]]; then - error "$(basename "$_entry") not found: $_script_path" - exit 1 - fi -done - # ============================================================================================== # ━━━ Pre-run State Snapshot ━━━ # ============================================================================================== @@ -188,36 +178,22 @@ fi # ============================================================================================== # transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run — no flag here # suppresses that; it owns the log write for this cycle regardless of position ✅ -DRY_FLAG="" -[[ "$DRY_RUN" == true ]] && DRY_FLAG="--dry-run" - -WORST_EXIT=0 -PASS_COUNT=0 -FAIL_NAMES=() +JOB_PASS=() +JOB_FAIL=() for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do - _script_path="$SCRIPT_DIR/../$_entry" - _script_name=$(basename "$_entry" .sh) - _start=$(date +%s) - bash "$_script_path" $DRY_FLAG - _exit=$? - log "$_script_name: $(format_duration $(( $(date +%s) - _start ))) (exit $_exit)" - if [[ "$_exit" -ne 0 ]]; then - WORST_EXIT=1 - FAIL_NAMES+=("$_script_name") - else - PASS_COUNT=$(( PASS_COUNT + 1 )) - fi + [[ -z "$_entry" ]] && continue + run_orch_child "$_entry" done # ============================================================================================== # ━━━ Summary — minimal one-liner by default (7-min cadence — keep it quiet when healthy) ━━━ # ============================================================================================== -if [[ "$WORST_EXIT" -eq 0 ]]; then - echo "$ICON_SUCCESS Transcode cycle — $PASS_COUNT/${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} passed" +if [[ "${#JOB_FAIL[@]}" -eq 0 ]]; then + echo "$ICON_SUCCESS Transcode cycle — ${#JOB_PASS[@]}/${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} passed" else - error "Transcode cycle — failed: ${FAIL_NAMES[*]}" + error "Transcode cycle — failed: ${JOB_FAIL[*]}" if [[ "$DRY_RUN" != true ]]; then - notify "Transcode management failure on $(hostname) ($MY_ID) — ${FAIL_NAMES[*]}" \ + notify "Transcode management failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \ "Transcode Management" "warning" fi fi @@ -225,5 +201,5 @@ fi # ============================================================================================== # ━━━ Exit ━━━ # ============================================================================================== -# Return worst exit code — caller knows if any script failed -exit "$WORST_EXIT" \ No newline at end of file +[[ "${#JOB_FAIL[@]}" -gt 0 ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/Orchestrators/watchdog_orchestrator.sh b/Orchestrators/watchdog_orchestrator.sh index de3e431..dff0618 100755 --- a/Orchestrators/watchdog_orchestrator.sh +++ b/Orchestrators/watchdog_orchestrator.sh @@ -177,40 +177,12 @@ log "Startup grace: past — uptime $(format_duration $UPTIME_SECONDS)" # ━━━ Run Watchdog Cycle ━━━ # ============================================================================================== CYCLE_START=$(date +%s) -PASS=() -FAIL=() - -run_watchdog() { - local name="$1" script="$2" - - if [[ ! -f "$script" ]]; then - error "$name — not found: $script" - FAIL+=("$name:missing") - return 1 - fi - - [[ ! -x "$script" ]] && chmod +x "$script" - - local extra_args=() - [[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run") - [[ "$ENABLE_LOGGING" == true ]] && extra_args+=("--log") - - local _ws - _ws=$(date +%s) - log "$ICON_START $name" - if bash "$script" "${extra_args[@]}"; then - log "$ICON_DONE $name — done in $(format_duration $(( $(date +%s) - _ws )))" - PASS+=("$name") - return 0 - else - error "$name — non-zero exit ($(format_duration $(( $(date +%s) - _ws ))))" - FAIL+=("$name") - return 1 - fi -} +JOB_PASS=() +JOB_FAIL=() for _entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do - run_watchdog "$(_watchdog_display_name "$_entry")" "$ECOSYSTEM_ROOT/$_entry" + [[ -z "$_entry" ]] && continue + run_orch_child "$_entry" done CYCLE_END=$(date +%s) @@ -236,19 +208,19 @@ fi # ============================================================================================== # ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━ # ============================================================================================== -if [[ "${#FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then +if [[ "${#JOB_FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━" - for p in "${PASS[@]}"; do log " $ICON_DONE $p"; done - for f in "${FAIL[@]}"; do error " $ICON_ERROR $f"; done + for p in "${JOB_PASS[@]}"; do log " $ICON_DONE $p"; done + for f in "${JOB_FAIL[@]}"; do error " $ICON_ERROR $f"; done echo "$ICON_TIME Duration: $(format_duration $DURATION)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" else - echo "$ICON_DONE Watchdog cycle — ${#PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))" + echo "$ICON_DONE Watchdog cycle — ${#JOB_PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))" fi -if [[ "${#FAIL[@]}" -gt 0 ]]; then - notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \ +if [[ "${#JOB_FAIL[@]}" -gt 0 ]]; then + notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \ "Watchdog Orchestrator" "warning" exit 1 fi diff --git a/Partnership/partnership_manager.sh b/Partnership/partnership_manager.sh index 29113db..fa6f159 100755 --- a/Partnership/partnership_manager.sh +++ b/Partnership/partnership_manager.sh @@ -249,22 +249,7 @@ fi detect_hosts # ── Derive owner and mirror from PARTNERSHIP_OWNER_HOST ─────────────────────────────────────── -OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}" # e.g. "HOST1" -MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" ) - -OWNER="${!OWNER_ID}" # hostname string -MIRROR="${!MIRROR_ID}" -# SSH_KEY (set by detect_hosts) is this server's own private key. -# The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh. -# With sparse checkout, each server only has its own host{N}.conf — the other server's -# key path is never available here. Use SSH_KEY for all outbound SSH regardless of mode. -MIRROR_SSH_KEY="$SSH_KEY" -OWNER_SSH_KEY="$SSH_KEY" - -AM_OWNER=false -AM_MIRROR=false -[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true -[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true +partnership_resolve_roles # State files LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db" @@ -355,7 +340,7 @@ push_state_to_remote() { warn "DRY RUN — would push state file to remote" return 0 fi - timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \ + timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes \ "$local_file" "root@${remote_ip}:${local_file}" 2>/dev/null && \ echo "State file pushed to remote ✅" || \ warn "Could not push state file to remote — will propagate on next sync" @@ -364,7 +349,7 @@ push_state_to_remote() { read_remote_state() { local remote_ip="$1" ssh_key="$2" remote_file="$3" timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ - -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \ "cat '$remote_file' 2>/dev/null" 2>/dev/null } @@ -524,34 +509,9 @@ gather_partner_fallback_containers() { done } -# Read a scalar var from the mirror's own config via SSH. -# Sources load_config.sh + detect_hosts() on the remote so HOST* aliasing works. -read_remote_conf_var() { - local mirror_ip="$1" var_name="$2" - timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ - -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ - "source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null - detect_hosts 2>/dev/null - printf '%s' \"\${${var_name}:-}\"" 2>/dev/null -} +# read_remote_conf_var() / read_remote_conf_array() — provided by common.sh -# Read an array var from the mirror's own config via SSH — one element per line. -read_remote_conf_array() { - local mirror_ip="$1" var_name="$2" - timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ - -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ - "source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null - detect_hosts 2>/dev/null - printf '%s\n' \"\${${var_name}[@]:-}\"" 2>/dev/null -} - -# Returns just the short name portion: "unRAID-Gmer4Lfe" → "Gmer4Lfe" -derive_short_name() { - local hostname="$1" - local short="${hostname,,}" - [[ "$short" == unraid-* ]] && short="${short:7}" - echo "${short^}" -} +# derive_short_name() — provided by common.sh # Start this server's own parked containers after partnership ends. start_own_stack() { @@ -644,7 +604,7 @@ cleanup_owner_containers_on_mirror() { local container_list container_list=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ - -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ "docker ps -a --format '{{.Names}}' 2>/dev/null | grep -i -- '-${owner_short}$'" 2>/dev/null) if [[ -z "$container_list" ]]; then @@ -658,12 +618,12 @@ cleanup_owner_containers_on_mirror() { # Collect appdata paths before removal local appdata_paths appdata_paths=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ - -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ "docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$container' 2>/dev/null \ | awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null) timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ - -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ "docker stop '$container' >/dev/null 2>&1 docker rm '$container' >/dev/null 2>&1 && echo removed" 2>/dev/null | \ grep -q removed && \ @@ -674,7 +634,7 @@ cleanup_owner_containers_on_mirror() { while IFS= read -r path; do [[ -z "$path" ]] && continue timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ - -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ "rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \ echo " Appdata removed on $MIRROR: $path ✅" || \ warn " Failed to remove appdata on $MIRROR: $path" @@ -707,7 +667,7 @@ start_mirror_own_stack() { continue fi timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ - -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ "docker start '$container' >/dev/null 2>&1 && echo started" 2>/dev/null | \ grep -q started && \ echo "$container started on $MIRROR ✅" || \ @@ -881,7 +841,7 @@ check_both_healthy() { [[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; } timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \ - -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ "mountpoint -q '$REMOTE_STORAGE_PATH' && timeout 10 docker ps" >/dev/null 2>&1 || { error "Mirror $MIRROR not healthy" return 1 @@ -913,7 +873,8 @@ do_final_sync() { warn "Final sync complete — mirror has current state ✅" } -# Safe master.conf modification with error handling +# Safe master.conf modification with error handling — appends the key if not already present, +# since sed -i returns 0 whether or not it matched anything. update_master_conf() { local key="$1" value="$2" local conf="$CONF_DIR/master.conf" @@ -921,12 +882,22 @@ update_master_conf() { error "master.conf not found at $conf" return 1 fi - if sed -i "s|^[[:space:]]*${key}=.*| ${key}=${value}|" "$conf" 2>/dev/null; then - echo "master.conf updated: ${key}=${value}" - return 0 + if grep -q "^[[:space:]]*${key}=" "$conf" 2>/dev/null; then + if sed -i "s|^[[:space:]]*${key}=.*| ${key}=${value}|" "$conf" 2>/dev/null; then + echo "master.conf updated: ${key}=${value}" + return 0 + else + error "Failed to update master.conf: ${key}=${value}" + return 1 + fi else - error "Failed to update master.conf: ${key}=${value}" - return 1 + if echo " ${key}=${value}" >> "$conf" 2>/dev/null; then + echo "master.conf updated: ${key}=${value} (appended)" + return 0 + else + error "Failed to append to master.conf: ${key}=${value}" + return 1 + fi fi } @@ -1162,13 +1133,7 @@ if [[ "$MODE" == "onboard" ]]; then echo "" echo "Enabling partnership in master.conf..." if [[ "$DRY_RUN" == false ]]; then - _master_conf="$CONF_DIR/master.conf" - if grep -q "^[[:space:]]*PARTNERSHIP_ENABLED=" "$_master_conf" 2>/dev/null; then - sed -i "s|^[[:space:]]*PARTNERSHIP_ENABLED=.*|PARTNERSHIP_ENABLED=true|" "$_master_conf" - else - echo "PARTNERSHIP_ENABLED=true" >> "$_master_conf" - fi - echo "PARTNERSHIP_ENABLED=true in master.conf ✅" + update_master_conf "PARTNERSHIP_ENABLED" "true" platform_push_conf | while IFS= read -r line; do log "$line"; done else warn "DRY RUN — would set PARTNERSHIP_ENABLED=true in master.conf and push" @@ -1435,9 +1400,9 @@ if false; then # Update remote master.conf timeout "$SSH_TIMEOUT" ssh -i "$NEW_MIRROR_SSH_KEY" \ - -o ConnectTimeout="$SSH_TIMEOUT" root@"$NEW_MIRROR_IP" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$NEW_MIRROR_IP" \ "sed -i 's|^[[:space:]]*PARTNERSHIP_OWNER_HOST=.*| PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\"|' \ - '$SCRIPT_DIR/../master.conf'" 2>/dev/null && \ + '$SCRIPT_DIR/../Configurations/master.conf'" 2>/dev/null && \ echo "Remote master.conf updated ✅" || \ error "Failed to update remote master.conf — update manually" else diff --git a/Partnership/partnership_offboard.sh b/Partnership/partnership_offboard.sh index 4640583..0334383 100755 --- a/Partnership/partnership_offboard.sh +++ b/Partnership/partnership_offboard.sh @@ -108,17 +108,7 @@ fi detect_hosts -OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}" -MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" ) -OWNER="${!OWNER_ID}" -MIRROR="${!MIRROR_ID}" -MIRROR_SSH_KEY="$SSH_KEY" -OWNER_SSH_KEY="$SSH_KEY" - -AM_OWNER=false -AM_MIRROR=false -[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true -[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true +partnership_resolve_roles LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db" REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db" diff --git a/Partnership/partnership_onboard.sh b/Partnership/partnership_onboard.sh index 1b421bb..35aca04 100755 --- a/Partnership/partnership_onboard.sh +++ b/Partnership/partnership_onboard.sh @@ -223,20 +223,7 @@ fi detect_hosts -OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}" -MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" ) -OWNER="${!OWNER_ID}" -MIRROR="${!MIRROR_ID}" -# SSH_KEY (set by detect_hosts) is this server's own private key. -# The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh. -# HOST{N}_SSH_KEY lives in host{N}.conf — with sparse checkout, the other server's -# conf is never present here. Always use SSH_KEY (local private key) for outbound SSH. -MIRROR_SSH_KEY="$SSH_KEY" - -AM_OWNER=false -AM_MIRROR=false -[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true -[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true +partnership_resolve_roles EXTRA_FLAGS=() [[ "$DRY_RUN" == true ]] && EXTRA_FLAGS+=("--dry-run") @@ -250,11 +237,7 @@ write_onboard_phase() { local key="${target_id}_PHASE${phase}_DONE" local state_file="$(platform_setup_db_path)" [[ "$DRY_RUN" == true ]] && { warn "DRY RUN — would write ${key}=true"; return 0; } - if grep -q "^${key}=" "$state_file" 2>/dev/null; then - sed -i "s|^${key}=.*|${key}=true|" "$state_file" - else - echo "${key}=true" >> "$state_file" - fi + set_state_var "$state_file" "$key" "true" platform_push_setup_state } @@ -282,13 +265,7 @@ stop_mirror_stack() { local config_var="$1" label="$2" local -a to_stop=() - mapfile -t to_stop < <( - timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \ - -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \ - "source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null - detect_hosts 2>/dev/null - printf '%s\n' \"\${${config_var}[@]:-}\"" 2>/dev/null | grep -v '^$' - ) + mapfile -t to_stop < <(read_remote_conf_array "$MIRROR_IP" "$config_var" | grep -v '^$') if [[ ${#to_stop[@]} -eq 0 ]]; then log "No $label containers to stop on $MIRROR — skipping" @@ -302,7 +279,7 @@ stop_mirror_stack() { continue fi timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \ - -o ConnectTimeout="$SSH_TIMEOUT" root@"$MIRROR_IP" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \ "docker stop '$container' 2>/dev/null docker rm '$container' 2>/dev/null && echo removed" 2>/dev/null | \ grep -q removed && \ @@ -338,11 +315,7 @@ if [[ "$AM_MIRROR" == true ]]; then if [[ -n "$OWNER_IP" ]]; then # Read OWNER's SCRIPTS_DIR via platform probe command — don't assume same path as mirror - _probe_cmd=$(platform_scripts_dir_probe_cmd) - OWNER_SCRIPTS_DIR=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ - -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \ - "$_probe_cmd" 2>/dev/null | tr -d '[:space:]') - OWNER_SCRIPTS_DIR="${OWNER_SCRIPTS_DIR:-$SCRIPTS_DIR}" + OWNER_SCRIPTS_DIR=$(resolve_remote_scripts_dir "$OWNER_IP") if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2" @@ -444,11 +417,9 @@ elif [[ "$PHASE1_ONLY" == true ]]; then echo " Then click 'Push Conf' in the Partnership tab." # Write key-ready flag so UI can show the manual-install state [[ "$DRY_RUN" == false ]] && { - local kflag="${MIRROR_ID}_KEY_READY" - local _setup_f="$(platform_setup_db_path)" - grep -q "^${kflag}=" "$_setup_f" 2>/dev/null \ - && sed -i "s|^${kflag}=.*|${kflag}=true|" "$_setup_f" \ - || echo "${kflag}=true" >> "$_setup_f" + kflag="${MIRROR_ID}_KEY_READY" + _setup_f="$(platform_setup_db_path)" + set_state_var "$_setup_f" "$kflag" "true" } fi STEP_SSH_OK=false diff --git a/Partnership/partnership_transfer.sh b/Partnership/partnership_transfer.sh index f915179..350ab83 100755 --- a/Partnership/partnership_transfer.sh +++ b/Partnership/partnership_transfer.sh @@ -128,17 +128,7 @@ fi detect_hosts -OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}" -MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" ) -OWNER="${!OWNER_ID}" -MIRROR="${!MIRROR_ID}" -MIRROR_SSH_KEY="$SSH_KEY" -OWNER_SSH_KEY="$SSH_KEY" - -AM_OWNER=false -AM_MIRROR=false -[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true -[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true +partnership_resolve_roles LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db" REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db" @@ -286,14 +276,11 @@ if [[ "$DRY_RUN" == false ]]; then # Push updated master.conf to new owner so both servers agree immediately. # master.conf is shared — host-specific credentials live in host*.conf. - _probe_cmd=$(platform_scripts_dir_probe_cmd) - _REMOTE_SD=$(ssh -i "$SSH_KEY" -o ConnectTimeout=5 -o StrictHostKeyChecking=no \ - "root@${MIRROR_IP}" "$_probe_cmd" \ - 2>/dev/null | tr -d '[:space:]') - _REMOTE_SD="${_REMOTE_SD:-$SCRIPTS_DIR}" + _REMOTE_SD=$(resolve_remote_scripts_dir "$MIRROR_IP" "$SSH_KEY" "no") scp -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" \ -o StrictHostKeyChecking=no \ + -o BatchMode=yes \ "$SCRIPTS_ROOT/Configurations/master.conf" \ "root@${MIRROR_IP}:${_REMOTE_SD}/Configurations/master.conf" 2>/dev/null && \ echo "master.conf pushed to $NEW_OWNER ✅" || \ diff --git a/Partnership/share_setup.sh b/Partnership/share_setup.sh index 99f7f22..1170b43 100755 --- a/Partnership/share_setup.sh +++ b/Partnership/share_setup.sh @@ -52,7 +52,7 @@ MIRROR_IP=$(resolve_tailscale_ip "$MIRROR") [[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP — is Tailscale running?"; exit 1; } _ssh() { ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" "$@"; } -_scp() { scp -i "$SSH_KEY" -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$@"; } +_scp() { scp -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes -o StrictHostKeyChecking=no "$@"; } # ── Detect remote appdata pool ──────────────────────────────────────────────────────────────── diff --git a/Partnership/ssh_setup.sh b/Partnership/ssh_setup.sh index 65117aa..bcdd32d 100755 --- a/Partnership/ssh_setup.sh +++ b/Partnership/ssh_setup.sh @@ -74,9 +74,10 @@ acquire_lock detect_hosts # ── Derive key name from hostname ───────────────────────────────────────────────────────────── -# Strip unraid- prefix (case-insensitive) if present → gmer4lfe_rsync_automation -SHORT_NAME="${LOCAL_SERVER_NAME,,}" -[[ "${SHORT_NAME}" == unraid-* ]] && SHORT_NAME="${SHORT_NAME:7}" +# derive_short_name() title-cases the result (for display elsewhere) — lowercase it here, +# same as before, since this feeds a filename → gmer4lfe_rsync_automation +SHORT_NAME="$(derive_short_name "$LOCAL_SERVER_NAME")" +SHORT_NAME="${SHORT_NAME,,}" SSH_KEY_NAME="${SHORT_NAME}_rsync_automation" SSH_KEY_PATH="/root/.ssh/${SSH_KEY_NAME}" SSH_PUB_PATH="${SSH_KEY_PATH}.pub" diff --git a/Plugin/unraid/Partnership/containers.sh b/Plugin/unraid/Partnership/containers.sh index 4ac81d5..6dd11fc 100755 --- a/Plugin/unraid/Partnership/containers.sh +++ b/Plugin/unraid/Partnership/containers.sh @@ -220,7 +220,7 @@ deploy_container_from_xml() { log "Deploying $name..." if [[ "$DRY_RUN" == false ]]; then - timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \ + timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes \ "$xml_file" "root@${remote_ip}:${TEMPLATES_DIR}/${xml_name}" 2>/dev/null || { rm -f "$_gpu_tmp" warn " SCP failed for $xml_name — skipping $name" @@ -291,9 +291,9 @@ deploy_container_from_xml() { local remote_script="/tmp/deploy_${name//[^a-zA-Z0-9_]/_}.sh" - if timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \ + if timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes \ "$tmp_script" "root@${remote_ip}:${remote_script}" 2>/dev/null && \ - timeout 120 ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \ + timeout 120 ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \ "bash '$remote_script' 2>&1; rc=\$?; rm -f '$remote_script'; exit \$rc" 2>/dev/null | \ grep -q "deployed:${name}"; then echo " $name deployed ✅" @@ -386,7 +386,7 @@ cleanup_deployed_stack_on_remote() { | awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null) timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ - -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \ "docker stop '$cname' >/dev/null 2>&1 docker rm '$cname' >/dev/null 2>&1 && echo removed" 2>/dev/null | \ grep -q removed && \ @@ -396,7 +396,7 @@ cleanup_deployed_stack_on_remote() { while IFS= read -r path; do [[ -z "$path" ]] && continue timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ - -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \ "rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \ echo " Appdata removed on $MIRROR: $path ✅" || \ warn " Failed to remove appdata on $MIRROR: $path" @@ -504,7 +504,7 @@ reconfigure_webui() { local template template=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ - -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \ "grep -rl '' '$TEMPLATES_DIR/' 2>/dev/null | \ xargs grep -l '\"$container\"' 2>/dev/null | head -1" 2>/dev/null) @@ -514,7 +514,7 @@ reconfigure_webui() { fi timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ - -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \ "sed -i 's|.*|http://${target_ip}:${port}/|g' '$template'" \ 2>/dev/null && \ echo "$container → http://${target_ip}:${port}/ ✅" || { diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index a6505df..fe223bf 100755 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -56,7 +56,7 @@ # # Remote Health Pre-flights # check_connectivity() — Tailscale IP reachable before any SSH -# check_remote_rootfs() — aborts if remote rootfs exceeds ROOTFS_WARN_PCT +# check_remote_rootfs() — aborts if remote rootfs exceeds ROOTFS_WARN # check_remote_share() — aborts if target directory missing or empty on remote # check_remote_disks() — verifies all backing disks online on remote # @@ -98,7 +98,7 @@ # SLEEP # Default seconds between retry attempts. (default: 60) # -# ROOTFS_WARN_PCT +# ROOTFS_WARN # Abort threshold for remote rootfs percentage full. (default: 75) # # PROFILES["profile_KEY"] @@ -223,6 +223,11 @@ VV_LAST_LOG="/tmp/unraid_locks/rsync_${PROFILE_NAME}.last.log" exec 1> >(tee -a "$VV_LIVE_LOG") 2>&1 # ── Load profile settings ───────────────────────────────────────────────────────────────────── +# NOTE: DEFAULT_RSYNC_OPTS and PROFILE_RSYNC_OPTS are bash arrays defined in master.conf — +# any "$BW_LIMIT" inside them is expanded once, when master.conf is sourced, before this +# override runs. So this recomputed $BW_LIMIT only affects the log line below and --status; +# it does NOT change the actual rsync --bwlimit unless a profile also bakes its own +# --bwlimit="${PROFILE_BW_LIMIT[name]}" directly into that profile's PROFILE_RSYNC_OPTS entry. BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT} RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT} SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP} diff --git a/System_Essentials/conf_cache_restore.sh b/System_Essentials/conf_cache_restore.sh index 11e904c..ae4bbaa 100755 --- a/System_Essentials/conf_cache_restore.sh +++ b/System_Essentials/conf_cache_restore.sh @@ -53,7 +53,7 @@ parse_args "$@" detect_hosts require_partnership -RAM_CACHE="/tmp/.cache/vv/d" +RAM_CACHE="$CONF_RAM_CACHE_DIR" SAVE_DIR="$PERSISTENT_CONF_CACHE" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" @@ -67,7 +67,7 @@ restored=0 for conf in "$SAVE_DIR"/host*.conf; do [[ -f "$conf" ]] || continue base="$(basename "$conf")" - [[ "${base,,}" == "${MY_ID,,}.conf" ]] && continue + is_own_conf_file "$base" && continue if [[ -f "$RAM_CACHE/$base" ]]; then log "$base already in RAM cache (conf_sync succeeded) — skipping" diff --git a/System_Essentials/conf_cache_save.sh b/System_Essentials/conf_cache_save.sh index bc18c4f..4189d7e 100755 --- a/System_Essentials/conf_cache_save.sh +++ b/System_Essentials/conf_cache_save.sh @@ -47,7 +47,7 @@ parse_args "$@" detect_hosts require_partnership -RAM_CACHE="/tmp/.cache/vv/d" +RAM_CACHE="$CONF_RAM_CACHE_DIR" SAVE_DIR="$PERSISTENT_CONF_CACHE" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be written" @@ -61,7 +61,7 @@ saved=0 for conf in "$RAM_CACHE"/host*.conf; do [[ -f "$conf" ]] || continue base="$(basename "$conf")" - [[ "${base,,}" == "${MY_ID,,}.conf" ]] && continue + is_own_conf_file "$base" && continue if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would save $base → $SAVE_DIR/" diff --git a/System_Essentials/conf_sync.sh b/System_Essentials/conf_sync.sh index c192360..57145be 100755 --- a/System_Essentials/conf_sync.sh +++ b/System_Essentials/conf_sync.sh @@ -81,7 +81,7 @@ if [[ "${CONF_SYNC_ENABLED:-true}" == false ]]; then exit 0 fi -CACHE_DIR="/tmp/.cache/vv/d" +CACHE_DIR="$CONF_RAM_CACHE_DIR" MY_CONF="$SCRIPTS_DIR/Configurations/${MY_ID,,}.conf" SSH_TIMEOUT=10 diff --git a/Tools/container_data_export.sh b/Tools/container_data_export.sh index b74e21f..95befa3 100755 --- a/Tools/container_data_export.sh +++ b/Tools/container_data_export.sh @@ -146,17 +146,7 @@ cleanup_on_exit() { warn "Removing partial archive: $ARCHIVE_PATH" rm -f "$ARCHIVE_PATH" 2>/dev/null fi - # Always restart container if it was running - if [[ "$CONTAINER_WAS_RUNNING" == true && "$DRY_RUN" == false ]]; then - local status - status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \ - "$CONTAINER_NAME" 2>/dev/null) - if [[ "$status" != "true" ]]; then - warn "Restarting $CONTAINER_NAME (cleanup)..." - timeout "$DOCKER_TIMEOUT" docker start "$CONTAINER_NAME" >/dev/null 2>&1 || \ - error "Failed to restart $CONTAINER_NAME — start it manually" - fi - fi + container_force_restart_if_needed "$CONTAINER_NAME" "$CONTAINER_WAS_RUNNING" } trap cleanup_on_exit EXIT @@ -166,35 +156,10 @@ trap cleanup_on_exit EXIT echo "" echo "━━━ $ICON_STOP Stop Container ━━━" -STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \ - "$CONTAINER_NAME" 2>/dev/null) - -case "$STATUS" in - true) - CONTAINER_WAS_RUNNING=true - log "Stopping $CONTAINER_NAME for clean export..." - if [[ "$DRY_RUN" == false ]]; then - if timeout "$DOCKER_TIMEOUT" docker stop "$CONTAINER_NAME" >/dev/null 2>&1; then - echo "$CONTAINER_NAME stopped ✅" - else - error "Failed to stop $CONTAINER_NAME — aborting export" - exit 1 - fi - else - warn "DRY RUN — would stop $CONTAINER_NAME" - fi - ;; - false) - echo "$CONTAINER_NAME is not running — archiving as-is (was stopped state respected)" - ;; - "") - error "$CONTAINER_NAME not found — check container name" - exit 1 - ;; - *) - warn "$CONTAINER_NAME status: $STATUS — proceeding with caution" - ;; -esac +container_stop_for_maintenance "$CONTAINER_NAME" CONTAINER_WAS_RUNNING \ + "Stopping $CONTAINER_NAME for clean export..." log || exit 1 +[[ "$CONTAINER_WAS_RUNNING" == false ]] && \ + echo "$CONTAINER_NAME is not running — archiving as-is (was stopped state respected)" # ============================================================================================== # ━━━ Archive ━━━ @@ -249,37 +214,7 @@ END=$(date +%s) echo "" echo "━━━ $ICON_START Restart Container ━━━" -RESTART_OK=false - -if [[ "$CONTAINER_WAS_RUNNING" == true ]]; then - log "Restarting $CONTAINER_NAME..." - if [[ "$DRY_RUN" == false ]]; then - if timeout "$DOCKER_TIMEOUT" docker start "$CONTAINER_NAME" >/dev/null 2>&1; then - # Brief settle then verify - sleep 3 - POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ - '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null) - if [[ "$POST_STATUS" == "true" ]]; then - echo "$CONTAINER_NAME restarted and running ✅" - RESTART_OK=true - else - error "$CONTAINER_NAME started but crashed immediately — check container logs" - notify "$CONTAINER_NAME failed to stay running after export on $(hostname)" \ - "Container Export" "warning" - fi - else - error "Failed to restart $CONTAINER_NAME — start it manually" - notify "$CONTAINER_NAME failed to restart after export on $(hostname)" \ - "Container Export" "warning" - fi - else - warn "DRY RUN — would restart $CONTAINER_NAME" - RESTART_OK=true - fi -else - echo "$CONTAINER_NAME was not running — leaving stopped (state respected) ✅" - RESTART_OK=true -fi +container_restart_after_maintenance "$CONTAINER_NAME" "$CONTAINER_WAS_RUNNING" 3 "Container Export" # Clear trap — clean exit, cleanup_on_exit no longer needed trap - EXIT diff --git a/Tools/emby_database_repair.sh b/Tools/emby_database_repair.sh index 0e6f210..87088f4 100755 --- a/Tools/emby_database_repair.sh +++ b/Tools/emby_database_repair.sh @@ -171,16 +171,7 @@ EMBY_WAS_RUNNING=false cleanup_on_exit() { local exit_code=$? - if [[ "$EMBY_WAS_RUNNING" == true && "$DRY_RUN" == false ]]; then - local status - status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \ - "$EMBY_CONTAINER" 2>/dev/null) - if [[ "$status" != "true" ]]; then - warn "Restarting $EMBY_CONTAINER (cleanup)..." - timeout "$DOCKER_TIMEOUT" docker start "$EMBY_CONTAINER" >/dev/null 2>&1 || \ - error "Failed to restart $EMBY_CONTAINER — start it manually" - fi - fi + container_force_restart_if_needed "$EMBY_CONTAINER" "$EMBY_WAS_RUNNING" } trap cleanup_on_exit EXIT @@ -191,36 +182,10 @@ trap cleanup_on_exit EXIT echo "" echo "━━━ $ICON_STOP Stop Emby ━━━" -STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \ - "$EMBY_CONTAINER" 2>/dev/null) - -case "$STATUS" in - true) - EMBY_WAS_RUNNING=true - warn "Stopping $EMBY_CONTAINER — active sessions will be interrupted" - if [[ "$DRY_RUN" == false ]]; then - if timeout "$DOCKER_TIMEOUT" docker stop "$EMBY_CONTAINER" >/dev/null 2>&1; then - echo "$EMBY_CONTAINER stopped ✅" - sleep 3 # let file handles release - else - error "Failed to stop $EMBY_CONTAINER — aborting" - exit 1 - fi - else - warn "DRY RUN — would stop $EMBY_CONTAINER" - fi - ;; - false) - log "$EMBY_CONTAINER is not running — proceeding with checks" - ;; - "") - error "$EMBY_CONTAINER not found — check container name" - exit 1 - ;; - *) - warn "$EMBY_CONTAINER status: $STATUS — proceeding with caution" - ;; -esac +container_stop_for_maintenance "$EMBY_CONTAINER" EMBY_WAS_RUNNING \ + "Stopping $EMBY_CONTAINER — active sessions will be interrupted" warn 3 || exit 1 +[[ "$EMBY_WAS_RUNNING" == false ]] && \ + log "$EMBY_CONTAINER is not running — proceeding with checks" # ============================================================================================== # ━━━ Database Integrity Check ━━━ @@ -301,37 +266,8 @@ END=$(date +%s) echo "" echo "━━━ $ICON_START Restart Emby ━━━" -RESTART_OK=false - -if [[ "$EMBY_WAS_RUNNING" == true ]]; then - if [[ "$DRY_RUN" == false ]]; then - log "Restarting $EMBY_CONTAINER..." - if timeout "$DOCKER_TIMEOUT" docker start "$EMBY_CONTAINER" >/dev/null 2>&1; then - sleep 5 # Emby takes longer to initialise than most containers - POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ - '{{.State.Running}}' "$EMBY_CONTAINER" 2>/dev/null) - if [[ "$POST_STATUS" == "true" ]]; then - echo "$EMBY_CONTAINER restarted and running ✅" - RESTART_OK=true - else - error "$EMBY_CONTAINER started but crashed — database may be corrupt" - error "Check Docker logs: docker logs $EMBY_CONTAINER" - notify "$EMBY_CONTAINER crashed on restart — possible database corruption on $(hostname)" \ - "Emby DB Repair" "warning" - fi - else - error "Failed to restart $EMBY_CONTAINER — start it manually" - notify "$EMBY_CONTAINER failed to restart after integrity check on $(hostname)" \ - "Emby DB Repair" "warning" - fi - else - warn "DRY RUN — would restart $EMBY_CONTAINER" - RESTART_OK=true - fi -else - echo "$EMBY_CONTAINER was not running — leaving stopped (state respected) ✅" - RESTART_OK=true -fi +container_restart_after_maintenance "$EMBY_CONTAINER" "$EMBY_WAS_RUNNING" 5 "Emby DB Repair" +[[ "$RESTART_OK" == false ]] && error "Check Docker logs: docker logs $EMBY_CONTAINER" # Clear EXIT trap — clean exit trap - EXIT diff --git a/Tools/smart_long_test.sh b/Tools/smart_long_test.sh index ea5dbcc..7123707 100755 --- a/Tools/smart_long_test.sh +++ b/Tools/smart_long_test.sh @@ -138,11 +138,8 @@ _build_drive_list() { [[ ! -e "$drive" ]] && continue local drive_name drive_name=$(basename "$drive") - local ignored=false - for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do - [[ "$drive_name" == "$ignore" ]] && ignored=true && break - done - [[ "$ignored" == true ]] && { log "Skipping $drive_name (SMART_IGNORE_DRIVES)"; continue; } + is_in_list "$drive_name" "${SMART_IGNORE_DRIVES[@]:-}" && \ + { log "Skipping $drive_name (SMART_IGNORE_DRIVES)"; continue; } if ! smartctl -i "$drive" 2>/dev/null | grep -q "SMART support is: Enabled"; then log "Skipping $drive_name — SMART not enabled" continue @@ -166,11 +163,7 @@ if [[ "$SHOW_STATUS" == true ]]; then drive_name=$(basename "$drive") local_result=$(smartctl -l selftest "$drive" 2>/dev/null | \ grep -m1 "Extended" | awk '{print $NF}') - ignored=false - for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do - [[ "$drive_name" == "$ignore" ]] && ignored=true && break - done - if [[ "$ignored" == true ]]; then + if is_in_list "$drive_name" "${SMART_IGNORE_DRIVES[@]:-}"; then echo " $ICON_WARN $drive_name — ignored" else echo " $ICON_SMART $drive_name — last extended: ${local_result:-no result}" diff --git a/Transcodes/transcode_cleanup.sh b/Transcodes/transcode_cleanup.sh index e46032b..5b451d6 100755 --- a/Transcodes/transcode_cleanup.sh +++ b/Transcodes/transcode_cleanup.sh @@ -240,15 +240,7 @@ cleanup_location() { # Format bytes freed local freed_human - if (( bytes_freed > 1073741824 )); then - freed_human=$(awk "BEGIN {printf \"%.1fGB\", $bytes_freed / 1073741824}") - elif (( bytes_freed > 1048576 )); then - freed_human=$(awk "BEGIN {printf \"%.1fMB\", $bytes_freed / 1048576}") - elif (( bytes_freed > 0 )); then - freed_human="${bytes_freed}B" - else - freed_human="0B" - fi + freed_human=$(format_bytes "$bytes_freed") log "$label — removed $files_removed ($freed_human) | active(fresh): $files_too_young | streaming(lsof): $files_streaming | protected(old+open): $files_active | skipped: $files_skipped | failed: $files_failed" @@ -299,7 +291,7 @@ fi # Post-cleanup — check if ramdisk recovered enough to flip symlink back to ramdisk if [[ "$DRY_RUN" == false ]] && mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ') - RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}") + RAMDISK_USED_GB=$(kb_to_gb "$RAMDISK_USED_KB") LOW_RECOVERED=$(awk "BEGIN {print ($RAMDISK_USED_GB < $RAMDISK_LOW_GB) ? 1 : 0}") CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d= -f2) diff --git a/Transcodes/transcode_manager.sh b/Transcodes/transcode_manager.sh index 2177ea6..9d7417a 100755 --- a/Transcodes/transcode_manager.sh +++ b/Transcodes/transcode_manager.sh @@ -234,18 +234,15 @@ fix_permissions() { } get_ramdisk_used_gb() { - df "$RAMDISK_PATH" --output=used 2>/dev/null | \ - tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}' + kb_to_gb "$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')" } get_ramdisk_avail_gb() { - df "$RAMDISK_PATH" --output=avail 2>/dev/null | \ - tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}' + kb_to_gb "$(df "$RAMDISK_PATH" --output=avail 2>/dev/null | tail -1 | tr -d ' ')" } get_ssd_free_gb() { - df "$TRANSCODE_SSD" --output=avail 2>/dev/null | \ - tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}' + kb_to_gb "$(df "$TRANSCODE_SSD" --output=avail 2>/dev/null | tail -1 | tr -d ' ')" } get_flip_count() { diff --git a/Watchdogs/System/network_watchdog.sh b/Watchdogs/System/network_watchdog.sh index 9d1d74e..a4c9d79 100755 --- a/Watchdogs/System/network_watchdog.sh +++ b/Watchdogs/System/network_watchdog.sh @@ -125,16 +125,9 @@ log "$ICON_NET DDNS: ${DDNS_DOMAIN:-not configured} → ${DDNS_CONTAINER:-no c touch "${NETWORK_WATCHDOG_NPM_STATE_FILE}" 2>/dev/null -# ━━━ Strike helpers ━━━ -get_strikes() { grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"; } -set_strikes() { - local key="$1" count="$2" file="$3" - if grep -q "^${key}:" "$file" 2>/dev/null; then - sed -i "s|^${key}:.*|${key}:${count}|" "$file" - else - echo "${key}:${count}" >> "$file" - fi -} +# ━━━ Strike helpers — wrap common.sh's wd_state_get()/wd_state_set() ━━━ +get_strikes() { wd_state_get "$1" "$2"; } +set_strikes() { wd_state_set "$1" "$2" "$3"; } # ============================================================================================== # ━━━ Status ━━━ diff --git a/Watchdogs/System/storage_watchdog.sh b/Watchdogs/System/storage_watchdog.sh index cf9cdd2..5435afc 100755 --- a/Watchdogs/System/storage_watchdog.sh +++ b/Watchdogs/System/storage_watchdog.sh @@ -164,16 +164,9 @@ log "$ICON_DISK Paths: ${WATCHDOG_APPDATA_PATHS[*]:-none configured}" touch "$STORAGE_WATCHDOG_STATE_FILE" 2>/dev/null touch "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null -# ━━━ Strike helpers ━━━ -get_strikes() { grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"; } -set_strikes() { - local key="$1" count="$2" file="$3" - if grep -q "^${key}:" "$file" 2>/dev/null; then - sed -i "s|^${key}:.*|${key}:${count}|" "$file" - else - echo "${key}:${count}" >> "$file" - fi -} +# ━━━ Strike helpers — wrap common.sh's wd_state_get()/wd_state_set() ━━━ +get_strikes() { wd_state_get "$1" "$2"; } +set_strikes() { wd_state_set "$1" "$2" "$3"; } # ============================================================================================== # ━━━ Status ━━━ diff --git a/Watchdogs/docker_watchdog.sh b/Watchdogs/docker_watchdog.sh index aa3949b..c70b0af 100755 --- a/Watchdogs/docker_watchdog.sh +++ b/Watchdogs/docker_watchdog.sh @@ -194,6 +194,9 @@ # WATCHDOG_CONTAINER_RESTART_LIMIT / WATCHDOG_CONTAINER_RESTART_WINDOW # Restart loop protection: attempt limit and rolling window in hours # +# WATCHDOG_REQUIRED_STRIKE_LIMIT +# Consecutive down-checks on a required container before a restart is attempted (default: 2) +# # WATCHDOG_BATCH_NOTIFY # Collect cycle events and send as one notification (default: true) # @@ -337,19 +340,15 @@ fi # ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── # ============================================================================================== -# Get strike count for a container from state file +# Get strike count for a container from state file — wraps common.sh's wd_state_get() get_strikes() { - grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0" + wd_state_get "$1" "$2" } -# Set strike count for a container in state file +# Set strike count for a container in state file — wraps common.sh's wd_state_set() set_strikes() { local container="$1" count="$2" file="$3" - if grep -q "^${container}:" "$file" 2>/dev/null; then - sed -i "s/^${container}:.*/${container}:${count}/" "$file" - else - echo "${container}:${count}" >> "$file" - fi + wd_state_set "$container" "$count" "$file" } # Check if container is on the persistent skip list @@ -763,10 +762,10 @@ CYCLE_START=$(date +%s) STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE") STRIKES=$(( STRIKES + 1 )) set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE" - warn "$container — not running, exit ${LAST_EXIT} (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)" + warn "$container — not running, exit ${LAST_EXIT} (strike $STRIKES/${WATCHDOG_REQUIRED_STRIKE_LIMIT:-2})" ((T1_WARNINGS++)) - if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then + if [[ "$STRIKES" -ge "${WATCHDOG_REQUIRED_STRIKE_LIMIT:-2}" ]]; then result=0 safe_restart "$container" "required container down (exit ${LAST_EXIT})" || result=$? case $result in diff --git a/Watchdogs/resource_watchdog.sh b/Watchdogs/resource_watchdog.sh index 7846f35..05aa97b 100755 --- a/Watchdogs/resource_watchdog.sh +++ b/Watchdogs/resource_watchdog.sh @@ -165,27 +165,25 @@ trap "_release_all_locks; _rw_trap_restart_stopped" EXIT # ============================================================================================== # rm_state_get/set use : separator for RM-internal state # rm_state_get_eq/set_eq use = separator for docker_watchdog coordination flags +# Both wrap common.sh's wd_state_get()/wd_state_set() — file is implicit here (this +# script's own state file), unlike the generic helper which always takes it as an argument. rm_state_get() { - grep -E "^${1}:" "$RW_STATE_FILE" 2>/dev/null | cut -d: -f2- + wd_state_get "$1" "$RW_STATE_FILE" } rm_state_set() { local key="$1" val="$2" - grep -vE "^${key}:" "$RW_STATE_FILE" 2>/dev/null > "${RW_STATE_FILE}.tmp" - echo "${key}:${val}" >> "${RW_STATE_FILE}.tmp" - mv "${RW_STATE_FILE}.tmp" "$RW_STATE_FILE" + wd_state_set "$key" "$val" "$RW_STATE_FILE" } rm_state_get_eq() { - grep -E "^${1}=" "$RW_STATE_FILE" 2>/dev/null | cut -d= -f2- + wd_state_get "$1" "$RW_STATE_FILE" "=" } rm_state_set_eq() { local key="$1" val="$2" - grep -vE "^${key}=" "$RW_STATE_FILE" 2>/dev/null > "${RW_STATE_FILE}.tmp" - echo "${key}=${val}" >> "${RW_STATE_FILE}.tmp" - mv "${RW_STATE_FILE}.tmp" "$RW_STATE_FILE" + wd_state_set "$key" "$val" "$RW_STATE_FILE" "=" } # ============================================================================================== @@ -376,7 +374,8 @@ unpause_containers() { } # Stop a list of containers — returns comma-separated list of actually-stopped containers -stop_containers() { +# Named rw_ to avoid colliding with common.sh's stop_containers() (different signature/semantics) +rw_stop_containers() { local actually_stopped=() for container in "$@"; do [[ -z "$container" ]] && continue @@ -407,7 +406,8 @@ stop_containers() { } # Start a comma-separated list of containers (only those RM stopped) -start_containers() { +# Named rw_ to avoid colliding with common.sh's start_containers() (different signature/semantics) +rw_start_containers() { local IFS=',' for container in $1; do [[ -z "$container" ]] && continue @@ -468,7 +468,7 @@ apply_level_3() { if [[ ${#RW_STOP_CONTAINERS[@]} -gt 0 ]]; then local newly_stopped - newly_stopped=$(stop_containers "${RW_STOP_CONTAINERS[@]}") + newly_stopped=$(rw_stop_containers "${RW_STOP_CONTAINERS[@]}") if [[ -n "$newly_stopped" ]]; then if [[ -n "$STOPPED_LIST" ]]; then STOPPED_LIST="${STOPPED_LIST},${newly_stopped}" @@ -490,7 +490,7 @@ apply_level_3() { restore_level_3() { echo "Restoring from level 3 — starting stopped containers" if [[ -n "$STOPPED_LIST" ]]; then - start_containers "$STOPPED_LIST" + rw_start_containers "$STOPPED_LIST" STOPPED_LIST="" fi rm_state_set_eq "mem_shutdown_active" "false" diff --git a/Watchdogs/stability_watchdog.sh b/Watchdogs/stability_watchdog.sh index 414509c..9d61d15 100755 --- a/Watchdogs/stability_watchdog.sh +++ b/Watchdogs/stability_watchdog.sh @@ -192,15 +192,15 @@ fi # ── STATE HELPERS ───────────────────────────────────────────────────────────────────────────── # ============================================================================================== +# Wrap common.sh's wd_state_get()/wd_state_set() — file is implicit here (this script's +# own state file), unlike the generic helper which always takes it as an argument. get_strikes() { - grep -E "^${1}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2 + wd_state_get "$1" "$SYS_WATCHDOG_STATE_FILE" } set_strikes() { local key="$1" count="$2" - grep -vE "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp" - echo "${key}:${count}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp" - mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE" + wd_state_set "$key" "$count" "$SYS_WATCHDOG_STATE_FILE" } increment_strikes() { @@ -218,14 +218,12 @@ reset_strikes() { } get_state_val() { - grep -E "^${1}=" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d'=' -f2 + wd_state_get "$1" "$SYS_WATCHDOG_STATE_FILE" "=" } set_state_val() { local key="$1" val="$2" - grep -vE "^${key}=" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp" - echo "${key}=${val}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp" - mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE" + wd_state_set "$key" "$val" "$SYS_WATCHDOG_STATE_FILE" "=" } purge_old_reboots() { diff --git a/common.sh b/common.sh index c279ad4..8a91141 100755 --- a/common.sh +++ b/common.sh @@ -49,8 +49,7 @@ # check_arr_version() added — API version safety gate before arr operations # check_api() added — pre-flight API reachability check # ping_remote(), ping_internet() — non-fatal ping for fallback use -# check_local_array(), check_remote_array() — array health checks -# check_remote_docker() — Docker daemon health check +# check_remote_array() — array health check # get_unraid_temp_thresholds() — reads thresholds from dynamix.cfg # check_local_disk_temps() — pre-rsync temp check with exit codes 0/1/2 # stop/start local containers added alongside existing remote variants @@ -79,15 +78,12 @@ # Called by lidarr/sonarr/radarr_cleanup.sh when files are deleted # # -# Three new safety functions added: +# Two new safety functions added: # check_os_version_parity() — refuses remote ops on version mismatch # reads OS version via platform_get_os_version() / platform_os_version_probe_cmd() # major mismatch → abort | minor mismatch → configurable warn/abort # check_remote_docker_daemon() — verifies remote Docker daemon before # issuing any remote container commands — strike system → skip/retry/exit -# validate_unraid_cmd() — verifies unRAID-specific commands exist and -# produce expected output before use — notifies and exits calling script -# if command changed or disappeared after upgrade — other scripts unaffected # # ── VERSION ─────────────────────────────────────────────────────────────────────────────────── # Current: v3.5 @@ -130,6 +126,7 @@ ICON_SYNC="🔄" # transfer section header ICON_RUN="🚀" # sync starting / rsync attempt ICON_RETRY="🔁" # retry attempt ICON_DONE="🏁" # transfer complete +ICON_GIT="🌱" # git pull/push section header # Summary ICON_SUMMARY="📋" # summary section header @@ -269,6 +266,33 @@ format_duration() { else echo "${rem}s"; fi } +# Tiered human-readable size — picks B/MB/GB based on magnitude. +# Usage: format_bytes 5368709120 → 5.0GB +format_bytes() { + local bytes=${1:-0} + if (( bytes > 1073741824 )); then + awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}" + elif (( bytes > 1048576 )); then + awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}" + else + echo "${bytes}B" + fi +} + +# Fixed-precision bytes → GB (no tiering) — for values always compared against a GB threshold. +# Usage: bytes_to_gb 5368709120 [decimals=2] → 5.00 +bytes_to_gb() { + local bytes=${1:-0} decimals=${2:-2} + awk "BEGIN {printf \"%.${decimals}f\", $bytes / 1073741824}" +} + +# Fixed-precision KB → GB (no tiering) — df --output=used/avail reports in KB. +# Usage: kb_to_gb 5242880 [decimals=2] → 5.00 +kb_to_gb() { + local kb=${1:-0} decimals=${2:-2} + awk "BEGIN {printf \"%.${decimals}f\", $kb / 1048576}" +} + # ============================================================================================== # ── ARG PARSER ──────────────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -355,6 +379,17 @@ validate_int() { log "$name validated: $value" } +# Membership check against a list of values (e.g. an ignore list). +# Usage: is_in_list "$needle" "${HAYSTACK_ARRAY[@]}" +is_in_list() { + local needle="$1"; shift + local item + for item in "$@"; do + [[ "$needle" == "$item" ]] && return 0 + done + return 1 +} + # ============================================================================================== # ── HOST DETECTION ──────────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -506,6 +541,7 @@ detect_hosts() { SYS_WATCHDOG_CHECK_MDSTAT SYS_WATCHDOG_CHECK_NETWORK SYS_WATCHDOG_CHECK_SSHD SYS_WATCHDOG_CHECK_RUNAWAY ) + local _wd_var for _wd_var in "${_wd_checks[@]}"; do local _wd_src="${MY_ID}_${_wd_var}" # Only alias if the host-specific var is set — preserves master.conf defaults @@ -625,6 +661,86 @@ detect_hosts() { log "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" } +# Derives owner/mirror roles from PARTNERSHIP_OWNER_HOST — used by every Partnership/*.sh +# script. Requires detect_hosts() to have already run (needs MY_ID, SSH_KEY). +# Sets: OWNER_ID MIRROR_ID OWNER MIRROR MIRROR_SSH_KEY OWNER_SSH_KEY AM_OWNER AM_MIRROR +# Does NOT set any *_STATE_FILE vars — callers that need those set them after calling this, +# since not every Partnership script needs the same set (transfer/onboard need fewer than +# manager/offboard). +partnership_resolve_roles() { + OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}" + MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" ) + OWNER="${!OWNER_ID}" + MIRROR="${!MIRROR_ID}" + # SSH_KEY (set by detect_hosts) is this server's own private key. The remote accepts it + # because this server's PUBLIC key was installed there via ssh_setup.sh. With sparse + # checkout, each server only has its own host{N}.conf — the other server's key path is + # never available here. Use SSH_KEY for all outbound SSH regardless of mode. + MIRROR_SSH_KEY="$SSH_KEY" + OWNER_SSH_KEY="$SSH_KEY" + + AM_OWNER=false + AM_MIRROR=false + [[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true + [[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true +} + +# Read a scalar/array var from a partner's own config via SSH — the partner sources its +# own load_config.sh + detect_hosts() remotely so the value reflects ITS host*.conf, not +# ours. Requires $SCRIPT_DIR to be set by the caller (one level above the repo root, same +# convention every Partnership/*.sh script already uses for its own SCRIPT_DIR). +# Usage: read_remote_conf_var "$mirror_ip" "VAR_NAME" +# read_remote_conf_array "$mirror_ip" "ARRAY_NAME" # one element per line +read_remote_conf_var() { + local mirror_ip="$1" var_name="$2" + timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ + "source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null + detect_hosts 2>/dev/null + printf '%s' \"\${${var_name}:-}\"" 2>/dev/null +} + +read_remote_conf_array() { + local mirror_ip="$1" var_name="$2" + timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ + "source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null + detect_hosts 2>/dev/null + printf '%s\n' \"\${${var_name}[@]:-}\"" 2>/dev/null +} + +# Probes a remote host's SCRIPTS_DIR (in case it differs from ours — e.g. one host runs +# internal storage mode, the other flash), falling back to our own $SCRIPTS_DIR if the +# probe fails or the remote isn't reachable. strict_host_key defaults to "yes" (matches +# existing behavior everywhere except partnership_transfer.sh, which passes "no" since +# it may be contacting a mirror for the first time during an ownership transfer). +# Usage: resolve_remote_scripts_dir "$ip" ["$ssh_key"] ["no"|"yes"] +resolve_remote_scripts_dir() { + local ip="$1" ssh_key="${2:-$SSH_KEY}" strict_host_key="${3:-yes}" + local probe_cmd result + local -a opts=(-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes) + [[ "$strict_host_key" == "no" ]] && opts+=(-o StrictHostKeyChecking=no) + probe_cmd=$(platform_scripts_dir_probe_cmd) + result=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" "${opts[@]}" root@"$ip" "$probe_cmd" \ + 2>/dev/null | tr -d '[:space:]') + echo "${result:-$SCRIPTS_DIR}" +} + +# ============================================================================================== +# ── MULTI-NODE DISCOVERY ────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Scans HOST1..HOST8 for every configured node other than this one. Requires detect_hosts() +# to have already run (needs MY_ID). Sets the global REMOTE_NODES array — empty if none found. +discover_remote_nodes() { + REMOTE_NODES=() + local _hv + for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do + [[ "$_hv" == "$MY_ID" ]] && continue + [[ -z "${!_hv:-}" ]] && continue + REMOTE_NODES+=("$_hv") + done +} + # ============================================================================================== # ── REMOTE IP RESOLUTION ────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -669,6 +785,34 @@ resolve_tailscale_ip() { echo "$ip" } +# Strips the "unraid-" prefix (case-insensitive) and title-cases what remains. +# Usage: derive_short_name "unRAID-Gmer4Lfe" → "Gmer4lfe" +derive_short_name() { + local hostname="$1" + local short="${hostname,,}" + [[ "$short" == unraid-* ]] && short="${short:7}" + echo "${short^}" +} + +# True if the given host*.conf basename belongs to this host (case-insensitive). +# Usage: is_own_conf_file "$(basename "$conf")" +is_own_conf_file() { + [[ "${1,,}" == "${MY_ID,,}.conf" ]] +} + +# Sets key=value in a flat state file (setup.db style) — updates in place if the +# key exists, appends if not. No indentation handling — for master.conf use +# partnership_manager.sh's update_master_conf() instead. +# Usage: set_state_var "$state_file" "$key" "$value" +set_state_var() { + local file="$1" key="$2" value="$3" + if grep -q "^${key}=" "$file" 2>/dev/null; then + sed -i "s|^${key}=.*|${key}=${value}|" "$file" + else + echo "${key}=${value}" >> "$file" + fi +} + # ============================================================================================== # ── CONNECTIVITY CHECKS ─────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -755,26 +899,6 @@ is_vm_manager_enabled() { # ── LOCAL HEALTH CHECKS ─────────────────────────────────────────────────────────────────────── # ============================================================================================== -# Verifies local storage is mounted and has shares. -# Non-fatal — returns status for caller to decide. -# Used by fallback before starting remote containers locally. -check_local_array() { - log "Checking local array..." - if ! platform_storage_healthy; then - error "$ICON_DISK Local array is not started — storage not mounted" - return 1 - fi - local _storage_path file_count - _storage_path=$(platform_storage_path) - file_count=$(ls "$_storage_path" 2>/dev/null | wc -l) - if [[ "$file_count" -eq 0 ]]; then - error "$ICON_DISK Local array appears empty — shares may not be available" - return 1 - fi - log "Local array is healthy" - return 0 -} - # ============================================================================================== # ── REMOTE HEALTH CHECKS ───────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -796,19 +920,6 @@ check_remote_array() { return 0 } -# Verifies remote Docker daemon is responding. -# Non-fatal — returns status. A hung daemon means container commands silently fail. -check_remote_docker() { - log "Checking remote Docker daemon on $REMOTE_SERVER_NAME..." - if ! ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ - "timeout 10 docker ps" >/dev/null 2>&1; then - error "$ICON_CONTAINERS Remote Docker daemon not responding on $REMOTE_SERVER_NAME" - return 1 - fi - log "Remote Docker daemon is healthy" - return 0 -} - # Aborts if remote rootfs (/) usage is at or above ROOTFS_WARN threshold. # Fatal — exits the calling script. # When remote array is down rsync writes land on rootfs and fill it rapidly. @@ -861,9 +972,14 @@ check_remote_share() { # Returns 0 (true) if device is non-rotational (SSD/NVMe), 1 (false) if HDD. # Checks /sys/block//queue/rotational — 0=SSD, 1=HDD. is_ssd() { - local base - base=$(basename "$1") - [[ "$(cat "/sys/block/$base/queue/rotational" 2>/dev/null)" == "0" ]] + local drive="$1" + local base rotational + base=$(basename "$drive") + rotational="/sys/block/$base/queue/rotational" + [[ -f "$rotational" ]] && [[ "$(cat "$rotational" 2>/dev/null)" == "0" ]] && return 0 + # NVMe is always SSD — some NVMe controllers don't expose rotational correctly + [[ "$drive" == *nvme* ]] && return 0 + return 1 } # Sets globals: UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX @@ -878,6 +994,37 @@ get_unraid_temp_thresholds() { } # Checks local disk temperatures before rsync. +# Fetches a domain's TLS cert expiry via openssl s_client + x509 and computes days +# remaining. Pure data — no logging, no thresholds — callers classify warn/crit +# themselves against their own CERT_WARN_DAYS/CERT_CRIT_DAYS and log however they like. +# +# Sets _CERT_DAYS, _CERT_EXPIRY (YYYY-MM-DD) on success. Sets _CERT_EXPIRY_RAW to the +# unparsed openssl date string only when parsing failed (empty when unreachable) so a +# caller that wants a more specific error message can distinguish the two failure modes. +# +# Usage: check_cert_expiry "$domain" [port=443] [timeout=$CERT_TIMEOUT or 10] +# Returns: 0 = fetched and parsed | 1 = unreachable or unparseable +check_cert_expiry() { + local domain="$1" port="${2:-443}" cert_timeout="${3:-${CERT_TIMEOUT:-10}}" + _CERT_DAYS="" + _CERT_EXPIRY="" + _CERT_EXPIRY_RAW="" + local expiry_str + expiry_str=$(echo | timeout "$cert_timeout" openssl s_client \ + -connect "${domain}:${port}" -servername "$domain" \ + 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) + [[ -z "$expiry_str" ]] && return 1 + local expiry_epoch + expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null) + if [[ -z "$expiry_epoch" ]]; then + _CERT_EXPIRY_RAW="$expiry_str" + return 1 + fi + _CERT_DAYS=$(( (expiry_epoch - $(date +%s)) / 86400 )) + _CERT_EXPIRY=$(date -d "$expiry_str" '+%Y-%m-%d' 2>/dev/null) + return 0 +} + # Reads temps and rotational flag from /var/local/emhttp/disks.ini. # Uses unRAID's own thresholds from dynamix.cfg via get_unraid_temp_thresholds(). # @@ -1305,6 +1452,62 @@ start_local_containers() { done } +# Builds a dependency-safe restart order — dependency containers (anything named as a +# value in WATCHDOG_DEPENDENCIES) restart first, then everything else in original order. +# Sets ORDERED_RESTART. Usage: build_restart_order CONTAINERS_ARRAY_NAME (bare name, e.g. +# build_restart_order DAILY_RESTART_CONTAINERS — not "${DAILY_RESTART_CONTAINERS[@]}"). +build_restart_order() { + local -n _bro_containers="$1" + ORDERED_RESTART=() + local remaining=("${_bro_containers[@]}") + local placed=() + + # First pass — add dependency containers that appear in our list + for container in "${remaining[@]}"; do + [[ -z "$container" ]] && continue + local is_dependency=false + for dependent in "${!WATCHDOG_DEPENDENCIES[@]}"; do + if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then + is_dependency=true + break + fi + done + if [[ "$is_dependency" == true ]]; then + local already=false + for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done + if [[ "$already" == false ]]; then + ORDERED_RESTART+=("$container") + placed+=("$container") + fi + fi + done + + # Second pass — add remaining containers (dependents and independents) + for container in "${remaining[@]}"; do + [[ -z "$container" ]] && continue + local already=false + for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done + if [[ "$already" == false ]]; then + ORDERED_RESTART+=("$container") + placed+=("$container") + fi + done + + log "Restart order: ${ORDERED_RESTART[*]}" +} + +# Waits CONTAINER_DELAY if this container depends on the last restarted one. +# Usage: check_dependency_delay "$container" "$last_restarted" +check_dependency_delay() { + local container="$1" last="$2" + [[ -z "$last" ]] && return + local deps="${WATCHDOG_DEPENDENCIES[$container]:-}" + if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then + log "Waiting ${CONTAINER_DELAY}s — $container depends on $last..." + sleep "$CONTAINER_DELAY" + fi +} + # ============================================================================================== # ── RSYNC OPTIONS ───────────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -1380,12 +1583,6 @@ _lock_file() { echo "$LOCK_DIR/${1:-$(_lock_name)}.lock" } -# Internal — release lock on exit (registered via EXIT trap) -_release_on_exit() { - local lockfile="$1" - [[ -f "$lockfile" ]] && rm -f "$lockfile" -} - # Acquire exclusive lock for this script. # Usage: acquire_lock [strict|wait|continuous] acquire_lock() { @@ -1524,17 +1721,30 @@ acquire_rsync_lock() { log "$ICON_LOCK rsync lock acquired: profile '$profile' (PID $$, active: $(( current_count + 1 ))/$RSYNC_MAX_CONCURRENT)" } -# Internal — release rsync lock and decrement global counter on exit -_release_rsync_on_exit() { - local profile_lock="$1" - [[ -f "$profile_lock" ]] && rm -f "$profile_lock" - if [[ -f "$RSYNC_COUNT_FILE" ]]; then - local count - count=$(cat "$RSYNC_COUNT_FILE" 2>/dev/null || echo 1) - count=$(( count - 1 )) - [[ "$count" -lt 0 ]] && count=0 - echo "$count" > "$RSYNC_COUNT_FILE" - fi +# ============================================================================================== +# ── FLAT STATE-FILE KEY/VALUE HELPERS ───────────────────────────────────────────────────────── +# ============================================================================================== +# Generic get/set for flat "keyvalue" state files (one entry per line) — the pattern +# every watchdog's strike-tracking and state-file logic was independently reimplementing. +# grep -v + tempfile-swap on write, not sed -i in place — avoids sed treating a key containing +# regex metacharacters (container names, etc.) as part of the substitution pattern. +# +# Usage: wd_state_get "$key" "$file" [sep=:] +# wd_state_set "$key" "$value" "$file" [sep=:] +# +# Callers with a fixed state file and/or separator (e.g. stability_watchdog.sh's +# get_strikes/get_state_val, resource_watchdog.sh's rm_state_get/rm_state_get_eq) should +# keep their own thin same-named wrapper around these rather than changing call sites. +wd_state_get() { + local key="$1" file="$2" sep="${3:-:}" + grep -E "^${key}${sep}" "$file" 2>/dev/null | cut -d"$sep" -f2- +} + +wd_state_set() { + local key="$1" value="$2" file="$3" sep="${4:-:}" + grep -vE "^${key}${sep}" "$file" 2>/dev/null > "${file}.tmp" + echo "${key}${sep}${value}" >> "${file}.tmp" + mv "${file}.tmp" "$file" } # ============================================================================================== @@ -1632,6 +1842,348 @@ translate_path() { fi } +# ============================================================================================== +# ── CONTAINER STOP/RESTART FOR MAINTENANCE ──────────────────────────────────────────────────── +# ============================================================================================== +# The "stop a container for a maintenance window, guarantee it comes back" shape shared by +# container_data_export.sh and emby_database_repair.sh. Callers still register their own EXIT +# trap (they may need extra cleanup, e.g. removing a partial archive) — the trap should call +# container_force_restart_if_needed() for the actual "still down? bring it back" part. + +# Usage: container_stop_for_maintenance "$container" was_running_var_name \ +# ["pre-stop reason message"] [log|warn] [post_stop_sleep=0] +# The reason message (if given) prints via the chosen level (default: log) right before the +# stop attempt, only when the container was actually running — callers use this for their +# own context ("stopping for clean export" vs "active sessions will be interrupted"). +# Sets the named var true/false. Returns 1 (caller should exit) if the container doesn't +# exist or fails to stop; 0 otherwise (including the "wasn't running" case — caller logs +# its own context-specific message for that, since wording/visibility differs per caller). +container_stop_for_maintenance() { + local container="$1" + local -n _csm_was_running="$2" + local pre_stop_msg="${3:-}" pre_stop_level="${4:-log}" post_stop_sleep="${5:-0}" + _csm_was_running=false + + local status + status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + case "$status" in + true) + _csm_was_running=true + if [[ -n "$pre_stop_msg" ]]; then + if [[ "$pre_stop_level" == "warn" ]]; then warn "$pre_stop_msg"; else log "$pre_stop_msg"; fi + fi + if [[ "$DRY_RUN" == false ]]; then + if timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1; then + echo "$container stopped ✅" + [[ "$post_stop_sleep" -gt 0 ]] && sleep "$post_stop_sleep" + else + error "Failed to stop $container — aborting" + return 1 + fi + else + warn "DRY RUN — would stop $container" + fi + ;; + false) : ;; # not running — caller logs its own context-specific message + "") + error "$container not found — check container name" + return 1 + ;; + *) + warn "$container status: $status — proceeding with caution" + ;; + esac + return 0 +} + +# Force-restart from inside a script's own EXIT trap if the container is still down after +# a crash mid-maintenance. No-op if it wasn't running before, in dry-run, or already up. +# Usage: container_force_restart_if_needed "$container" "$was_running" +container_force_restart_if_needed() { + local container="$1" was_running="$2" + [[ "$was_running" == true && "$DRY_RUN" == false ]] || return 0 + local status + status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + if [[ "$status" != "true" ]]; then + warn "Restarting $container (cleanup)..." + timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1 || \ + error "Failed to restart $container — start it manually" + fi +} + +# Normal-path restart after maintenance completes — verifies it stayed up, notifies on failure. +# Sets RESTART_OK true/false. +# Usage: container_restart_after_maintenance "$container" "$was_running" [settle_sleep=3] [notify_label] +container_restart_after_maintenance() { + local container="$1" was_running="$2" settle_sleep="${3:-3}" notify_label="${4:-Container Maintenance}" + RESTART_OK=false + + if [[ "$was_running" != true ]]; then + echo "$container was not running — leaving stopped (state respected) ✅" + RESTART_OK=true + return 0 + fi + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would restart $container" + RESTART_OK=true + return 0 + fi + + log "Restarting $container..." + if timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1; then + sleep "$settle_sleep" + local post_status + post_status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + if [[ "$post_status" == "true" ]]; then + echo "$container restarted and running ✅" + RESTART_OK=true + else + error "$container started but crashed — check container logs" + notify "$container failed to stay running after maintenance on $(hostname)" \ + "$notify_label" "warning" + fi + else + error "Failed to restart $container — start it manually" + notify "$container failed to restart after maintenance on $(hostname)" \ + "$notify_label" "warning" + fi +} + +# ============================================================================================== +# ── ARR CLEANUP SAFETY GATES ────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Shared by lidarr/radarr/sonarr_cleanup.sh — these scripts DELETE files, so every function +# here is a direct byte-for-byte port of what was independently duplicated three times, not +# a rewrite. Callers should behave identically to before this consolidation. + +# Safety Layer 1 — container running, not starting/unhealthy. Exits 1 (with notify) on failure. +# Usage: check_container_health "$LIDARR_CONTAINER" "$ARR_DOCKER_TIMEOUT" "Lidarr Cleanup" +check_container_health() { + local container="$1" docker_timeout="$2" notify_label="$3" + + local running + running=$(timeout "$docker_timeout" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + if [[ "$running" != "true" ]]; then + error "$container is not running — aborting" + notify "$notify_label aborted on $(hostname) — container not running" "$notify_label" "warning" + exit 1 + fi + + local health + health=$(timeout "$docker_timeout" docker inspect -f '{{.State.Health.Status}}' "$container" 2>/dev/null) + case "$health" in + healthy) info "$container is healthy" ;; + "") info "$container has no health check — proceeding" ;; + starting) + error "$container is still starting — aborting" + notify "$notify_label aborted on $(hostname) — container still starting" "$notify_label" "warning" + exit 1 ;; + unhealthy) + error "$container is unhealthy — aborting" + notify "$notify_label aborted on $(hostname) — container unhealthy" "$notify_label" "warning" + exit 1 ;; + *) warn "$container health: $health — proceeding with caution" ;; + esac + + info "Safety layer 1 passed — container healthy" +} + +# Safety Layer 6 — abort if tracked count dropped below min_pct of the last known count +# (protects against a misconfigured path / partial API response silently wiping the library). +# Writes the new baseline only if the check passes. Exits 1 (with notify) on failure. +# Usage: check_tracked_count_floor "$TRACKED_COUNT" "$LIDARR_TRACKED_COUNT_FILE" "$LIDARR_MIN_TRACKED_PCT" "Lidarr Cleanup" +check_tracked_count_floor() { + local tracked_count="$1" baseline_file="$2" min_pct="$3" notify_label="$4" + + if [[ -f "$baseline_file" ]]; then + local last_count pct + last_count=$(cat "$baseline_file" 2>/dev/null || echo 0) + if [[ "$last_count" -gt 0 ]]; then + pct=$(awk "BEGIN {printf \"%d\", ($tracked_count / $last_count) * 100}") + if [[ "$pct" -lt "$min_pct" ]]; then + error "Tracked count dropped to ${pct}% of last run ($tracked_count vs $last_count)" + error "Suggests API issue — aborting to prevent mass deletion" + error "If expected (large library removal) delete: $baseline_file" + notify "$notify_label aborted on $(hostname) — tracked count dropped to ${pct}%" \ + "$notify_label" "warning" + exit 1 + fi + info "Tracked count: ${pct}% of last run ($tracked_count vs $last_count) ✅" + fi + else + info "No previous count on record — first run, saving baseline" + fi + + echo "$tracked_count" > "$baseline_file" +} + +# Filters --i-know-what-im-doing / --skip-age-check out of "$@" before parse_args sees them +# (both are cleanup-script-specific, not part of the shared arg parser). +# Sets I_KNOW, SKIP_AGE_CHECK, FILTERED_ARGS — call parse_args "${FILTERED_ARGS[@]}" after. +parse_destructive_flags() { + I_KNOW=false + SKIP_AGE_CHECK=false + FILTERED_ARGS=() + local arg + for arg in "$@"; do + case "$arg" in + --i-know-what-im-doing) I_KNOW=true ;; + --skip-age-check) SKIP_AGE_CHECK=true ;; + *) FILTERED_ARGS+=("$arg") ;; + esac + done +} + +# Prints the nuclear-mode banner and sleeps 10s (Ctrl+C window) when both destructive flags +# are set and this isn't a dry run. Call after parse_args. No-op otherwise. +nuclear_mode_warning() { + if [[ "$I_KNOW" == true ]] && [[ "$SKIP_AGE_CHECK" == true ]] && [[ "$DRY_RUN" != true ]]; then + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "⚠️ WARNING — NUCLEAR MODE ACTIVE" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " Flags: --i-know-what-im-doing --skip-age-check" + echo " Age check: BYPASSED — deletes on first pass" + echo " Size threshold: BYPASSED — no GB limit" + echo " Data recovery: NOT POSSIBLE after deletion" + echo "" + echo " Review --dry-run output before proceeding." + echo " You have 10 seconds to cancel (Ctrl+C)..." + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + sleep 10 + echo " Proceeding..." + echo "" + fi +} + +# Safety Layer 7 — abort if total deletion size exceeds max_gb and --i-know-what-im-doing +# wasn't passed. Sets TOTAL_HUMAN. Exits 1 (with notify) on failure; warns and continues +# if I_KNOW is true (set by parse_destructive_flags). +# Usage: check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$LIDARR_MAX_DELETE_GB" "Lidarr Cleanup" +check_delete_size_threshold() { + local total_bytes="$1" max_gb="$2" notify_label="$3" + local max_bytes + max_bytes=$(awk "BEGIN {printf \"%d\", $max_gb * 1073741824}") + if [[ "$total_bytes" -gt "$max_bytes" ]]; then + TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $total_bytes / 1073741824}") + if [[ "$I_KNOW" != true ]]; then + echo "" + error "Deletion would exceed ${max_gb}GB — $TOTAL_HUMAN would be deleted" + error "Review ORPHAN lines above carefully before proceeding" + error "Rerun with: --i-know-what-im-doing" + error "To also bypass age check: add --skip-age-check" + notify "$notify_label halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \ + "$notify_label" "warning" + exit 1 + else + warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing" + fi + fi +} + +# True if filepath's extension (case-insensitive) matches one of the given extensions. +# Usage: has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}" +has_extension() { + local filepath="$1"; shift + local ext="${filepath##*.}" + ext="${ext,,}" + local valid_ext + for valid_ext in "$@"; do + [[ "$ext" == "$valid_ext" ]] && return 0 + done + return 1 +} + +# True if filepath's basename matches one of the given glob patterns. +# Usage: matches_pattern_list "$filepath" "${LIDARR_PROTECTED_PATTERNS[@]}" +matches_pattern_list() { + local filepath="$1"; shift + local filename + filename=$(basename "$filepath") + local pattern + for pattern in "$@"; do + # shellcheck disable=SC2254 + case "$filename" in + $pattern) return 0 ;; + esac + done + return 1 +} + +# Builds the global ARR_PATH_MAP associative array (container path → host path) from +# ${MY_ID}__PATH_MAP, for translate_path() to use. Usage: build_arr_path_map "LIDARR" +build_arr_path_map() { + local arr_type="$1" + declare -gA ARR_PATH_MAP=() + local _bapm_var="${MY_ID}_${arr_type}_PATH_MAP" + eval "for key in \"\${!${_bapm_var}[@]}\"; do + ARR_PATH_MAP[\"\$key\"]=\"\${${_bapm_var}[\$key]}\" + done" +} + +# Generic arr API GET wrapper with HTTP status check. +# Usage: arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "movie" "Radarr" +arr_api() { + local base_url="$1" api_key="$2" api_version="$3" endpoint="$4" label="${5:-Arr}" + local response http_code body + + response=$(curl -sf \ + --max-time 30 \ + -H "X-Api-Key: $api_key" \ + -w "\n%{http_code}" \ + "${base_url}/api/${api_version}/${endpoint}" 2>/dev/null) + + http_code=$(echo "$response" | tail -1) + body=$(echo "$response" | head -n -1) + + if [[ "$http_code" != "200" ]]; then + error "$label API HTTP $http_code for: $endpoint" + return 1 + fi + echo "$body" +} + +# Triggers an arr "command" endpoint with the given JSON payload, then polls every 10s until +# completed/failed/timeout, logging progress every 60s. Best-effort pre-flight — never treats +# an unreachable/timed-out scan as fatal, matching original per-script behavior. +# Usage: trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" \ +# '{"name": "DownloadedAlbumsScan"}' "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}" +trigger_and_await_command() { + local base_url="$1" api_key="$2" api_version="$3" payload="$4" poll_timeout="${5:-600}" + + local scan_response scan_cmd_id + scan_response=$(curl -sf --max-time 30 -X POST \ + -H "X-Api-Key: $api_key" \ + -H "Content-Type: application/json" \ + -d "$payload" \ + "${base_url}/api/${api_version}/command" 2>/dev/null) + + scan_cmd_id=$(echo "$scan_response" | jq -r '.id // empty' 2>/dev/null) + + if [[ -z "$scan_cmd_id" ]]; then + warn "Could not trigger import scan — proceeding without pre-flight" + return 0 + fi + + info "Import scan queued (command ID: $scan_cmd_id) — waiting for completion..." + local polled=0 scan_status + while [[ "$polled" -lt "$poll_timeout" ]]; do + scan_status=$(curl -sf --max-time 10 \ + -H "X-Api-Key: $api_key" \ + "${base_url}/api/${api_version}/command/${scan_cmd_id}" 2>/dev/null | \ + jq -r '.status // empty' 2>/dev/null) + case "$scan_status" in + completed) info "Import scan complete ✅"; return 0 ;; + failed) warn "Import scan reported failed — proceeding anyway"; return 0 ;; + esac + sleep 10 + (( polled += 10 )) + [[ $(( polled % 60 )) -eq 0 ]] && log " Still scanning... (${polled}s elapsed)" + done + warn "Import scan timed out after ${poll_timeout}s — proceeding anyway" + return 0 +} + # ============================================================================================== # ── ARR VERSION CHECK ───────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -1692,6 +2244,48 @@ check_arr_version() { fi } +# ============================================================================================== +# ── TMDB DISCOVERY SCORING ──────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Shared by playback_aware_radarr_discovery.sh and playback_aware_sonarr_discovery.sh — stage 2 +# candidate scoring against TMDB rating/vote/seed-breadth data. Byte-for-byte identical between +# the two before this consolidation. + +# Formats a TMDB vote_average×10 integer back to one decimal place (e.g. 78 → 7.8). +_fmt_rating() { local v="${1:-0}"; echo "${v::-1}.${v: -1}" 2>/dev/null || echo "$v"; } + +# Stage 2: TMDB vote_average × 10 as integer (0-40) +_rating_score_s2() { + local v="$1" + if (( v >= 80 )); then echo 40 + elif (( v >= 75 )); then echo 32 + elif (( v >= 70 )); then echo 25 + elif (( v >= 65 )); then echo 18 + elif (( v >= 60 )); then echo 12 + else echo 5 + fi +} + +# Stage 2: vote count (0-20) +_votes_score() { + local c="$1" + if (( c >= 10000 )); then echo 20 + elif (( c >= 5000 )); then echo 15 + elif (( c >= 1000 )); then echo 10 + elif (( c >= 200 )); then echo 5 + else echo 2 + fi +} + +# Stage 2: seed breadth (0-40) +_breadth_score() { + local seeds="$1" + if (( seeds >= 3 )); then echo 40 + elif (( seeds == 2 )); then echo 25 + else echo 10 + fi +} + # ============================================================================================== # ── EMBY LIBRARY SCAN ───────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -1911,54 +2505,6 @@ check_remote_docker_daemon() { return 1 } -# ============================================================================================== -# ── UNRAID COMMAND VALIDATION ───────────────────────────────────────────────────────────────── -# ============================================================================================== -# Verifies a unRAID-specific command exists and produces expected output before use. -# Protects against commands moving, changing format, or disappearing after upgrades. -# -# If validation fails — notifies and returns 1. Caller exits its own section only. -# Other scripts that pass validation are unaffected. -# -# Usage: -# validate_unraid_cmd # "/usr/local/emhttp/plugins/dynamix/scripts/notify" # "--help" # "Usage" # "unRAID notify script" -# -# Arguments: -# $1 = full path to command -# $2 = test argument to pass (use "" for no argument) -# $3 = expected string in output (use "" to skip output check) -# $4 = human readable name for error messages - -validate_unraid_cmd() { - local cmd_path="$1" - local test_arg="$2" - local expected_output="$3" - local cmd_name="${4:-$1}" - - # Check command exists and is executable - if [[ ! -x "$cmd_path" ]]; then - error "unRAID command not found or not executable: $cmd_path" - error "$cmd_name may have moved or been removed — check after recent unRAID upgrade" - notify "$cmd_name not found on $(hostname) at $cmd_path — check after unRAID upgrade" "Command Validation" "warning" - return 1 - fi - - # Check output matches expected pattern if provided - if [[ -n "$expected_output" ]]; then - local actual_output - actual_output=$(timeout 10 "$cmd_path" $test_arg 2>&1 || true) - if ! echo "$actual_output" | grep -q "$expected_output"; then - error "$cmd_name output format changed — expected '$expected_output' not found" - error "Command may have changed after unRAID upgrade — review $cmd_path" - notify "$cmd_name output format changed on $(hostname) — may need script update after unRAID upgrade" "Command Validation" "warning" - return 1 - fi - fi - - log "$cmd_name validated: $cmd_path ✅" - return 0 -} - # ============================================================================================== # ── STATUS DISPLAY ──────────────────────────────────────────────────────────────────────────── # ============================================================================================== diff --git a/load_config.sh b/load_config.sh index 985f46d..ab2dfaf 100755 --- a/load_config.sh +++ b/load_config.sh @@ -135,7 +135,8 @@ CONF_DIR="${SCRIPTS_DIR}/Configurations" LOG_DIR="/var/log/varaverk" VV_CACHE_DIR="/tmp/vv_cache" - export CONF_DIR LOG_DIR VV_CACHE_DIR + CONF_RAM_CACHE_DIR="/tmp/.cache/vv/d" # tmpfs — cleared every reboot, repopulated by conf_sync.sh + export CONF_DIR LOG_DIR VV_CACHE_DIR CONF_RAM_CACHE_DIR # ━━━ Cleanup ━━━ unset _conf _host_confs_loaded _adapter LOAD_CONFIG_DIR \ No newline at end of file