From b3f72417a4a8b567558e7408b60dc671210d0047 Mon Sep 17 00:00:00 2001 From: FailedProxy Date: Fri, 24 Apr 2026 17:22:34 -0400 Subject: [PATCH] fixed arrs clean up showing only orphans --- Docker_Essentials/docker_network_connect.sh | 4 +- Master.conf | 41 + Media/lidarr_cleanup.sh | 190 ++- Media/radarr_cleanup.sh | 30 +- Media/sonarr_cleanup.sh | 30 +- common.sh | 54 +- unRAID_Essentials/system_watchdog.sh | 1218 +++++++++++++------ 7 files changed, 1115 insertions(+), 452 deletions(-) diff --git a/Docker_Essentials/docker_network_connect.sh b/Docker_Essentials/docker_network_connect.sh index d40b522..90fbe81 100644 --- a/Docker_Essentials/docker_network_connect.sh +++ b/Docker_Essentials/docker_network_connect.sh @@ -87,7 +87,7 @@ for container in "${NETWORK_CONNECT_CONTAINERS[@]}"; do # Verify container exists if ! docker inspect "$container" &>/dev/null; then warn "$container not found — skipping" - FAILED+=("$container") + continue echo "" continue fi @@ -98,7 +98,7 @@ for container in "${NETWORK_CONNECT_CONTAINERS[@]}"; do # Verify network exists if ! docker network inspect "$network" &>/dev/null; then warn "Network $network not found — skipping" - FAILED+=("$container:$network") + continue continue fi diff --git a/Master.conf b/Master.conf index c42cf7f..4d3c778 100644 --- a/Master.conf +++ b/Master.conf @@ -760,6 +760,17 @@ HOST1_LIDARR_URL="http://192.168.50.2:8686" HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc" HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New" +# Container path → host path translation +# Lidarr stores file paths using container paths — script scans host paths +# Add one entry per root folder configured in Lidarr Settings → Media Management → Root Folders +declare -A HOST1_LIDARR_PATH_MAP=( + ["/ext-music"]="/mnt/user/Music-New" +) +declare -A HOST2_LIDARR_PATH_MAP=( + # HOST2 does not run Lidarr — fill in if that changes + # ["/ext-music"]="/mnt/user/Music-New" +) + LIDARR_ORPHAN_AGE=7 LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") LIDARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.lrc") @@ -772,10 +783,25 @@ HOST1_SONARR_URL="http://192.168.50.2:8989" HOST1_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f" HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows" +# Container path → host path translation +# Add one entry per root folder configured in Sonarr Settings → Media Management → Root Folders +# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder +declare -A HOST1_SONARR_PATH_MAP=( + ["/tv"]="/mnt/user/Tv_Shows" + ["/ext-standup-comedy"]="/mnt/user/stand-up_comedy" + ["/kids tv"]="/mnt/user/Kids_Tv_Shows" + ["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old" +) + HOST2_SONARR_URL="http://localhost:8989" HOST2_SONARR_API_KEY="your-host2-sonarr-api-key" HOST2_SONARR_TV_ROOT="/mnt/user/Anime_Shows" +declare -A HOST2_SONARR_PATH_MAP=( + # Fill in when HOST2 is back online + # ["/tv"]="/mnt/user/Anime_Shows" +) + SONARR_ORPHAN_AGE=7 SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") SONARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa") @@ -785,10 +811,25 @@ HOST1_RADARR_URL="http://192.168.50.2:7878" HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies" +# Container path → host path translation +# Add one entry per root folder configured in Radarr Settings → Media Management → Root Folders +# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder +declare -A HOST1_RADARR_PATH_MAP=( + ["/movies"]="/mnt/user/Movies" + ["/kids movies"]="/mnt/user/Kids_Movies" + ["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy" + ["/anime-movies"]="/mnt/user/Anime_Movies-Old" +) + HOST2_RADARR_URL="http://localhost:7878" HOST2_RADARR_API_KEY="your-host2-radarr-api-key" HOST2_RADARR_MOVIES_ROOT="/mnt/user/Anime_Movies" +declare -A HOST2_RADARR_PATH_MAP=( + # Fill in when HOST2 is back online + # ["/anime-movies"]="/mnt/user/Anime_Movies" +) + RADARR_ORPHAN_AGE=7 RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") RADARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa") diff --git a/Media/lidarr_cleanup.sh b/Media/lidarr_cleanup.sh index 605549f..aa2217d 100644 --- a/Media/lidarr_cleanup.sh +++ b/Media/lidarr_cleanup.sh @@ -19,6 +19,18 @@ # not include these in its tracked file API response. Without protection these would # be classified as orphans and deleted — breaking Lidarr and Emby metadata display. # +# Safety layers — all must pass before any file is touched: +# 1. Container must be running and healthy +# 2. API must be reachable +# 3. Artist count must be > 0 +# 4. Tracked file count must be > 0 +# 5. Tracked count must be >= LIDARR_MIN_TRACKED_PCT% of last known count +# 6. Deletion size must be < LIDARR_MAX_DELETE_GB — or --i-know-what-im-doing required +# +# --i-know-what-im-doing flag: +# Required when deletion would exceed LIDARR_MAX_DELETE_GB +# Long and annoying by design — cannot be added accidentally +# # Lidarr runs on HOST1 only — music library is HOST1's source of truth. # If run on HOST2 this script exits cleanly with no action. # All configuration in Master.conf under Arr Cleanup section. @@ -30,7 +42,18 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../Master.conf" source "$SCRIPT_DIR/../common.sh" -parse_args "$@" +# Check for --i-know-what-im-doing flag before parse_args +I_KNOW=false +FILTERED_ARGS=() +for arg in "$@"; do + if [[ "$arg" == "--i-know-what-im-doing" ]]; then + I_KNOW=true + else + FILTERED_ARGS+=("$arg") + fi +done + +parse_args "${FILTERED_ARGS[@]}" # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_GEAR Setup ━━━ @@ -67,6 +90,7 @@ fi LIDARR_URL="$HOST1_LIDARR_URL" LIDARR_API_KEY="$HOST1_LIDARR_API_KEY" LIDARR_MUSIC_ROOT="$HOST1_LIDARR_MUSIC_ROOT" +LIDARR_CONTAINER="Lidarr" # Docker container name for health check info "Lidarr instance: $LOCAL_SERVER_NAME → $LIDARR_URL" info "Music root: $LIDARR_MUSIC_ROOT" @@ -83,7 +107,43 @@ fi acquire_lock "wait" -success "Config validated" +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SHIELD Safety Layer 1 — Container Health ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_SHIELD Safety Checks ━━━" + +# Check container is running +CONTAINER_RUNNING=$(docker inspect -f '{{.State.Running}}' "$LIDARR_CONTAINER" 2>/dev/null) +if [[ "$CONTAINER_RUNNING" != "true" ]]; then + error "$LIDARR_CONTAINER container is not running — aborting" + notify "Lidarr cleanup aborted on $(hostname) — Lidarr container not running" "Lidarr Cleanup" "warning" + exit 1 +fi +success "$LIDARR_CONTAINER is running" + +# Check container health — only fail if explicitly unhealthy +CONTAINER_HEALTH=$(docker inspect -f '{{.State.Health.Status}}' "$LIDARR_CONTAINER" 2>/dev/null) +case "$CONTAINER_HEALTH" in + healthy) + success "$LIDARR_CONTAINER is healthy" ;; + "") + info "$LIDARR_CONTAINER has no health check configured — proceeding" ;; + starting) + error "$LIDARR_CONTAINER is still starting — aborting" + notify "Lidarr cleanup aborted on $(hostname) — Lidarr container still starting" "Lidarr Cleanup" "warning" + exit 1 ;; + unhealthy) + error "$LIDARR_CONTAINER is unhealthy — aborting" + notify "Lidarr cleanup aborted on $(hostname) — Lidarr container unhealthy" "Lidarr Cleanup" "warning" + exit 1 ;; + *) + warn "$LIDARR_CONTAINER health status: $CONTAINER_HEALTH — proceeding with caution" ;; +esac + +success "All safety checks passed" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted" +[[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing flag active" # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SUMMARY Status ━━━ @@ -94,6 +154,8 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "$ICON_GEAR Lidarr URL: $LIDARR_URL" echo "$ICON_GEAR Music root: $LIDARR_MUSIC_ROOT" echo "$ICON_TIME Orphan age: ${LIDARR_ORPHAN_AGE} days" + echo "$ICON_GEAR Max delete: ${LIDARR_MAX_DELETE_GB}GB (requires --i-know-what-im-doing)" + echo "$ICON_GEAR Min tracked %: ${LIDARR_MIN_TRACKED_PCT}% of last run" echo "$ICON_GEAR Extensions: ${LIDARR_EXTENSIONS[*]}" echo "$ICON_GEAR Protected patterns: ${LIDARR_PROTECTED_PATTERNS[*]}" echo "$ICON_GEAR Dry Run: $DRY_RUN" @@ -101,13 +163,10 @@ if [[ "$SHOW_STATUS" == true ]]; then exit 0 fi -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted" - # ----------------------------------------------------------------------------------------------- # HELPERS # ----------------------------------------------------------------------------------------------- -# Makes a GET request to the Lidarr API — exits on connection failure lidarr_api() { local endpoint="$1" local response http_code body @@ -129,7 +188,6 @@ lidarr_api() { echo "$body" } -# Returns 0 if file extension matches LIDARR_EXTENSIONS list is_music_file() { local ext="${1##*.}" ext="${ext,,}" @@ -139,7 +197,6 @@ is_music_file() { return 1 } -# Returns 0 if filename matches any pattern in LIDARR_PROTECTED_PATTERNS is_protected_file() { local filename filename=$(basename "$1") @@ -158,6 +215,7 @@ is_protected_file() { echo "" echo "━━━ $ICON_SYNC Fetching Lidarr Tracked Files ━━━" +# Safety Layer 2 — API reachability check_api "$LIDARR_URL" "Lidarr" || { notify "Lidarr cleanup aborted on $(hostname) — API unreachable" "Lidarr Cleanup" "warning" exit 1 @@ -165,9 +223,7 @@ check_api "$LIDARR_URL" "Lidarr" || { info "Querying Lidarr API: $LIDARR_URL" -# Lidarr requires artistId parameter for trackFile endpoint -# Step 1: Get all artist IDs -# Step 2: For each artist, get their track files +# Step 1 — Get all artists ARTIST_RESPONSE=$(lidarr_api "artist") || { error "Failed to fetch artists from Lidarr — check URL and API key" notify "Lidarr cleanup failed on $(hostname) — could not fetch artists" "Lidarr Cleanup" "warning" @@ -177,9 +233,11 @@ ARTIST_RESPONSE=$(lidarr_api "artist") || { ARTIST_IDS=$(echo "$ARTIST_RESPONSE" | jq -r '.[].id' 2>/dev/null) ARTIST_COUNT=$(echo "$ARTIST_IDS" | grep -c "[0-9]" 2>/dev/null || echo 0) +# Safety Layer 3 — artist count if [[ "$ARTIST_COUNT" -eq 0 ]]; then - warn "No artists found in Lidarr — library may be empty" - exit 0 + error "No artists returned from Lidarr API — aborting to prevent mass deletion" + notify "Lidarr cleanup aborted on $(hostname) — API returned 0 artists" "Lidarr Cleanup" "warning" + exit 1 fi info "Found $ARTIST_COUNT artists — fetching track files..." @@ -200,17 +258,38 @@ while IFS= read -r artist_id; do done <<< "$ARTIST_IDS" sort -u "$TRACKED_FILE" -o "$TRACKED_FILE" - TRACKED_COUNT=$(wc -l < "$TRACKED_FILE") -success "Lidarr tracks $TRACKED_COUNT files across $ARTIST_COUNT artists" +# Safety Layer 4 — tracked file count if [[ "$TRACKED_COUNT" -eq 0 ]]; then - warn "No tracked files returned — Lidarr may not have scanned yet or library is empty" - warn "Aborting to prevent mass deletion" - notify "Lidarr cleanup aborted on $(hostname) — no tracked files returned from API" "Lidarr Cleanup" "warning" + error "API returned 0 tracked files — aborting to prevent mass deletion" + notify "Lidarr cleanup aborted on $(hostname) — API returned 0 tracked files" "Lidarr Cleanup" "warning" exit 1 fi +success "Lidarr tracks $TRACKED_COUNT files across $ARTIST_COUNT artists" + +# Safety Layer 5 — percentage drop check against last known count +if [[ -f "$LIDARR_TRACKED_COUNT_FILE" ]]; then + LAST_COUNT=$(cat "$LIDARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0) + if [[ "$LAST_COUNT" -gt 0 ]]; then + PCT=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $LAST_COUNT) * 100}") + if [[ "$PCT" -lt "$LIDARR_MIN_TRACKED_PCT" ]]; then + error "Tracked file count dropped to ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT)" + error "This suggests an API issue — aborting to prevent mass deletion" + error "If this is expected (large library removal), delete: $LIDARR_TRACKED_COUNT_FILE" + notify "Lidarr cleanup aborted on $(hostname) — tracked count dropped to ${PCT}% of last run" "Lidarr Cleanup" "warning" + exit 1 + fi + info "Tracked count check: ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT) ✅" + fi +else + info "No previous count on record — first run, saving baseline" +fi + +# Save current count for next run comparison +echo "$TRACKED_COUNT" > "$LIDARR_TRACKED_COUNT_FILE" + # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_CLEAN Scanning Music Root ━━━ # ----------------------------------------------------------------------------------------------- @@ -231,17 +310,16 @@ JUNK_BYTES=0 AGE_SECONDS=$(( LIDARR_ORPHAN_AGE * 86400 )) NOW=$(date +%s) +MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $LIDARR_MAX_DELETE_GB * 1073741824}") while IFS= read -r filepath; do [[ -z "$filepath" ]] && continue - # TRACKED — Lidarr knows about this file if grep -qF "$filepath" "$TRACKED_FILE" 2>/dev/null; then log "TRACKED: $filepath" continue fi - # PROTECTED — never delete regardless of tracking status if is_protected_file "$filepath"; then log "PROTECTED: $filepath" ((PROTECTED_COUNT++)) @@ -251,7 +329,6 @@ while IFS= read -r filepath; do FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0) if is_music_file "$filepath"; then - # Music file not tracked — check age FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0) FILE_AGE=$(( NOW - FILE_MTIME )) @@ -261,35 +338,56 @@ while IFS= read -r filepath; do continue fi - # ORPHAN — old enough to delete warn "$ICON_TRASH ORPHAN: $filepath" - if [[ "$DRY_RUN" == false ]]; then - rm -f "$filepath" && { - ((ORPHAN_COUNT++)) - ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE)) - } || error "Failed to delete: $filepath" - else - ((ORPHAN_COUNT++)) - ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE)) - fi + ((ORPHAN_COUNT++)) + ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE)) else - # JUNK — not a music extension, not protected log "JUNK: $filepath" - if [[ "$DRY_RUN" == false ]]; then - rm -f "$filepath" && { - ((JUNK_COUNT++)) - JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE)) - } || error "Failed to delete: $filepath" - else - ((JUNK_COUNT++)) - JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE)) - fi + ((JUNK_COUNT++)) + JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE)) fi done < <(find "$LIDARR_MUSIC_ROOT" -type f 2>/dev/null) -# Empty folder cleanup +TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES )) +TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) + +# ----------------------------------------------------------------------------------------------- +# Safety Layer 6 — deletion size threshold +# ----------------------------------------------------------------------------------------------- +if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then + TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}") + if [[ "$I_KNOW" != true ]]; then + echo "" + error "Deletion would exceed ${LIDARR_MAX_DELETE_GB}GB threshold — $TOTAL_HUMAN would be deleted" + error "Review the ORPHAN lines above carefully before proceeding" + error "If this is expected, rerun with: --i-know-what-im-doing" + notify "Lidarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} deletion requires --i-know-what-im-doing" "Lidarr Cleanup" "warning" + exit 1 + else + warn "OVERRIDE — deletion is ${TOTAL_HUMAN} — proceeding because --i-know-what-im-doing" + fi +fi + +# All safety layers passed — execute deletions if [[ "$DRY_RUN" == false ]]; then + while IFS= read -r filepath; do + [[ -z "$filepath" ]] && continue + + if grep -qF "$filepath" "$TRACKED_FILE" 2>/dev/null; then continue; fi + if is_protected_file "$filepath"; then continue; fi + + FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0) + FILE_AGE=$(( NOW - FILE_MTIME )) + + if is_music_file "$filepath"; then + [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && continue + fi + + rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath" + + done < <(find "$LIDARR_MUSIC_ROOT" -type f 2>/dev/null) + echo "" info "Cleaning up empty folders..." find "$LIDARR_MUSIC_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null @@ -298,7 +396,6 @@ fi END=$(date +%s) -# Format bytes helper format_bytes() { local bytes=$1 if (( bytes > 1073741824 )); then @@ -312,17 +409,16 @@ format_bytes() { ORPHAN_HUMAN=$(format_bytes $ORPHAN_BYTES) JUNK_HUMAN=$(format_bytes $JUNK_BYTES) -TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SUMMARY Summary ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━━━ $ICON_SUMMARY LIDARR CLEANUP SUMMARY ━━━━━" -echo "$ICON_SYNC Tracked by Lidarr: $TRACKED_COUNT files" +echo "$ICON_SYNC Tracked by Lidarr: $TRACKED_COUNT files ($ARTIST_COUNT artists)" echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (cover art, metadata)" -echo "$ICON_TRASH Orphans removed: $ORPHAN_COUNT files ($ORPHAN_HUMAN)" -echo "$ICON_TRASH Junk removed: $JUNK_COUNT files ($JUNK_HUMAN)" +echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)" +echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)" echo "$ICON_TIME Recent skipped: $RECENT_COUNT files (under ${LIDARR_ORPHAN_AGE} days)" echo "$ICON_TIME Duration: $(format_duration $((END - START)))" echo "" @@ -332,7 +428,7 @@ elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then echo "$ICON_DONE Status: $ICON_SUCCESS CLEAN — nothing to remove" notify "Lidarr cleanup complete on $(hostname) — library is clean" "Lidarr Cleanup" "normal" else - echo "$ICON_DONE Status: $ICON_SUCCESS DONE — $TOTAL_REMOVED files removed" + echo "$ICON_DONE Status: $ICON_SUCCESS DONE — $TOTAL_REMOVED files removed (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" notify "Lidarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "normal" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Media/radarr_cleanup.sh b/Media/radarr_cleanup.sh index ac70a6b..46c95cb 100644 --- a/Media/radarr_cleanup.sh +++ b/Media/radarr_cleanup.sh @@ -69,6 +69,18 @@ else RADARR_MOVIES_ROOT="$HOST2_RADARR_MOVIES_ROOT" fi +# Load path map for this host +declare -A PATH_MAP +if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then + for key in "${!HOST1_RADARR_PATH_MAP[@]}"; do + PATH_MAP["$key"]="${HOST1_RADARR_PATH_MAP[$key]}" + done +else + for key in "${!HOST2_RADARR_PATH_MAP[@]}"; do + PATH_MAP["$key"]="${HOST2_RADARR_PATH_MAP[$key]}" + done +fi + info "Radarr instance: $LOCAL_SERVER_NAME → $RADARR_URL" info "Movies root: $RADARR_MOVIES_ROOT" @@ -174,7 +186,11 @@ mkdir -p "$TMP_DIR" trap "rm -rf $TMP_DIR" EXIT TRACKED_FILE="$TMP_DIR/tracked_paths.txt" -echo "$MOVIEFILE_RESPONSE" | jq -r '.[].path' 2>/dev/null | sort > "$TRACKED_FILE" +while IFS= read -r api_path; do + [[ -z "$api_path" ]] && continue + translate_path "$api_path" PATH_MAP >> "$TRACKED_FILE" +done < <(echo "$MOVIEFILE_RESPONSE" | jq -r '.[].path' 2>/dev/null) +sort -u "$TRACKED_FILE" -o "$TRACKED_FILE" TRACKED_COUNT=$(wc -l < "$TRACKED_FILE") success "Radarr tracks $TRACKED_COUNT movie files" @@ -256,12 +272,20 @@ while IFS= read -r filepath; do fi fi -done < <(find "$RADARR_MOVIES_ROOT" -type f 2>/dev/null) +done < <( + # Scan all host paths defined in PATH_MAP — covers all root folders managed by Radarr + for host_path in "${PATH_MAP[@]}" "$RADARR_MOVIES_ROOT"; do + [[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null + done | sort -u +) if [[ "$DRY_RUN" == false ]]; then echo "" info "Cleaning up empty folders..." - find "$RADARR_MOVIES_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null + for host_path in "${PATH_MAP[@]}" "$RADARR_MOVIES_ROOT"; do + [[ -d "$host_path" ]] && \ + find "$host_path" -mindepth 1 -type d -empty -delete 2>/dev/null + done success "Empty folders removed" fi diff --git a/Media/sonarr_cleanup.sh b/Media/sonarr_cleanup.sh index aaed9be..810cee8 100644 --- a/Media/sonarr_cleanup.sh +++ b/Media/sonarr_cleanup.sh @@ -69,6 +69,18 @@ else SONARR_TV_ROOT="$HOST2_SONARR_TV_ROOT" fi +# Load path map for this host +declare -A PATH_MAP +if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then + for key in "${!HOST1_SONARR_PATH_MAP[@]}"; do + PATH_MAP["$key"]="${HOST1_SONARR_PATH_MAP[$key]}" + done +else + for key in "${!HOST2_SONARR_PATH_MAP[@]}"; do + PATH_MAP["$key"]="${HOST2_SONARR_PATH_MAP[$key]}" + done +fi + info "Sonarr instance: $LOCAL_SERVER_NAME → $SONARR_URL" info "TV root: $SONARR_TV_ROOT" @@ -174,7 +186,11 @@ mkdir -p "$TMP_DIR" trap "rm -rf $TMP_DIR" EXIT TRACKED_FILE="$TMP_DIR/tracked_paths.txt" -echo "$EPISODEFILE_RESPONSE" | jq -r '.[].path' 2>/dev/null | sort > "$TRACKED_FILE" +while IFS= read -r api_path; do + [[ -z "$api_path" ]] && continue + translate_path "$api_path" PATH_MAP >> "$TRACKED_FILE" +done < <(echo "$EPISODEFILE_RESPONSE" | jq -r '.[].path' 2>/dev/null) +sort -u "$TRACKED_FILE" -o "$TRACKED_FILE" TRACKED_COUNT=$(wc -l < "$TRACKED_FILE") success "Sonarr tracks $TRACKED_COUNT episode files" @@ -256,12 +272,20 @@ while IFS= read -r filepath; do fi fi -done < <(find "$SONARR_TV_ROOT" -type f 2>/dev/null) +done < <( + # Scan all host paths defined in PATH_MAP — covers all root folders managed by Sonarr + for host_path in "${PATH_MAP[@]}" "$SONARR_TV_ROOT"; do + [[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null + done | sort -u +) if [[ "$DRY_RUN" == false ]]; then echo "" info "Cleaning up empty folders..." - find "$SONARR_TV_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null + for host_path in "${PATH_MAP[@]}" "$SONARR_TV_ROOT"; do + [[ -d "$host_path" ]] && \ + find "$host_path" -mindepth 1 -type d -empty -delete 2>/dev/null + done success "Empty folders removed" fi diff --git a/common.sh b/common.sh index d329e26..6f4d172 100644 --- a/common.sh +++ b/common.sh @@ -720,25 +720,18 @@ acquire_lock() { warn "Stale lock detected for $script_name (PID $existing_pid gone) — clearing" rm -f "$lockfile" else - # Lock is active — check age + # Lock is active — check age (skip warning for continuous scripts) local lock_age lock_age=$(( $(date +%s) - $(stat -c %Y "$lockfile" 2>/dev/null || echo 0) )) - if [[ "$lock_age" -gt "$LOCK_WARN_AGE" ]]; then + if [[ "$lock_age" -gt "$LOCK_WARN_AGE" ]] && [[ "$mode" != "continuous" ]]; then warn "$script_name has been running for ${lock_age}s — may be stuck (PID $existing_pid)" fi if [[ "$mode" == "continuous" ]]; then - # Continuous scripts (watchdogs, failover) — healthy instance = skip gracefully - # Only force-restart if PID is stuck/unresponsive beyond LOCK_WARN_AGE - if [[ "$lock_age" -gt "$LOCK_WARN_AGE" ]]; then - warn "$script_name appears stuck (running ${lock_age}s) — clearing and restarting" - kill "$existing_pid" 2>/dev/null - sleep 2 - rm -f "$lockfile" - else - log "$ICON_SKIP $script_name already running healthy (PID $existing_pid) — skipping" - exit 0 - fi + # Continuous scripts (watchdogs, failover) — healthy instance = always skip + # PID is alive and responding — this is correct behavior, not stuck + log "$ICON_SKIP $script_name already running healthy (PID $existing_pid) — skipping" + exit 0 elif [[ "$mode" == "wait" ]]; then info "Another instance of $script_name is running — waiting up to ${LOCK_WAIT_TIMEOUT}s" local waited=0 @@ -857,6 +850,41 @@ _release_rsync_on_exit() { # Verifies API endpoint is reachable before attempting operations # Usage: check_api "http://localhost:8989" "Sonarr" || exit 1 # ----------------------------------------------------------------------------------------------- +# ----------------------------------------------------------------------------------------------- +# translate_path — translate container path to host path using a PATH_MAP array +# Used by arr cleanup scripts — arrs store paths using container mounts, not host paths +# +# Usage: +# translate_path "/ext-music/Artist/Album/track.flac" HOST1_LIDARR_PATH_MAP +# Returns: /mnt/user/Music-New/Artist/Album/track.flac +# +# If no match found in PATH_MAP — returns path unchanged +# Handles longest-match first to avoid partial path collisions +# ----------------------------------------------------------------------------------------------- +translate_path() { + local api_path="$1" + local -n _path_map="$2" # nameref to the associative array + + local best_match="" + local best_len=0 + + # Find longest matching prefix (handles overlapping container paths) + for container_path in "${!_path_map[@]}"; do + if [[ "$api_path" == "$container_path"* ]]; then + if [[ "${#container_path}" -gt "$best_len" ]]; then + best_match="$container_path" + best_len="${#container_path}" + fi + fi + done + + if [[ -n "$best_match" ]]; then + echo "${_path_map[$best_match]}${api_path#$best_match}" + else + echo "$api_path" # no match — return unchanged + fi +} + check_api() { local url="$1" local service="${2:-API}" diff --git a/unRAID_Essentials/system_watchdog.sh b/unRAID_Essentials/system_watchdog.sh index a423d33..f03721d 100644 --- a/unRAID_Essentials/system_watchdog.sh +++ b/unRAID_Essentials/system_watchdog.sh @@ -1,433 +1,883 @@ #!/bin/bash # ----------------------------------------------------------------------------------------------- -# --------------------------------- System Watchdog -------------------------------------------- +# ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ---------------------------- # ----------------------------------------------------------------------------------------------- -# Last line of defense — reboots the system cleanly if it is about to become unstable. -# Runs continuously as a background process — started by array_start.sh at array start. -# Works alongside docker_watchdog.sh which handles container-level healing first. -# -# Checks (all toggleable in Master.conf): -# rootfs usage — high rootfs fills rapidly when array is down, crash imminent -# /var/log usage — log spam can fill rootfs, indicates something is broken -# free RAM — critically low RAM means OOM or swap imminent -# ZFS ARC pinned — ARC not releasing after reclaim means memory is stuck -# CPU temperature — sustained tjmax causes throttling or kernel panic -# load average — sustained high load means something is stuck or runaway -# zombie processes — large zombie count indicates serious process management failure -# Docker daemon — unresponsive daemon means containers cannot be managed -# Required containers — stopped containers that should be running (after watchdog skip list) -# -# Abort conditions (toggleable): -# ZFS pool unhealthy — reboot with bad pool risks data loss -# Parity running — aborting parity is better than crashing mid-check -# Mover running — aborting move is better than crashing mid-move -# -# Reboot loop protection: -# Tracks reboot timestamps in persistent log on /boot/ -# Rolling window — old entries purge automatically -# If reboot count hits limit in window → shutdown instead of reboot -# -# Continuous loop: -# Checks run every SYSTEM_WATCHDOG_INTERVAL seconds (default 300 = 5min) -# Master.conf re-sourced each cycle — config changes picked up without restart -# Silent when healthy — only verbose when trigger or reboot -# Clean shutdown on SIGTERM/SIGINT — sent by array stop -# -# All configuration in Master.conf under System Watchdog section. -# Supports --dry-run to show triggered conditions without rebooting. +# Version: 2.9 +# ----------------------------------------------------------------------------------------------- +# Changelog: +# v1.0 — Initial stable framework +# v1.1 — format_duration moved here from daily_sync_maintenance.sh for shared use +# SSH_KEY collision resolved — gitea key renamed GITEA_SSH_KEY in Master.conf +# Version and changelog tracking added +# v1.2 — Consistent function header comment blocks across all functions +# check_connectivity added as standalone function +# check_connectivity friendlier error output with tailscale hint +# v1.3 — check_remote_rootfs added — aborts if remote rootfs exceeds ROOTFS_WARN threshold +# check_remote_share added — aborts if target directory is missing or empty on remote +# Both protect against rsync running when remote array is down or drives are missing +# v1.4 — check_remote_disks added — verifies all physical disks backing a share are mounted +# Discovers disk layout automatically at runtime, no configuration required +# Aborts if any single disk backing the share is offline or unmounted +# v1.5 — Full icon set expanded — each operation and state has its own distinct icon +# All function output updated to use correct icon per context +# Icons grouped and commented by category for clarity +# v1.6 — ICON_CONTAINERS added — 📦 anchors all container sections for visual consistency +# ICON_NOT_RUNNING changed to ⭕ — distinct from ICON_STOPPED 🔴 +# Section dividers updated from --- to ━━━ for cleaner log readability +# Summary passed/failed lines use ICON_SUCCESS and ICON_ERROR consistently +# v1.7 — ICON_MOVER added for mover operations +# ICON_CONTAINERS replaces ICON_DOCKER for docker/container operations +# validate_int added — reusable integer validation for any script +# v1.8 — ICON_PHP added for PHP-FPM operations +# v1.9 — ICON_REBOOT added for server reboot operations +# v2.0 — ICON_PLUGIN added for User Scripts plugin operations +# v2.1 — ICON_ZFS and ICON_MEM added for ZFS and memory diagnostics +# Diagnostics icon group added to icon block +# v2.2 — ICON_WATCHDOG added for Docker watchdog monitoring operations +# v2.3 — ICON_NOTIFY added for notification operations +# notify() added — shared notification function supporting unRAID native and Discord +# NOTIFY_UNRAID and DISCORD_WEBHOOK configured in Master.conf +# v2.4 — ICON_CLEAN, ICON_TRASH added for media cleaner operations +# ICON_PERMS, ICON_UNLOCKED added for media permissions operations +# ICON_REBOOT_SMART added for smart conditional reboot +# v2.5 — ICON_RAM added for ramdisk operations +# ICON_LINK added for symlink state and management +# Transcode scripts group added to ecosystem +# v2.6 — ICON_FAILOVER added for failover operations +# check_local_array added — verifies local /mnt/user is mounted and healthy +# check_remote_array added — verifies remote /mnt/user is mounted and healthy +# check_remote_docker added — verifies remote Docker daemon is responding +# ping_remote added — non-fatal ping returning status for failover use +# ping_internet added — non-fatal external ping for failover use +# v2.7 — ICON_WEBGUI added for WebGUI watchdog operations +# ICON_DOCKER_NET added for Docker network connect operations +# v2.8 — ICON_CERT added for SSL certificate monitoring operations +# v2.9 — ICON_MONITOR added for monitoring section headers +# v3.0 — Script locking system added — prevents concurrent execution conflicts +# acquire_lock() — create PID lock file, register EXIT trap +# release_lock() — remove lock file on exit +# acquire_rsync_lock() — per-profile lock + global concurrent limit +# release_rsync_lock() — decrement global counter, remove profile lock +# check_api() — pre-flight API reachability check +# ICON_LOCK added — 🔏 script instance lock acquired/released +# ICON_SMART added for drive SMART health operations +# ICON_BANDWIDTH added for bandwidth tracking operations +# ICON_DIGEST added for health digest operations +# ICON_EMBY added for Emby session reporting +# ICON_VERIFY added for backup verification operations +# Monitor/ folder added to ecosystem # ----------------------------------------------------------------------------------------------- -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# ----------------------------------------------------------------------------------------------- +# ICONS +# Each icon has one job — do not reuse across different contexts. +# Adding a new icon: add it to the appropriate group below with a comment describing its job. +# ----------------------------------------------------------------------------------------------- -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +# System / Host +ICON_HOST="🖥️" # host detection +ICON_NET="🌐" # network / IP resolution +ICON_PING="📡" # connectivity check +ICON_GEAR="⚙️" # setup section header / profile load -parse_args "$@" +# Health Checks +ICON_DISK="💾" # disk checks +ICON_HEALTH="🩺" # rootfs / share health checks +ICON_SHIELD="🛡️" # pre-flight section header + +# Containers +ICON_CONTAINERS="📦" # container section anchor +ICON_STOP="⛔" # stop command being issued +ICON_STOPPED="🔴" # container confirmed stopped +ICON_START="▶️" # start command being issued +ICON_STARTED="💚" # container confirmed started +ICON_RUNNING="🟢" # container already running when checked +ICON_SKIP="⏭️" # skipping — already running healthy instance +ICON_NOT_RUNNING="⭕" # container already stopped when checked + +# Transfer +ICON_SYNC="🔄" # transfer section header +ICON_RUN="🚀" # sync starting / rsync attempt +ICON_RETRY="🔁" # retry attempt +ICON_DONE="🏁" # transfer complete + +# Summary +ICON_SUMMARY="📋" # summary section header +ICON_TIME="⏱️" # duration line + +# System Operations +ICON_MOVER="🔃" # mover operations +ICON_REBOOT="⚡" # scheduled server reboot +ICON_REBOOT_SMART="🚨" # smart conditional reboot triggered +ICON_PLUGIN="🧩" # user scripts plugin operations +ICON_PHP="👥" # PHP-FPM operations +ICON_WEBGUI="💻" # WebGUI / nginx / emhttp operations + +# Diagnostics +ICON_ZFS="📊" # ZFS ARC statistics +ICON_MEM="🧠" # memory status +ICON_WATCHDOG="🐾" # docker watchdog monitoring operations + +# Media Operations +ICON_CLEAN="🧹" # media cleaner operations +ICON_TRASH="🗑️" # files being deleted +ICON_PERMS="🔐" # permissions operation / section header +ICON_UNLOCKED="🔓" # permissions successfully applied to a share +ICON_LOCK="🔏" # script instance lock — acquired/released + +# Transcode Operations +ICON_RAM="💨" # ramdisk operations — fast ephemeral storage +ICON_LINK="🔗" # symlink state and management + +# Failover Operations +ICON_FAILOVER="🔀" # failover state changes and operations + +# Docker Network Operations +ICON_DOCKER_NET="🔌" # Docker network connect operations + +# Security / Certificate Operations +ICON_CERT="🔒" # SSL certificate monitoring + +# Monitor Operations +ICON_MONITOR="📈" # monitoring section headers and general monitoring +ICON_SMART="🔧" # drive SMART health attribute monitoring +ICON_BANDWIDTH="📶" # bandwidth usage tracking and reporting +ICON_DIGEST="📰" # health digest — aggregated system summary +ICON_EMBY="🎬" # Emby media server session reporting +ICON_VERIFY="✔️" # backup verification — checksum comparison + +# Notifications +ICON_NOTIFY="🔔" # notification operations + +# Output +ICON_INFO="ℹ️" +ICON_WARN="⚠️" +ICON_ERROR="❌" +ICON_SUCCESS="✅" # ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup — runs once at start ━━━ +# OUTPUT HELPERS +# Standardised output functions used across all scripts. +# log() is gated by ENABLE_LOGGING — set in Master.conf or via --log flag. # ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" +info() { echo "$ICON_INFO [INFO] $*"; } +warn() { echo "$ICON_WARN [WARN] $*"; } +error() { echo "$ICON_ERROR [ERROR] $*"; } +success() { echo "$ICON_SUCCESS [OK] $*"; } -if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root" - exit 1 -fi - -success "Running as root" - -acquire_lock "continuous" - -TOTAL_CORES=$(nproc) - -# Ensure state and persistent files exist -touch "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || { - error "Cannot create state file: $SYS_WATCHDOG_STATE_FILE" - exit 1 -} - -touch "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || { - error "Cannot create reboot log: $SYS_WATCHDOG_REBOOT_LOG" - exit 1 -} - -touch "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || { - error "Cannot create failed container list: $SYS_WATCHDOG_FAILED_FILE" - exit 1 +log() { + [[ "${ENABLE_LOGGING:-false}" == true ]] && echo "[LOG] $*" } # ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ +# NOTIFICATION +# Sends a notification via unRAID native system and/or Discord webhook. +# Both channels are optional and independently controlled via Master.conf. +# Severity levels: normal, warning, alert +# Usage: notify "message" "subject" "severity" # ----------------------------------------------------------------------------------------------- -if [[ "$SHOW_STATUS" == true ]]; then - echo "" - echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_HEALTH rootfs threshold: ${SYS_WATCHDOG_ROOTFS_PCT}% toggle: $SYS_WATCHDOG_CHECK_ROOTFS" - echo "$ICON_HEALTH log threshold: ${SYS_WATCHDOG_LOG_PCT}% toggle: $SYS_WATCHDOG_CHECK_LOG" - echo "$ICON_MEM RAM threshold: ${SYS_WATCHDOG_MEM_GB}GB free toggle: $SYS_WATCHDOG_CHECK_RAM" - echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}% toggle: $SYS_WATCHDOG_CHECK_ARC" - echo "$ICON_ZFS ARC release: ${SYS_WATCHDOG_ARC_RELEASE_PCT}%" - echo "$ICON_GEAR CPU temp max: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C toggle: $SYS_WATCHDOG_CHECK_CPU_TEMP" - echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x cores toggle: $SYS_WATCHDOG_CHECK_LOAD" - echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT} toggle: $SYS_WATCHDOG_CHECK_ZOMBIES" - echo "$ICON_CONTAINERS Docker daemon: toggle: $SYS_WATCHDOG_CHECK_DOCKER_DAEMON" - echo "$ICON_CONTAINERS Containers: toggle: $SYS_WATCHDOG_CHECK_CONTAINERS" - echo "$ICON_SHIELD Strike limit: $SYS_WATCHDOG_STRIKE_LIMIT" - echo "$ICON_TIME Interval: ${SYSTEM_WATCHDOG_INTERVAL}s" - echo "$ICON_REBOOT_SMART Reboot limit: $SYS_WATCHDOG_REBOOT_LIMIT in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs" - echo "$ICON_ZFS Abort ZFS unhealthy: $SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" - echo "$ICON_GEAR Abort on parity: $SYS_WATCHDOG_ABORT_ON_PARITY" - echo "$ICON_MOVER Abort on mover: $SYS_WATCHDOG_ABORT_ON_MOVER" - echo "$ICON_GEAR Dry Run: $DRY_RUN" - echo "━━━━━━━━━━━━━━━━━━━━━━━" - exit 0 -fi +notify() { + local message="$1" + local subject="${2:-unRAID Notification}" + local severity="${3:-normal}" -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboot will be executed" + log "$ICON_NOTIFY Sending notification: $subject — $message" -# ----------------------------------------------------------------------------------------------- -# STATE HELPERS — defined once, used every cycle -# ----------------------------------------------------------------------------------------------- - -get_strikes() { - local key="$1" - grep -E "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2 -} - -set_strikes() { - local key="$1" count="$2" - grep -vE "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp" - echo "${key}:${count}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp" - mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE" -} - -increment_strikes() { - local key="$1" - local current - current=$(get_strikes "$key") - [[ -z "$current" ]] && current=0 - ((current++)) - set_strikes "$key" "$current" - echo "$current" -} - -reset_strikes() { - local key="$1" - set_strikes "$key" 0 -} - -purge_old_reboots() { - local now - now=$(date +%s) - local cutoff=$(( now - SYS_WATCHDOG_REBOOT_WINDOW )) - grep -v "^$" "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | while IFS= read -r ts; do - [[ "$ts" -gt "$cutoff" ]] && echo "$ts" - done > "${SYS_WATCHDOG_REBOOT_LOG}.tmp" - mv "${SYS_WATCHDOG_REBOOT_LOG}.tmp" "$SYS_WATCHDOG_REBOOT_LOG" -} - -count_recent_reboots() { - purge_old_reboots - grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0 -} - -log_reboot() { - date +%s >> "$SYS_WATCHDOG_REBOOT_LOG" -} - -check_abort_conditions() { - local should_abort=false - - if command -v zpool >/dev/null 2>&1; then - local unhealthy - unhealthy=$(zpool list -H -o health 2>/dev/null | grep -v ONLINE || true) - if [[ -n "$unhealthy" ]]; then - if [[ "$SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" == true ]]; then - error "$ICON_ZFS ZFS pool unhealthy — aborting reboot" - notify "System watchdog aborted reboot on $(hostname) — ZFS pool unhealthy" "System Watchdog" "warning" - should_abort=true - else - warn "$ICON_ZFS ZFS pool unhealthy — continuing reboot" - fi - fi - fi - - if [[ -f /var/local/emhttp/parity-date.txt ]]; then - if grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null; then - if [[ "$SYS_WATCHDOG_ABORT_ON_PARITY" == true ]]; then - error "$ICON_DISK Parity check running — aborting reboot" - notify "System watchdog aborted reboot on $(hostname) — parity running" "System Watchdog" "warning" - should_abort=true - else - warn "$ICON_DISK Parity check running — continuing reboot" - fi - fi - fi - - if pgrep -f "mover" >/dev/null 2>&1; then - if [[ "$SYS_WATCHDOG_ABORT_ON_MOVER" == true ]]; then - error "$ICON_MOVER Mover running — aborting reboot" - notify "System watchdog aborted reboot on $(hostname) — mover running" "System Watchdog" "warning" - should_abort=true + if [[ "${NOTIFY_UNRAID:-false}" == true ]]; then + local notify_script="/usr/local/emhttp/plugins/dynamix/scripts/notify" + if [[ -x "$notify_script" ]]; then + "$notify_script" -s "$subject" -d "$message" -i "$severity" 2>/dev/null + log "$ICON_NOTIFY unRAID notification sent" else - warn "$ICON_MOVER Mover running — continuing reboot" + log "$ICON_NOTIFY unRAID notify script not found — skipping" fi fi - [[ "$should_abort" == true ]] && return 1 + if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then + local payload + payload=$(printf '{"content": "%s — **%s**\\n%s"}' \ + "$ICON_NOTIFY" "$subject" "$message") + if curl -s -H "Content-Type: application/json" \ + -d "$payload" "$DISCORD_WEBHOOK" >/dev/null 2>&1; then + log "$ICON_NOTIFY Discord notification sent" + else + warn "Discord notification failed — check DISCORD_WEBHOOK in Master.conf" + fi + fi +} + +# ----------------------------------------------------------------------------------------------- +# DURATION FORMATTER +# Converts raw seconds into a human readable string — e.g. 10m53s or 47s +# ----------------------------------------------------------------------------------------------- +format_duration() { + local secs=$1 + local mins=$((secs / 60)) + local rem=$((secs % 60)) + [[ $mins -gt 0 ]] && echo "${mins}m${rem}s" || echo "${rem}s" +} + +# ----------------------------------------------------------------------------------------------- +# ARG PARSER +# Processes all flags and key=value pairs passed to any script. +# Supported flags: --dry-run, --log, --no-log, --status, --help +# Supported key=value: LOG=true/false, or any declared variable e.g. BW_LIMIT=5000 +# Unparsed positional args returned in PARSED_ARGS array. +# ----------------------------------------------------------------------------------------------- +parse_args() { + ENABLE_LOGGING=${ENABLE_LOGGING:-false} + DRY_RUN=${DRY_RUN:-false} + SHOW_STATUS=${SHOW_STATUS:-false} + CLEAN_ARGS=() + + for ARG in "$@"; do + if [[ "$ARG" == *=* ]]; then + VAR="${ARG%%=*}" + VAL="${ARG#*=}" + case "$VAR" in + LOG) + [[ "$VAL" == "true" ]] && ENABLE_LOGGING=true + [[ "$VAL" == "false" ]] && ENABLE_LOGGING=false + ;; + *) + if declare -p "$VAR" &>/dev/null; then + printf -v "$VAR" '%s' "$VAL" + log "Set $VAR=$VAL" + else + warn "Unknown variable: $VAR" + fi + ;; + esac + else + case "$ARG" in + --dry-run|-n) DRY_RUN=true ;; + --log) ENABLE_LOGGING=true ;; + --no-log) ENABLE_LOGGING=false ;; + --status|--summary) SHOW_STATUS=true ;; + --help|-h) + echo "Usage: script [--dry-run] [--log] [--status]" + exit 0 + ;; + *) CLEAN_ARGS+=("$ARG") ;; + esac + fi + done + + PARSED_ARGS=("${CLEAN_ARGS[@]}") +} + +# ----------------------------------------------------------------------------------------------- +# VALIDATION HELPERS +# ----------------------------------------------------------------------------------------------- + +# Exits with error if a required variable is empty or unset. +# Usage: require_var VAR_NAME +require_var() { + [[ -z "${!1:-}" ]] && error "Missing required: $1" && exit 1 +} + +# Exits with error if a variable is not a valid positive integer. +# Usage: validate_int VAR_NAME "$VAR_VALUE" +validate_int() { + local name="$1" value="$2" + if [[ -z "$value" ]]; then + error "$name is not set — check Master.conf" + exit 1 + fi + if ! [[ "$value" =~ ^[0-9]+$ ]]; then + error "$name must be a positive integer — got: '$value'" + exit 1 + fi + log "$name validated: $value" +} + +# ----------------------------------------------------------------------------------------------- +# HOST DETECTION +# Determines which server is local and which is remote by comparing hostname against +# HOST1 and HOST2 in Master.conf. Sets LOCAL_SERVER_NAME, REMOTE_SERVER_NAME and SSH_KEY. +# Both servers run identical scripts — this is what makes them bidirectional. +# ----------------------------------------------------------------------------------------------- +detect_hosts() { + LOCAL_HOSTNAME="$(hostname)" + + if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then + LOCAL_SERVER_NAME="$HOST1" + REMOTE_SERVER_NAME="$HOST2" + elif [[ "$LOCAL_HOSTNAME" == "$HOST2" ]]; then + LOCAL_SERVER_NAME="$HOST2" + REMOTE_SERVER_NAME="$HOST1" + else + error "Unknown host: $LOCAL_HOSTNAME" + exit 1 + fi + + declare -A SSH_KEYS + SSH_KEYS["$HOST1|$HOST2"]="$HOST1_SSH_KEY" + SSH_KEYS["$HOST2|$HOST1"]="$HOST2_SSH_KEY" + SSH_KEY="${SSH_KEYS[$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME]}" + + [[ -z "$SSH_KEY" ]] && error "Missing SSH key mapping" && exit 1 + info "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" +} + +# ----------------------------------------------------------------------------------------------- +# REMOTE IP RESOLUTION +# Resolves the Tailscale IPv4 address of the remote server. +# Sets REMOTE_SERVER used by all subsequent SSH and rsync calls. +# Exits if resolution fails — Tailscale may be down or peer offline. +# ----------------------------------------------------------------------------------------------- +resolve_remote_ip() { + log "Resolving remote IP for $REMOTE_SERVER_NAME..." + REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) + [[ -z "$REMOTE_SERVER" ]] && error "Failed to resolve Tailscale IP for $REMOTE_SERVER_NAME" && exit 1 + info "$ICON_NET Remote IP: $REMOTE_SERVER" +} + +# ----------------------------------------------------------------------------------------------- +# CONNECTIVITY CHECK — fatal, used by rsync scripts +# Pings remote and exits if unreachable. +# For failover use ping_remote() which returns status without exiting. +# ----------------------------------------------------------------------------------------------- +check_connectivity() { + log "Checking connectivity to $REMOTE_SERVER..." + if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then + error "$ICON_PING Remote $REMOTE_SERVER ($REMOTE_SERVER_NAME) is unreachable" + info "Hint: tailscale status | grep $REMOTE_SERVER_NAME" + exit 1 + fi + info "$ICON_PING $REMOTE_SERVER_NAME is reachable" +} + +# ----------------------------------------------------------------------------------------------- +# PING REMOTE — non-fatal, used by failover +# Returns 0 if reachable, 1 if not — does NOT exit. +# ----------------------------------------------------------------------------------------------- +ping_remote() { + ping -c2 -W3 "$REMOTE_SERVER" &>/dev/null +} + +# ----------------------------------------------------------------------------------------------- +# PING INTERNET — non-fatal external connectivity check +# Returns 0 if internet reachable, 1 if not — does NOT exit. +# ----------------------------------------------------------------------------------------------- +ping_internet() { + ping -c2 -W3 "${EXTERNAL_IP:-8.8.8.8}" &>/dev/null +} + +# ----------------------------------------------------------------------------------------------- +# LOCAL ARRAY CHECK — non-fatal, returns status +# Verifies local /mnt/user is mounted and has shares. +# Used by failover before starting remote containers locally. +# Returns 0 if healthy, 1 if not. +# ----------------------------------------------------------------------------------------------- +check_local_array() { + log "Checking local array..." + if ! mountpoint -q /mnt/user 2>/dev/null; then + error "$ICON_DISK Local array is not started — /mnt/user is not mounted" + return 1 + fi + local file_count + file_count=$(ls /mnt/user 2>/dev/null | wc -l) + if [[ "$file_count" -eq 0 ]]; then + error "$ICON_DISK Local array appears empty — shares may not be available" + return 1 + fi + log "Local array is healthy" return 0 } -run_strike_check() { - local key="$1" triggered="$2" description="$3" - if [[ "$triggered" == true ]]; then - local strikes - strikes=$(increment_strikes "$key") - warn "$description — strike $strikes/$SYS_WATCHDOG_STRIKE_LIMIT" - if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then - error "$description hit strike limit — reboot triggered" - reset_strikes "$key" - return 0 - fi - else - local current - current=$(get_strikes "$key") - if [[ -n "$current" && "$current" -gt 0 ]]; then - reset_strikes "$key" - fi +# ----------------------------------------------------------------------------------------------- +# REMOTE ARRAY CHECK — non-fatal, returns status +# Verifies remote /mnt/user is mounted via SSH. +# Used before handback rsync — syncing to remote with no array fills rootfs. +# Returns 0 if healthy, 1 if not. +# ----------------------------------------------------------------------------------------------- +check_remote_array() { + log "Checking remote array on $REMOTE_SERVER_NAME..." + local result + result=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ + "mountpoint -q /mnt/user && echo yes || echo no" 2>/dev/null) + if [[ "$result" != "yes" ]]; then + error "$ICON_DISK Remote array not started on $REMOTE_SERVER_NAME" + return 1 fi - return 1 + log "Remote array is healthy" + return 0 } -do_reboot() { - local triggers=("$@") +# ----------------------------------------------------------------------------------------------- +# REMOTE DOCKER CHECK — non-fatal, returns status +# Verifies remote Docker daemon is responding before container operations. +# A hung daemon means start/stop commands will silently fail. +# Returns 0 if healthy, 1 if not. +# ----------------------------------------------------------------------------------------------- +check_remote_docker() { + log "Checking remote Docker daemon on $REMOTE_SERVER_NAME..." + if ! ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ + "timeout 10 docker ps" >/dev/null 2>&1; then + error "$ICON_CONTAINERS Remote Docker daemon not responding on $REMOTE_SERVER_NAME" + return 1 + fi + log "Remote Docker daemon is healthy" + return 0 +} - if ! check_abort_conditions; then - return +# ----------------------------------------------------------------------------------------------- +# REMOTE ROOTFS SPACE CHECK — fatal +# Aborts if remote rootfs exceeds ROOTFS_WARN threshold. +# ----------------------------------------------------------------------------------------------- +check_remote_rootfs() { + log "Checking remote rootfs usage..." + REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null) + if [[ -z "$REMOTE_USAGE" ]]; then + error "Could not retrieve rootfs usage from $REMOTE_SERVER_NAME" + exit 1 + fi + if [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then + error "$ICON_HEALTH Remote rootfs ${REMOTE_USAGE}% — threshold ${ROOTFS_WARN:-75}%" + exit 1 + fi + info "$ICON_HEALTH Remote rootfs: ${REMOTE_USAGE}% (threshold: ${ROOTFS_WARN:-75}%)" +} + +# ----------------------------------------------------------------------------------------------- +# REMOTE SHARE VALIDATION — fatal +# Verifies target directory exists and is not empty on remote. +# Usage: check_remote_share "/mnt/user/Movies" +# ----------------------------------------------------------------------------------------------- +check_remote_share() { + local dir="$1" + log "Checking remote share: $dir..." + + SHARE_EXISTS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null) + if [[ "$SHARE_EXISTS" != "yes" ]]; then + error "$ICON_HEALTH Remote share does not exist: $dir" + exit 1 fi - RECENT_REBOOTS=$(count_recent_reboots) - info "Recent reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr window: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT" + SHARE_EMPTY=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null) + if [[ "$SHARE_EMPTY" == "yes" ]]; then + warn "$ICON_HEALTH Remote share exists but is empty: $dir — aborting to protect data" + exit 1 + fi - if [[ "$RECENT_REBOOTS" -ge "$SYS_WATCHDOG_REBOOT_LIMIT" ]]; then - error "Reboot limit hit — shutting down instead" - notify "Reboot loop detected on $(hostname) — shutting down after $RECENT_REBOOTS reboots — conditions: ${triggers[*]}" "System Watchdog" "warning" - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would shutdown now" - return + info "$ICON_HEALTH Remote share verified: $dir" +} + +# ----------------------------------------------------------------------------------------------- +# REMOTE DISK CHECK — fatal +# Verifies all physical disks backing a share are online on the remote server. +# Skipped when PROFILE_SKIP_DISK_CHECK is true (ZFS pools have no /mnt/disk* structure). +# Usage: check_remote_disks "/mnt/user/Movies" +# ----------------------------------------------------------------------------------------------- +check_remote_disks() { + local dir="$1" + local share_name + share_name=$(basename "$dir") + + info "$ICON_DISK Checking disks backing $share_name on $REMOTE_SERVER_NAME..." + + DISK_PATHS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "ls -d /mnt/disk*/$share_name 2>/dev/null" 2>/dev/null) + + if [[ -z "$DISK_PATHS" ]]; then + error "$ICON_DISK No disks found backing $share_name on $REMOTE_SERVER_NAME" + exit 1 + fi + + local all_ok=true + while IFS= read -r disk_share_path; do + local disk_mount disk_name + disk_mount=$(dirname "$disk_share_path") + disk_name=$(basename "$disk_mount") + MOUNTED=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "mountpoint -q '$disk_mount' && echo yes || echo no" 2>/dev/null) + if [[ "$MOUNTED" == "yes" ]]; then + info "$ICON_DISK $disk_name $ICON_RUNNING — $share_name present" + else + error "$ICON_DISK $disk_name $ICON_STOPPED — $share_name missing" + all_ok=false fi - sync - /sbin/poweroff + done <<< "$DISK_PATHS" + + if [[ "$all_ok" == false ]]; then + error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME" + exit 1 + fi + success "All disks backing $share_name are online" +} + +# ----------------------------------------------------------------------------------------------- +# CONTAINER MANAGEMENT — STOP (remote via SSH) +# Stops containers in CRITICAL_CONTAINER_NAMES on remote server. +# Tracks running containers in RUNNING_CONTAINERS for restart after rsync. +# ----------------------------------------------------------------------------------------------- +RUNNING_CONTAINERS=() + +stop_containers() { + if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \ + [[ "${CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then + log "No remote containers configured for this profile, skipping stop." return fi + info "Stopping remote containers..." + RUNNING_CONTAINERS=() + for c in "${CRITICAL_CONTAINER_NAMES[@]}"; do + [[ -z "$c" ]] && continue + STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "docker inspect -f '{{.State.Running}}' $c 2>/dev/null" 2>/dev/null || echo "unknown") + if [[ "$STATUS" == "true" ]]; then + echo "$ICON_STOP Stopping $c..." + RUNNING_CONTAINERS+=("$c") + if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $c" >/dev/null; then + echo "$ICON_STOPPED $c stopped" + else + error "Failed to stop $c" + fi + elif [[ "$STATUS" == "false" ]]; then + echo "$ICON_NOT_RUNNING $c is not running, skipping" + else + log "$c not found on remote — skipping" + fi + done +} - notify "System watchdog reboot triggered on $(hostname) — conditions: ${triggers[*]}" "System Watchdog" "warning" - - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — reboot sequence would begin now" +# ----------------------------------------------------------------------------------------------- +# CONTAINER MANAGEMENT — STOP LOCAL +# Stops containers on the LOCAL server before rsync pushes data out. +# Uses PROFILE_LOCAL_CRITICAL_CONTAINER_NAMES — same naming scheme as remote. +# If container not found on this server → skipped gracefully, not errored. +# Only containers that were running get tracked for restart. +# ----------------------------------------------------------------------------------------------- +stop_local_containers() { + if [[ ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \ + [[ "${LOCAL_CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then + log "No local containers configured for this profile, skipping local stop." return fi + info "Stopping local containers..." + LOCAL_RUNNING_CONTAINERS=() + for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]}"; do + [[ -z "$c" ]] && continue + STATUS=$(docker inspect -f '{{.State.Running}}' "$c" 2>/dev/null || echo "unknown") + if [[ "$STATUS" == "true" ]]; then + echo "$ICON_STOP Stopping local $c..." + LOCAL_RUNNING_CONTAINERS+=("$c") + if docker stop "$c" >/dev/null; then + echo "$ICON_STOPPED $c stopped" + else + error "Failed to stop local $c" + fi + elif [[ "$STATUS" == "false" ]]; then + echo "$ICON_NOT_RUNNING $c is not running, skipping" + else + log "$c not found locally — skipping" + fi + done +} - log_reboot - - info "Shutting down VMs..." - if command -v virsh >/dev/null 2>&1; then - for VM in $(virsh list --name 2>/dev/null); do - [[ -z "$VM" ]] && continue - virsh shutdown "$VM" >/dev/null 2>&1 +# ----------------------------------------------------------------------------------------------- +# CONTAINER MANAGEMENT — START (remote via SSH) +# Restarts only containers tracked in RUNNING_CONTAINERS. +# Delayed containers receive CONTAINER_DELAY seconds before starting. +# ----------------------------------------------------------------------------------------------- +start_containers() { + if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then + log "No remote containers to restart." + return + fi + info "Starting remote containers..." + for c in "${RUNNING_CONTAINERS[@]}"; do + [[ -z "$c" ]] && continue + local needs_delay=false + for d in "${DELAYED_CONTAINERS[@]}"; do + [[ "$c" == "$d" ]] && needs_delay=true && break done - sleep 30 - fi - - info "Stopping Docker containers..." - if command -v docker >/dev/null 2>&1; then - docker ps -q | xargs -r docker stop >/dev/null 2>&1 - fi - - info "Stopping User Scripts..." - pkill -f "/tmp/user.scripts" 2>/dev/null || true - - info "Syncing disks..." - sync - - echo "" - echo "$ICON_REBOOT_SMART Rebooting system NOW..." - sleep 5 - /sbin/reboot + if [[ "$needs_delay" == true ]]; then + info "Waiting ${CONTAINER_DELAY}s before starting $c..." + sleep "$CONTAINER_DELAY" + fi + echo "$ICON_START Starting $c..." + if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $c" >/dev/null 2>&1; then + echo "$ICON_STARTED $c started" + else + error "Failed to start $c — start manually if needed" + fi + done } # ----------------------------------------------------------------------------------------------- -# CLEAN SHUTDOWN — trap SIGTERM/SIGINT from array stop +# CONTAINER MANAGEMENT — START LOCAL +# Restarts only containers tracked in LOCAL_RUNNING_CONTAINERS. +# Respects DELAYED_CONTAINERS and CONTAINER_DELAY same as remote start. +# If container not found → skipped gracefully. # ----------------------------------------------------------------------------------------------- -WATCHDOG_RUNNING=true - -cleanup() { - echo "" - info "System watchdog received shutdown signal — stopping cleanly" - WATCHDOG_RUNNING=false - exit 0 -} - -trap cleanup SIGTERM SIGINT - -# ----------------------------------------------------------------------------------------------- -# ━━━ CONTINUOUS MONITORING LOOP ━━━ -# ----------------------------------------------------------------------------------------------- -info "System watchdog started — checking every ${SYSTEM_WATCHDOG_INTERVAL}s" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -CYCLE=0 - -while [[ "$WATCHDOG_RUNNING" == true ]]; do - ((CYCLE++)) - - # Re-source Master.conf each cycle — picks up config changes without restart - source "$SCRIPT_DIR/../Master.conf" - SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 )) - TOTAL_CORES=$(nproc) - - # Per-cycle triggers — cleared each iteration - TRIGGERS=() - - # ── rootfs usage ───────────────────────────────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_ROOTFS" == true ]]; then - ROOTFS_USED=$(df / --output=pcent | tail -1 | tr -d ' %') - TRIGGERED=false - [[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]] && TRIGGERED=true - run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && \ - TRIGGERS+=("rootfs=${ROOTFS_USED}%") +start_local_containers() { + if [[ ${#LOCAL_RUNNING_CONTAINERS[@]} -eq 0 ]]; then + log "No local containers to restart." + return fi - - # ── /var/log usage ──────────────────────────────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_LOG" == true ]]; then - LOG_USED=$(df -P /var/log | awk 'NR==2 {print $5}' | tr -d '%') - TRIGGERED=false - [[ "$LOG_USED" -ge "$SYS_WATCHDOG_LOG_PCT" ]] && TRIGGERED=true - run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && \ - TRIGGERS+=("log=${LOG_USED}%") - fi - - # ── Free RAM ───────────────────────────────────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then - MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) - MEM_GB=$((MEM_KB / 1024 / 1024)) - TRIGGERED=false - [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]] && TRIGGERED=true - run_strike_check "ram" "$TRIGGERED" "RAM ${MEM_GB}GB free" && \ - TRIGGERS+=("low_ram=${MEM_GB}GB") - fi - - # ── ZFS ARC ────────────────────────────────────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]] && [[ -f /proc/spl/kstat/zfs/arcstats ]]; then - ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) - ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats) - ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX )) - TRIGGERED=false - if [[ "$ARC_PCT" -ge "$SYS_WATCHDOG_ARC_PINNED_PCT" ]]; then - sync; echo 3 > /proc/sys/vm/drop_caches; sleep 5 - ARC_AFTER=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) - ARC_AFTER_PCT=$(( ARC_AFTER * 100 / ARC_MAX )) - [[ "$ARC_AFTER_PCT" -ge "$SYS_WATCHDOG_ARC_RELEASE_PCT" ]] && TRIGGERED=true - fi - run_strike_check "arc" "$TRIGGERED" "ZFS ARC pinned" && \ - TRIGGERS+=("arc_pinned=${ARC_PCT}%") - fi - - # ── CPU temperature ────────────────────────────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_CPU_TEMP" == true ]]; then - CPU_TEMP="" - if command -v sensors >/dev/null 2>&1; then - CPU_TEMP=$(sensors 2>/dev/null | grep -i "Package id 0\|Tctl\|CPU Temp" | \ - awk '{print $NF}' | tr -d '+°C' | head -1) - fi - if [[ -n "$CPU_TEMP" ]]; then - CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP") - TRIGGERED=false - [[ "$CPU_TEMP_INT" -ge "$SYS_WATCHDOG_CPU_TEMP_MAX" ]] && TRIGGERED=true - run_strike_check "cpu_temp" "$TRIGGERED" "CPU temp ${CPU_TEMP_INT}°C" && \ - TRIGGERS+=("cpu_temp=${CPU_TEMP_INT}C") - fi - fi - - # ── Load average ───────────────────────────────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_LOAD" == true ]]; then - LOAD=$(awk '{print $1}' /proc/loadavg) - LOAD_INT=$(printf "%.0f" "$LOAD") - LOAD_THRESHOLD=$(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER )) - TRIGGERED=false - [[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]] && TRIGGERED=true - run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && \ - TRIGGERS+=("load=${LOAD}") - fi - - # ── Zombie processes ───────────────────────────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_ZOMBIES" == true ]]; then - ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" || echo 0) - TRIGGERED=false - [[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]] && TRIGGERED=true - run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && \ - TRIGGERS+=("zombies=${ZOMBIE_COUNT}") - fi - - # ── Docker daemon ──────────────────────────────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then - TRIGGERED=false - ! timeout 10 docker ps >/dev/null 2>&1 && TRIGGERED=true - run_strike_check "docker_daemon" "$TRIGGERED" "Docker daemon unresponsive" && \ - TRIGGERS+=("docker_daemon") - fi - - # ── Required containers from skip list ─────────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_CONTAINERS" == true ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then - FAILED_CONTAINERS=() - while IFS= read -r container; do - [[ -z "$container" ]] && continue - STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") - [[ "$STATUS" != "true" ]] && FAILED_CONTAINERS+=("$container") - done < "$SYS_WATCHDOG_FAILED_FILE" - - if [[ ${#FAILED_CONTAINERS[@]} -gt 0 ]]; then - run_strike_check "failed_containers" "true" "required containers stopped" && \ - TRIGGERS+=("containers=${FAILED_CONTAINERS[*]}") - fi - fi - - # ── Evaluate triggers ──────────────────────────────────────────────────────────────────── - if [[ ${#TRIGGERS[@]} -gt 0 ]]; then - echo "" - echo "━━━ $ICON_REBOOT_SMART System Watchdog — Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━" - for t in "${TRIGGERS[@]}"; do - echo " $ICON_REBOOT_SMART $t" + info "Starting local containers..." + for c in "${LOCAL_RUNNING_CONTAINERS[@]}"; do + [[ -z "$c" ]] && continue + local needs_delay=false + for d in "${DELAYED_CONTAINERS[@]}"; do + [[ "$c" == "$d" ]] && needs_delay=true && break done - echo "" - do_reboot "${TRIGGERS[@]}" + if [[ "$needs_delay" == true ]]; then + info "Waiting ${CONTAINER_DELAY}s before starting local $c..." + sleep "$CONTAINER_DELAY" + fi + echo "$ICON_START Starting local $c..." + if docker start "$c" >/dev/null 2>&1; then + echo "$ICON_STARTED $c started" + else + error "Failed to start local $c — start manually if needed" + fi + done +} + +# ----------------------------------------------------------------------------------------------- +# RSYNC OPTIONS +# Loads rsync options for current profile. Falls back to DEFAULT_RSYNC_OPTS if no match. +# Profile opts do NOT inherit from defaults — list all desired flags explicitly. +# ----------------------------------------------------------------------------------------------- +get_rsync_opts() { + if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then + read -r -a RSYNC_OPTS <<< "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]}" + log "Using profile rsync opts for $PROFILE_NAME: ${RSYNC_OPTS[*]}" else - log "Cycle $CYCLE — system healthy ($(date '+%H:%M:%S'))" + RSYNC_OPTS=("${DEFAULT_RSYNC_OPTS[@]}") + log "Using default rsync opts: ${RSYNC_OPTS[*]}" + fi +} + +# ----------------------------------------------------------------------------------------------- +# SCRIPT LOCKING — v3.0 +# Prevents multiple instances of the same script running simultaneously. +# All lock files live in /tmp/unraid_locks/ — auto-cleared on reboot. +# +# Usage in scripts: +# acquire_lock — strict: exit immediately if already running +# acquire_lock "wait" — wait mode: wait briefly then exit if still locked +# acquire_rsync_lock "$profile" — per-profile + global concurrent limit +# +# Stale lock detection — if lock file exists but PID is dead, clears and proceeds. +# Lock age warning — if lock is older than expected, warns but does not override. +# EXIT trap registered automatically — lock always released on exit, crash, or kill. +# ----------------------------------------------------------------------------------------------- + +LOCK_DIR="/tmp/unraid_locks" +RSYNC_COUNT_FILE="$LOCK_DIR/rsync_active_count" +RSYNC_MAX_CONCURRENT=3 +LOCK_WARN_AGE=300 # seconds — warn if lock older than this (5min default) +LOCK_WAIT_TIMEOUT=30 # seconds — how long "wait" mode waits before giving up + +# Internal — script name used as lock identifier +_lock_name() { + basename "${BASH_SOURCE[1]:-$0}" .sh +} + +# Internal — lock file path for this script +_lock_file() { + echo "$LOCK_DIR/${1:-$(_lock_name)}.lock" +} + +# Internal — release lock on exit +_release_on_exit() { + local lockfile="$1" + [[ -f "$lockfile" ]] && rm -f "$lockfile" +} + +# ----------------------------------------------------------------------------------------------- +# acquire_lock — acquire exclusive lock for this script +# Mode: strict (default) — exit immediately if locked +# wait — wait LOCK_WAIT_TIMEOUT seconds then exit +# continuous — for long-running scripts: skip gracefully if healthy, +# clear and restart if dead/stuck +# Stale lock: if PID in lock file is dead → clear and acquire +# Age warning: if lock older than LOCK_WARN_AGE → warn +# ----------------------------------------------------------------------------------------------- +acquire_lock() { + local mode="${1:-strict}" + local script_name + script_name=$(basename "${BASH_SOURCE[1]:-$0}" .sh) + local lockfile + lockfile="$(_lock_file "$script_name")" + + mkdir -p "$LOCK_DIR" + + # Check for existing lock + if [[ -f "$lockfile" ]]; then + local existing_pid + existing_pid=$(cat "$lockfile" 2>/dev/null) + + # Stale lock detection — PID no longer running + if [[ -n "$existing_pid" ]] && ! kill -0 "$existing_pid" 2>/dev/null; then + warn "Stale lock detected for $script_name (PID $existing_pid gone) — clearing" + rm -f "$lockfile" + else + # Lock is active — check age (skip warning for continuous scripts) + local lock_age + lock_age=$(( $(date +%s) - $(stat -c %Y "$lockfile" 2>/dev/null || echo 0) )) + if [[ "$lock_age" -gt "$LOCK_WARN_AGE" ]] && [[ "$mode" != "continuous" ]]; then + warn "$script_name has been running for ${lock_age}s — may be stuck (PID $existing_pid)" + fi + + if [[ "$mode" == "continuous" ]]; then + # Continuous scripts (watchdogs, failover) — healthy instance = always skip + # PID is alive and responding — this is correct behavior, not stuck + log "$ICON_SKIP $script_name already running healthy (PID $existing_pid) — skipping" + exit 0 + elif [[ "$mode" == "wait" ]]; then + info "Another instance of $script_name is running — waiting up to ${LOCK_WAIT_TIMEOUT}s" + local waited=0 + while [[ -f "$lockfile" ]] && [[ "$waited" -lt "$LOCK_WAIT_TIMEOUT" ]]; do + sleep 1 + ((waited++)) + # Re-check for stale + existing_pid=$(cat "$lockfile" 2>/dev/null) + if [[ -n "$existing_pid" ]] && ! kill -0 "$existing_pid" 2>/dev/null; then + warn "Lock became stale while waiting — clearing" + rm -f "$lockfile" + break + fi + done + if [[ -f "$lockfile" ]]; then + error "$script_name still locked after ${LOCK_WAIT_TIMEOUT}s — exiting" + exit 1 + fi + else + error "Another instance of $script_name is already running (PID $existing_pid) — exiting" + # Notify for critical scripts that should rarely overlap + case "$script_name" in + failover|transcode_management|media_management|daily_sync_maintenance|system_watchdog) + notify "$script_name lock collision on $(hostname) — concurrent instance detected" "$script_name" "warning" + ;; + esac + exit 1 + fi + fi fi - # Sleep until next cycle — interruptible by SIGTERM - sleep "$SYSTEM_WATCHDOG_INTERVAL" & - wait $! + # Acquire lock + echo $$ > "$lockfile" -done \ No newline at end of file + # Register EXIT trap to always release lock + trap "_release_on_exit '$lockfile'" EXIT + + log "$ICON_LOCK Lock acquired: $script_name (PID $$)" +} + +# ----------------------------------------------------------------------------------------------- +# acquire_rsync_lock — per-profile lock + global concurrent limit +# Prevents same profile running twice and limits total concurrent rsync instances +# ----------------------------------------------------------------------------------------------- +acquire_rsync_lock() { + local profile="$1" + local profile_lock + profile_lock="$(_lock_file "rsync_${profile}")" + + mkdir -p "$LOCK_DIR" + + # Per-profile lock — same profile cannot run twice + if [[ -f "$profile_lock" ]]; then + local existing_pid + existing_pid=$(cat "$profile_lock" 2>/dev/null) + if [[ -n "$existing_pid" ]] && kill -0 "$existing_pid" 2>/dev/null; then + error "rsync profile '$profile' is already running (PID $existing_pid) — exiting" + exit 1 + else + warn "Stale rsync lock for profile '$profile' — clearing" + rm -f "$profile_lock" + fi + fi + + # Global concurrent limit + local current_count=0 + if [[ -f "$RSYNC_COUNT_FILE" ]]; then + current_count=$(cat "$RSYNC_COUNT_FILE" 2>/dev/null || echo 0) + # Validate count — clean up if stale + local actual_count=0 + for lf in "$LOCK_DIR"/rsync_*.lock; do + [[ -f "$lf" ]] || continue + local lpid + lpid=$(cat "$lf" 2>/dev/null) + kill -0 "$lpid" 2>/dev/null && ((actual_count++)) + done + if [[ "$actual_count" -ne "$current_count" ]]; then + log "rsync count corrected: $current_count → $actual_count" + current_count=$actual_count + echo "$current_count" > "$RSYNC_COUNT_FILE" + fi + fi + + if [[ "$current_count" -ge "$RSYNC_MAX_CONCURRENT" ]]; then + error "Maximum concurrent rsync limit ($RSYNC_MAX_CONCURRENT) reached — exiting" + info "Active rsync locks: $(ls "$LOCK_DIR"/rsync_*.lock 2>/dev/null | xargs -I{} basename {} .lock | tr '\n' ' ')" + exit 1 + fi + + # Acquire profile lock and increment counter + echo $$ > "$profile_lock" + echo $(( current_count + 1 )) > "$RSYNC_COUNT_FILE" + + # Register EXIT trap + trap "_release_rsync_on_exit '$profile_lock'" EXIT + + log "$ICON_LOCK rsync lock acquired: profile '$profile' (PID $$, active: $(( current_count + 1 ))/$RSYNC_MAX_CONCURRENT)" +} + +# Internal — release rsync lock on exit +_release_rsync_on_exit() { + local profile_lock="$1" + [[ -f "$profile_lock" ]] && rm -f "$profile_lock" + # Decrement global counter + if [[ -f "$RSYNC_COUNT_FILE" ]]; then + local count + count=$(cat "$RSYNC_COUNT_FILE" 2>/dev/null || echo 1) + count=$(( count - 1 )) + [[ "$count" -lt 0 ]] && count=0 + echo "$count" > "$RSYNC_COUNT_FILE" + fi +} + +# ----------------------------------------------------------------------------------------------- +# check_api — pre-flight API reachability check +# Verifies API endpoint is reachable before attempting operations +# Usage: check_api "http://localhost:8989" "Sonarr" || exit 1 +# ----------------------------------------------------------------------------------------------- +check_api() { + local url="$1" + local service="${2:-API}" + local timeout="${3:-10}" + + if curl -sf --max-time "$timeout" "$url" >/dev/null 2>&1; then + log "$service API reachable: $url" + return 0 + else + error "$service API not reachable: $url" + return 1 + fi +} + +# ----------------------------------------------------------------------------------------------- +# STATUS DISPLAY +# Prints current runtime configuration — triggered by --status flag. +# ----------------------------------------------------------------------------------------------- +show_status() { + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "Local: $LOCAL_SERVER_NAME" + echo "Remote: $REMOTE_SERVER_NAME" + echo "IP: $REMOTE_SERVER" + echo "Profile: ${PROFILE_NAME:-n/a}" + echo "DryRun: $DRY_RUN" + echo "Logging: $ENABLE_LOGGING" + echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]:-n/a}" + echo "Delayed: ${DELAYED_CONTAINERS[*]:-n/a}" + echo "Excludes: ${EXCLUDE_DIRS[*]:-n/a}" + echo "━━━━━━━━━━━━━━━━━━━━━━━" +} \ No newline at end of file