big logic update to rsync and rsync called scripts

This commit is contained in:
2026-04-28 17:04:09 -04:00
parent f7074cfba0
commit 11872b802b
6 changed files with 1718 additions and 50 deletions
+6 -14
View File
@@ -167,7 +167,7 @@ ARRAY_START_SCRIPTS=(
"Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers
"unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop
"Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop
# "Failover/failover.sh" # mutual failover — continuous loop
"Failover/failover.sh" # mutual failover — continuous loop
)
# ━━━ Daily Sync Maintenance ━━━
@@ -255,9 +255,9 @@ MEDIA_MANAGEMENT_JOBS=(
"Media/media_shares_permissions.sh" # apply permissions — runs first
"Media/media_cleaner.sh anime" # remove junk from anime shares
"Media/media_cleaner.sh media" # remove junk from media shares
# "Media/lidarr_cleanup.sh" # remove orphaned music files
# "Media/sonarr_cleanup.sh" # remove orphaned TV files
# "Media/radarr_cleanup.sh" # remove orphaned movie files
"Media/lidarr_cleanup.sh" # remove orphaned music files
"Media/sonarr_cleanup.sh" # remove orphaned TV files
"Media/radarr_cleanup.sh" # remove orphaned movie files
"Docker_Essentials/downloaders_reset.sh" # clear stuck states + purge old history
)
@@ -403,16 +403,8 @@ declare -A PROFILE_EXCLUDE_DIRS=(
[emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root"
)
# Skip per-disk space check for these profiles — appdata syncs go to cache/appdata
# not to array disks, so disk space check is irrelevant and just slows things down
declare -A PROFILE_SKIP_DISK_CHECK=(
[arrs_stack]=true
[critical-data]=true
[gmer4lfe]=true
[important-data]=true
[emby]=true
[emby-failover]=true
)
# Skip per-disk space check is no longer needed — check_remote_disks() auto-detects
# XFS and ZFS filesystem types from disks.ini, no manual configuration required
# ==============================================================================================
# ── FAILOVER ──────────────────────────────────────────────────────────────────────────────────
+30 -5
View File
@@ -17,6 +17,10 @@
# 9. radarr_cleanup.sh — remove orphaned movie files
# 10. docker_daily_restart.sh — restart containers that need daily restart
#
# What triggers weekly_health_digest.sh:
# NOT this script — weekly_health_digest.sh runs on its own schedule (Saturday)
# This script writes no stats — it just syncs and maintains
#
# Configuration in Master.conf:
# DAILY_MAINTENANCE_SCRIPTS — pre/post-sync scripts (git pull, docker restart)
# MEDIA_MANAGEMENT_JOBS — media maintenance jobs run after sync
@@ -144,6 +148,8 @@ echo ""
SHARE_INDEX=0
ABORT_ALL_SYNCS=false
for SHARE in "${ALL_SHARES[@]}"; do
SHARE_INDEX=$((SHARE_INDEX + 1))
SHARE_NAME=$(basename "$SHARE")
@@ -151,14 +157,33 @@ for SHARE in "${ALL_SHARES[@]}"; do
echo "━━━ $ICON_SYNC Share $SHARE_INDEX of $SHARE_COUNT: $SHARE_NAME ━━━"
if bash "$RSYNC_SCRIPT" "$SHARE"; then
SHARE_END=$(date +%s)
SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))")
if [[ "$ABORT_ALL_SYNCS" == true ]]; then
warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)"
FAIL+=("$SHARE_NAME:temp-critical")
echo ""
continue
fi
bash "$RSYNC_SCRIPT" "$SHARE"
RSYNC_EXIT=$?
SHARE_END=$(date +%s)
SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))")
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
PASS+=("$SHARE_NAME")
echo "$ICON_DONE $SHARE_NAME complete"
elif [[ "$RSYNC_EXIT" -eq 1 ]]; then
# Temp warning — skip this profile, continue to next
FAIL+=("$SHARE_NAME:temp-warn")
warn "$SHARE_NAME skipped — drive temps too high"
elif [[ "$RSYNC_EXIT" -eq 2 ]]; then
# Temp critical — abort all remaining syncs
FAIL+=("$SHARE_NAME:temp-critical")
ABORT_ALL_SYNCS=true
error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs"
notify "Daily sync aborted on $(hostname) — drive temps CRITICAL during $SHARE_NAME sync" "Daily Sync" "warning"
else
SHARE_END=$(date +%s)
SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))")
FAIL+=("$SHARE_NAME")
error "$SHARE_NAME failed — continuing to next share"
fi
+15 -9
View File
@@ -80,9 +80,6 @@ read -r -a EXCLUDE_DIRS <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_N
# Local and remote use the same container list — same naming scheme on both servers
LOCAL_CRITICAL_CONTAINER_NAMES=("${CRITICAL_CONTAINER_NAMES[@]}")
# Disk check toggle
SKIP_DISK_CHECK=${PROFILE_SKIP_DISK_CHECK[$PROFILE_NAME]:-false}
[[ "$SHOW_STATUS" == true ]] && show_status && exit 0
# -----------------------------------------------------------------------------------------------
@@ -91,15 +88,24 @@ SKIP_DISK_CHECK=${PROFILE_SKIP_DISK_CHECK[$PROFILE_NAME]:-false}
echo ""
echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━"
# Disk temp check — before touching remote or moving any data
# Returns: 0=OK 1=warn(skip this profile) 2=crit(abort all remaining)
check_local_disk_temps
TEMP_RESULT=$?
if [[ "$TEMP_RESULT" -eq 2 ]]; then
error "Drive temps CRITICAL — aborting sync for all remaining profiles"
exit 2 # caller (daily_sync_maintenance.sh) sees exit 2 → stops all syncs
elif [[ "$TEMP_RESULT" -eq 1 ]]; then
warn "Drive temps too high — skipping profile [$PROFILE_NAME]"
exit 1 # caller sees exit 1 → skips this profile, continues to next
else
success "Drive temps OK — $TEMP_CHECK_RESULT"
fi
check_connectivity
check_remote_rootfs
check_remote_share "$DIRECTORY"
if [[ "$SKIP_DISK_CHECK" == "true" ]]; then
info "$ICON_DISK Disk check skipped for profile [$PROFILE_NAME] — ZFS pool on remote"
else
check_remote_disks "$DIRECTORY"
fi
check_remote_disks "$DIRECTORY"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_STOP $ICON_CONTAINERS Containers ━━━
+260 -20
View File
@@ -469,9 +469,122 @@ check_remote_share() {
}
# -----------------------------------------------------------------------------------------------
# REMOTE DISK CHECK — fatal
# Verifies all physical disks backing a share are online on the remote server.
# Skipped when PROFILE_SKIP_DISK_CHECK is true (ZFS pools have no /mnt/disk* structure).
# get_unraid_temp_thresholds — read disk temp thresholds from unRAID's dynamix.cfg
# Sets globals: UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX
# Falls back to safe defaults if file not found
# -----------------------------------------------------------------------------------------------
get_unraid_temp_thresholds() {
local cfg="/boot/config/plugins/dynamix/dynamix.cfg"
if [[ -f "$cfg" ]]; then
UNRAID_DISK_HOT=$(grep '^hot=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"')
UNRAID_DISK_MAX=$(grep '^max=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"')
UNRAID_SSD_HOT=$(grep '^hotssd=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"')
UNRAID_SSD_MAX=$(grep '^maxssd=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"')
fi
# Safe defaults if not found
UNRAID_DISK_HOT="${UNRAID_DISK_HOT:-45}"
UNRAID_DISK_MAX="${UNRAID_DISK_MAX:-55}"
UNRAID_SSD_HOT="${UNRAID_SSD_HOT:-60}"
UNRAID_SSD_MAX="${UNRAID_SSD_MAX:-70}"
}
# -----------------------------------------------------------------------------------------------
# check_local_disk_temps — check local disk temps before rsync
# Reads temps and rotational flag from /var/local/emhttp/disks.ini
# Uses unRAID's own thresholds from dynamix.cfg
#
# Returns:
# 0 = all temps OK
# 1 = warn threshold exceeded (skip this profile)
# 2 = critical threshold exceeded (abort all remaining profiles)
#
# Sets global TEMP_CHECK_RESULT with human readable summary
# -----------------------------------------------------------------------------------------------
check_local_disk_temps() {
get_unraid_temp_thresholds
local disks_ini="/var/local/emhttp/disks.ini"
if [[ ! -f "$disks_ini" ]]; then
warn "disks.ini not found — skipping temp check"
TEMP_CHECK_RESULT="temp check skipped (disks.ini not found)"
return 0
fi
local worst_result=0
local hot_drives=()
local crit_drives=()
local current_name=""
local current_device=""
local current_rotational=""
local current_temp=""
check_drive() {
[[ -z "$current_name" ]] || [[ -z "$current_temp" ]] && return
[[ "$current_temp" -eq 0 ]] && return # spun down
local warn_thresh crit_thresh
if [[ "$current_rotational" == "0" ]]; then
warn_thresh="$UNRAID_SSD_HOT"
crit_thresh="$UNRAID_SSD_MAX"
else
warn_thresh="$UNRAID_DISK_HOT"
crit_thresh="$UNRAID_DISK_MAX"
fi
if [[ "$current_temp" -ge "$crit_thresh" ]]; then
crit_drives+=("${current_name}(${current_device}):${current_temp}°C≥${crit_thresh}°C")
[[ $worst_result -lt 2 ]] && worst_result=2
elif [[ "$current_temp" -ge "$warn_thresh" ]]; then
hot_drives+=("${current_name}(${current_device}):${current_temp}°C≥${warn_thresh}°C")
[[ $worst_result -lt 1 ]] && worst_result=1
fi
}
while IFS= read -r ini_line; do
if echo "$ini_line" | grep -qE '^\["(disk[0-9]+|parity[0-9]?|cache[0-9]?)"\]'; then
# Save previous drive before starting new one
check_drive
current_name=$(echo "$ini_line" | grep -o '"[^"]*"' | head -1 | tr -d '"')
current_device=""
current_rotational="1" # default HDD
current_temp=""
elif echo "$ini_line" | grep -q '^device='; then
current_device=$(echo "$ini_line" | cut -d= -f2 | tr -d '"')
elif echo "$ini_line" | grep -q '^rotational='; then
current_rotational=$(echo "$ini_line" | cut -d= -f2 | tr -d '"')
elif echo "$ini_line" | grep -q '^temp='; then
current_temp=$(echo "$ini_line" | cut -d= -f2 | tr -d '"')
current_temp="${current_temp//[^0-9]/}"
current_temp="${current_temp:-0}"
fi
done < "$disks_ini"
check_drive # process last drive
if [[ ${#crit_drives[@]} -gt 0 ]]; then
TEMP_CHECK_RESULT="CRITICAL temps: ${crit_drives[*]}"
error "$ICON_WARN Drive temp CRITICAL — aborting all remaining syncs: ${crit_drives[*]}"
notify "Rsync aborted on $(hostname) — drive temp CRITICAL: ${crit_drives[*]}" "Rsync Temp Check" "warning"
return 2
elif [[ ${#hot_drives[@]} -gt 0 ]]; then
TEMP_CHECK_RESULT="HOT drives: ${hot_drives[*]}"
warn "$ICON_WARN Drive temp WARNING — skipping this profile: ${hot_drives[*]}"
notify "Rsync profile skipped on $(hostname) — drive temp WARNING: ${hot_drives[*]}" "Rsync Temp Check" "normal"
return 1
else
TEMP_CHECK_RESULT="all normal (HDD warn:${UNRAID_DISK_HOT}°C crit:${UNRAID_DISK_MAX}°C SSD warn:${UNRAID_SSD_HOT}°C crit:${UNRAID_SSD_MAX}°C)"
return 0
fi
}
# -----------------------------------------------------------------------------------------------
# check_remote_disks — verify all disks backing a share are healthy on the remote server
# Auto-detects filesystem type from disks.ini — handles XFS and ZFS correctly
# No SKIP_DISK_CHECK needed — detection is automatic
#
# XFS array disks: checks mountpoint is active via mountpoint -q
# ZFS disks/pools: checks zpool status is ONLINE
# Shares spanning multiple disks: all must pass
#
# Usage: check_remote_disks "/mnt/user/Movies"
# -----------------------------------------------------------------------------------------------
check_remote_disks() {
@@ -481,34 +594,104 @@ check_remote_disks() {
info "$ICON_DISK Checking disks backing $share_name on $REMOTE_SERVER_NAME..."
DISK_PATHS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"ls -d /mnt/disk*/$share_name 2>/dev/null" 2>/dev/null)
# Get disk→fsType mapping from remote disks.ini
local disks_ini_content
disks_ini_content=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"cat /var/local/emhttp/disks.ini 2>/dev/null" 2>/dev/null)
if [[ -z "$DISK_PATHS" ]]; then
error "$ICON_DISK No disks found backing $share_name on $REMOTE_SERVER_NAME"
if [[ -z "$disks_ini_content" ]]; then
error "$ICON_DISK Cannot read disks.ini from $REMOTE_SERVER_NAME"
exit 1
fi
local all_ok=true
while IFS= read -r disk_share_path; do
local disk_mount disk_name
disk_mount=$(dirname "$disk_share_path")
disk_name=$(basename "$disk_mount")
MOUNTED=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"mountpoint -q '$disk_mount' && echo yes || echo no" 2>/dev/null)
if [[ "$MOUNTED" == "yes" ]]; then
info "$ICON_DISK $disk_name $ICON_RUNNING$share_name present"
# Find which disk(s) back this share on remote
local backing_disks
backing_disks=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"ls -d /mnt/disk*/$share_name 2>/dev/null | awk -F/ '{print \$3}'" 2>/dev/null)
# Also check ZFS standalone pools (cache, gaming, media-servers etc.)
local zfs_pool_paths
zfs_pool_paths=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"zpool list -H -o name 2>/dev/null | while read pool; do
[[ -d \"/mnt/\${pool}/$share_name\" ]] && echo \"\$pool\"
done" 2>/dev/null)
if [[ -z "$backing_disks" ]] && [[ -z "$zfs_pool_paths" ]]; then
# Check cache pool directly
local on_cache
on_cache=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"[[ -d '/mnt/cache/$share_name' ]] && echo yes" 2>/dev/null)
if [[ "$on_cache" == "yes" ]]; then
backing_disks=""
zfs_pool_paths="cache"
else
error "$ICON_DISK $disk_name $ICON_STOPPED$share_name missing"
all_ok=false
error "$ICON_DISK No disks found backing $share_name on $REMOTE_SERVER_NAME"
exit 1
fi
done <<< "$DISK_PATHS"
fi
local all_ok=true
# Check array disks (XFS or ZFS single-disk-in-array)
if [[ -n "$backing_disks" ]]; then
while IFS= read -r disk_name; do
[[ -z "$disk_name" ]] && continue
# Get fsType for this disk from disks.ini
local fs_type
fs_type=$(echo "$disks_ini_content" | awk -F= -v disk="$disk_name" '
/^\["'"'"'?/ { current=substr($0,3,length($0)-4) }
current==disk && /^fsType=/ { print $2; exit }
' | tr -d '"')
fs_type="${fs_type:-xfs}"
if [[ "$fs_type" == "zfs" ]]; then
# ZFS single disk in array — check zpool status
local pool_health
pool_health=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"zpool list -H -o health '$disk_name' 2>/dev/null" 2>/dev/null)
if [[ "$pool_health" == "ONLINE" ]]; then
info "$ICON_DISK $disk_name (ZFS) $ICON_RUNNING$share_name ONLINE"
else
error "$ICON_DISK $disk_name (ZFS) $ICON_STOPPED — pool ${pool_health:-offline}"
all_ok=false
fi
else
# XFS — check mountpoint
local mounted
mounted=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"mountpoint -q '/mnt/$disk_name' && echo yes || echo no" 2>/dev/null)
if [[ "$mounted" == "yes" ]]; then
info "$ICON_DISK $disk_name (XFS) $ICON_RUNNING$share_name present"
else
error "$ICON_DISK $disk_name (XFS) $ICON_STOPPED — not mounted"
all_ok=false
fi
fi
done <<< "$backing_disks"
fi
# Check ZFS standalone pools
if [[ -n "$zfs_pool_paths" ]]; then
while IFS= read -r pool_name; do
[[ -z "$pool_name" ]] && continue
local pool_health
pool_health=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"zpool list -H -o health '$pool_name' 2>/dev/null" 2>/dev/null)
if [[ "$pool_health" == "ONLINE" ]]; then
info "$ICON_DISK $pool_name (ZFS pool) $ICON_RUNNING$share_name ONLINE"
else
error "$ICON_DISK $pool_name (ZFS pool) $ICON_STOPPED — pool ${pool_health:-offline}"
all_ok=false
fi
done <<< "$zfs_pool_paths"
fi
if [[ "$all_ok" == false ]]; then
error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME"
exit 1
fi
success "All disks backing $share_name are online"
success "All disks backing $share_name are online"
}
# -----------------------------------------------------------------------------------------------
@@ -905,6 +1088,63 @@ translate_path() {
fi
}
# -----------------------------------------------------------------------------------------------
# check_arr_version — verify arr major version matches tested version in Master.conf
# Exits the calling script if version doesn't match — prevents running against untested API
#
# Usage:
# check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" "Radarr"
#
# Arguments:
# $1 = arr base URL
# $2 = API key
# $3 = API path prefix (v3 or v1)
# $4 = expected major version number
# $5 = arr name for error messages
# -----------------------------------------------------------------------------------------------
check_arr_version() {
local url="$1"
local api_key="$2"
local api_prefix="$3"
local expected_major="$4"
local arr_name="${5:-Arr}"
local status_response version major_version
status_response=$(curl -sf --max-time 10 \
-H "X-Api-Key: $api_key" \
"${url}/api/${api_prefix}/system/status" 2>/dev/null)
if [[ -z "$status_response" ]]; then
warn "$arr_name version check failed — could not reach system/status endpoint"
warn "Proceeding without version verification — monitor for API errors"
return 0
fi
version=$(echo "$status_response" | \
grep -o '"version":"[^"]*"' | \
grep -o '[0-9][^"]*' | head -1)
if [[ -z "$version" ]]; then
warn "$arr_name version check failed — could not parse version from response"
warn "Proceeding without version verification — monitor for API errors"
return 0
fi
major_version="${version%%.*}"
if [[ "$major_version" == "$expected_major" ]]; then
success "$arr_name version: $version (major $major_version — tested ✅)"
return 0
else
error "$arr_name version mismatch — running v${major_version}, tested against v${expected_major}"
error "The API endpoint structure may have changed — exiting to protect your library"
error "Update ${arr_name^^}_VERSION_MAJOR in Master.conf after verifying the script works with v${major_version}"
notify "$arr_name version mismatch on $(hostname) — running v${major_version}, script tested against v${expected_major}" "$arr_name Cleanup" "warning"
exit 1
fi
}
check_api() {
local url="$1"
local service="${2:-API}"
+1407
View File
File diff suppressed because it is too large Load Diff
-2
View File
@@ -221,9 +221,7 @@
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Critical-Data
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Important-Data
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Media_Server/Emby --profile=emby-failover
# ^^ schedule every 30-60min — dirty sync, Emby running, critical data only
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Media_Server/Emby
# ^^ do NOT schedule — called by weekly_sync_maintenance.sh Sunday 2:30am only
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Gmer4Lfe
#
# ━━━ Rsync — Individual Media Shares (ad hoc) ━━━