big logic update to rsync and rsync called scripts
This commit is contained in:
@@ -469,9 +469,122 @@ check_remote_share() {
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# REMOTE DISK CHECK — fatal
|
||||
# Verifies all physical disks backing a share are online on the remote server.
|
||||
# Skipped when PROFILE_SKIP_DISK_CHECK is true (ZFS pools have no /mnt/disk* structure).
|
||||
# get_unraid_temp_thresholds — read disk temp thresholds from unRAID's dynamix.cfg
|
||||
# Sets globals: UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX
|
||||
# Falls back to safe defaults if file not found
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
get_unraid_temp_thresholds() {
|
||||
local cfg="/boot/config/plugins/dynamix/dynamix.cfg"
|
||||
if [[ -f "$cfg" ]]; then
|
||||
UNRAID_DISK_HOT=$(grep '^hot=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"')
|
||||
UNRAID_DISK_MAX=$(grep '^max=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"')
|
||||
UNRAID_SSD_HOT=$(grep '^hotssd=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"')
|
||||
UNRAID_SSD_MAX=$(grep '^maxssd=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"')
|
||||
fi
|
||||
# Safe defaults if not found
|
||||
UNRAID_DISK_HOT="${UNRAID_DISK_HOT:-45}"
|
||||
UNRAID_DISK_MAX="${UNRAID_DISK_MAX:-55}"
|
||||
UNRAID_SSD_HOT="${UNRAID_SSD_HOT:-60}"
|
||||
UNRAID_SSD_MAX="${UNRAID_SSD_MAX:-70}"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# check_local_disk_temps — check local disk temps before rsync
|
||||
# Reads temps and rotational flag from /var/local/emhttp/disks.ini
|
||||
# Uses unRAID's own thresholds from dynamix.cfg
|
||||
#
|
||||
# Returns:
|
||||
# 0 = all temps OK
|
||||
# 1 = warn threshold exceeded (skip this profile)
|
||||
# 2 = critical threshold exceeded (abort all remaining profiles)
|
||||
#
|
||||
# Sets global TEMP_CHECK_RESULT with human readable summary
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_local_disk_temps() {
|
||||
get_unraid_temp_thresholds
|
||||
|
||||
local disks_ini="/var/local/emhttp/disks.ini"
|
||||
if [[ ! -f "$disks_ini" ]]; then
|
||||
warn "disks.ini not found — skipping temp check"
|
||||
TEMP_CHECK_RESULT="temp check skipped (disks.ini not found)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local worst_result=0
|
||||
local hot_drives=()
|
||||
local crit_drives=()
|
||||
local current_name=""
|
||||
local current_device=""
|
||||
local current_rotational=""
|
||||
local current_temp=""
|
||||
|
||||
check_drive() {
|
||||
[[ -z "$current_name" ]] || [[ -z "$current_temp" ]] && return
|
||||
[[ "$current_temp" -eq 0 ]] && return # spun down
|
||||
|
||||
local warn_thresh crit_thresh
|
||||
if [[ "$current_rotational" == "0" ]]; then
|
||||
warn_thresh="$UNRAID_SSD_HOT"
|
||||
crit_thresh="$UNRAID_SSD_MAX"
|
||||
else
|
||||
warn_thresh="$UNRAID_DISK_HOT"
|
||||
crit_thresh="$UNRAID_DISK_MAX"
|
||||
fi
|
||||
|
||||
if [[ "$current_temp" -ge "$crit_thresh" ]]; then
|
||||
crit_drives+=("${current_name}(${current_device}):${current_temp}°C≥${crit_thresh}°C")
|
||||
[[ $worst_result -lt 2 ]] && worst_result=2
|
||||
elif [[ "$current_temp" -ge "$warn_thresh" ]]; then
|
||||
hot_drives+=("${current_name}(${current_device}):${current_temp}°C≥${warn_thresh}°C")
|
||||
[[ $worst_result -lt 1 ]] && worst_result=1
|
||||
fi
|
||||
}
|
||||
|
||||
while IFS= read -r ini_line; do
|
||||
if echo "$ini_line" | grep -qE '^\["(disk[0-9]+|parity[0-9]?|cache[0-9]?)"\]'; then
|
||||
# Save previous drive before starting new one
|
||||
check_drive
|
||||
current_name=$(echo "$ini_line" | grep -o '"[^"]*"' | head -1 | tr -d '"')
|
||||
current_device=""
|
||||
current_rotational="1" # default HDD
|
||||
current_temp=""
|
||||
elif echo "$ini_line" | grep -q '^device='; then
|
||||
current_device=$(echo "$ini_line" | cut -d= -f2 | tr -d '"')
|
||||
elif echo "$ini_line" | grep -q '^rotational='; then
|
||||
current_rotational=$(echo "$ini_line" | cut -d= -f2 | tr -d '"')
|
||||
elif echo "$ini_line" | grep -q '^temp='; then
|
||||
current_temp=$(echo "$ini_line" | cut -d= -f2 | tr -d '"')
|
||||
current_temp="${current_temp//[^0-9]/}"
|
||||
current_temp="${current_temp:-0}"
|
||||
fi
|
||||
done < "$disks_ini"
|
||||
check_drive # process last drive
|
||||
|
||||
if [[ ${#crit_drives[@]} -gt 0 ]]; then
|
||||
TEMP_CHECK_RESULT="CRITICAL temps: ${crit_drives[*]}"
|
||||
error "$ICON_WARN Drive temp CRITICAL — aborting all remaining syncs: ${crit_drives[*]}"
|
||||
notify "Rsync aborted on $(hostname) — drive temp CRITICAL: ${crit_drives[*]}" "Rsync Temp Check" "warning"
|
||||
return 2
|
||||
elif [[ ${#hot_drives[@]} -gt 0 ]]; then
|
||||
TEMP_CHECK_RESULT="HOT drives: ${hot_drives[*]}"
|
||||
warn "$ICON_WARN Drive temp WARNING — skipping this profile: ${hot_drives[*]}"
|
||||
notify "Rsync profile skipped on $(hostname) — drive temp WARNING: ${hot_drives[*]}" "Rsync Temp Check" "normal"
|
||||
return 1
|
||||
else
|
||||
TEMP_CHECK_RESULT="all normal (HDD warn:${UNRAID_DISK_HOT}°C crit:${UNRAID_DISK_MAX}°C SSD warn:${UNRAID_SSD_HOT}°C crit:${UNRAID_SSD_MAX}°C)"
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# check_remote_disks — verify all disks backing a share are healthy on the remote server
|
||||
# Auto-detects filesystem type from disks.ini — handles XFS and ZFS correctly
|
||||
# No SKIP_DISK_CHECK needed — detection is automatic
|
||||
#
|
||||
# XFS array disks: checks mountpoint is active via mountpoint -q
|
||||
# ZFS disks/pools: checks zpool status is ONLINE
|
||||
# Shares spanning multiple disks: all must pass
|
||||
#
|
||||
# Usage: check_remote_disks "/mnt/user/Movies"
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_remote_disks() {
|
||||
@@ -481,34 +594,104 @@ check_remote_disks() {
|
||||
|
||||
info "$ICON_DISK Checking disks backing $share_name on $REMOTE_SERVER_NAME..."
|
||||
|
||||
DISK_PATHS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"ls -d /mnt/disk*/$share_name 2>/dev/null" 2>/dev/null)
|
||||
# Get disk→fsType mapping from remote disks.ini
|
||||
local disks_ini_content
|
||||
disks_ini_content=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"cat /var/local/emhttp/disks.ini 2>/dev/null" 2>/dev/null)
|
||||
|
||||
if [[ -z "$DISK_PATHS" ]]; then
|
||||
error "$ICON_DISK No disks found backing $share_name on $REMOTE_SERVER_NAME"
|
||||
if [[ -z "$disks_ini_content" ]]; then
|
||||
error "$ICON_DISK Cannot read disks.ini from $REMOTE_SERVER_NAME"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local all_ok=true
|
||||
while IFS= read -r disk_share_path; do
|
||||
local disk_mount disk_name
|
||||
disk_mount=$(dirname "$disk_share_path")
|
||||
disk_name=$(basename "$disk_mount")
|
||||
MOUNTED=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"mountpoint -q '$disk_mount' && echo yes || echo no" 2>/dev/null)
|
||||
if [[ "$MOUNTED" == "yes" ]]; then
|
||||
info "$ICON_DISK $disk_name $ICON_RUNNING — $share_name present"
|
||||
# Find which disk(s) back this share on remote
|
||||
local backing_disks
|
||||
backing_disks=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"ls -d /mnt/disk*/$share_name 2>/dev/null | awk -F/ '{print \$3}'" 2>/dev/null)
|
||||
|
||||
# Also check ZFS standalone pools (cache, gaming, media-servers etc.)
|
||||
local zfs_pool_paths
|
||||
zfs_pool_paths=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"zpool list -H -o name 2>/dev/null | while read pool; do
|
||||
[[ -d \"/mnt/\${pool}/$share_name\" ]] && echo \"\$pool\"
|
||||
done" 2>/dev/null)
|
||||
|
||||
if [[ -z "$backing_disks" ]] && [[ -z "$zfs_pool_paths" ]]; then
|
||||
# Check cache pool directly
|
||||
local on_cache
|
||||
on_cache=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"[[ -d '/mnt/cache/$share_name' ]] && echo yes" 2>/dev/null)
|
||||
if [[ "$on_cache" == "yes" ]]; then
|
||||
backing_disks=""
|
||||
zfs_pool_paths="cache"
|
||||
else
|
||||
error "$ICON_DISK $disk_name $ICON_STOPPED — $share_name missing"
|
||||
all_ok=false
|
||||
error "$ICON_DISK No disks found backing $share_name on $REMOTE_SERVER_NAME"
|
||||
exit 1
|
||||
fi
|
||||
done <<< "$DISK_PATHS"
|
||||
fi
|
||||
|
||||
local all_ok=true
|
||||
|
||||
# Check array disks (XFS or ZFS single-disk-in-array)
|
||||
if [[ -n "$backing_disks" ]]; then
|
||||
while IFS= read -r disk_name; do
|
||||
[[ -z "$disk_name" ]] && continue
|
||||
|
||||
# Get fsType for this disk from disks.ini
|
||||
local fs_type
|
||||
fs_type=$(echo "$disks_ini_content" | awk -F= -v disk="$disk_name" '
|
||||
/^\["'"'"'?/ { current=substr($0,3,length($0)-4) }
|
||||
current==disk && /^fsType=/ { print $2; exit }
|
||||
' | tr -d '"')
|
||||
fs_type="${fs_type:-xfs}"
|
||||
|
||||
if [[ "$fs_type" == "zfs" ]]; then
|
||||
# ZFS single disk in array — check zpool status
|
||||
local pool_health
|
||||
pool_health=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"zpool list -H -o health '$disk_name' 2>/dev/null" 2>/dev/null)
|
||||
if [[ "$pool_health" == "ONLINE" ]]; then
|
||||
info "$ICON_DISK $disk_name (ZFS) $ICON_RUNNING — $share_name ONLINE"
|
||||
else
|
||||
error "$ICON_DISK $disk_name (ZFS) $ICON_STOPPED — pool ${pool_health:-offline}"
|
||||
all_ok=false
|
||||
fi
|
||||
else
|
||||
# XFS — check mountpoint
|
||||
local mounted
|
||||
mounted=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"mountpoint -q '/mnt/$disk_name' && echo yes || echo no" 2>/dev/null)
|
||||
if [[ "$mounted" == "yes" ]]; then
|
||||
info "$ICON_DISK $disk_name (XFS) $ICON_RUNNING — $share_name present"
|
||||
else
|
||||
error "$ICON_DISK $disk_name (XFS) $ICON_STOPPED — not mounted"
|
||||
all_ok=false
|
||||
fi
|
||||
fi
|
||||
done <<< "$backing_disks"
|
||||
fi
|
||||
|
||||
# Check ZFS standalone pools
|
||||
if [[ -n "$zfs_pool_paths" ]]; then
|
||||
while IFS= read -r pool_name; do
|
||||
[[ -z "$pool_name" ]] && continue
|
||||
local pool_health
|
||||
pool_health=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"zpool list -H -o health '$pool_name' 2>/dev/null" 2>/dev/null)
|
||||
if [[ "$pool_health" == "ONLINE" ]]; then
|
||||
info "$ICON_DISK $pool_name (ZFS pool) $ICON_RUNNING — $share_name ONLINE"
|
||||
else
|
||||
error "$ICON_DISK $pool_name (ZFS pool) $ICON_STOPPED — pool ${pool_health:-offline}"
|
||||
all_ok=false
|
||||
fi
|
||||
done <<< "$zfs_pool_paths"
|
||||
fi
|
||||
|
||||
if [[ "$all_ok" == false ]]; then
|
||||
error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME"
|
||||
exit 1
|
||||
fi
|
||||
success "All disks backing $share_name are online"
|
||||
success "All disks backing $share_name are online ✅"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
@@ -905,6 +1088,63 @@ translate_path() {
|
||||
fi
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# check_arr_version — verify arr major version matches tested version in Master.conf
|
||||
# Exits the calling script if version doesn't match — prevents running against untested API
|
||||
#
|
||||
# Usage:
|
||||
# check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" "Radarr"
|
||||
#
|
||||
# Arguments:
|
||||
# $1 = arr base URL
|
||||
# $2 = API key
|
||||
# $3 = API path prefix (v3 or v1)
|
||||
# $4 = expected major version number
|
||||
# $5 = arr name for error messages
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_arr_version() {
|
||||
local url="$1"
|
||||
local api_key="$2"
|
||||
local api_prefix="$3"
|
||||
local expected_major="$4"
|
||||
local arr_name="${5:-Arr}"
|
||||
|
||||
local status_response version major_version
|
||||
|
||||
status_response=$(curl -sf --max-time 10 \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
"${url}/api/${api_prefix}/system/status" 2>/dev/null)
|
||||
|
||||
if [[ -z "$status_response" ]]; then
|
||||
warn "$arr_name version check failed — could not reach system/status endpoint"
|
||||
warn "Proceeding without version verification — monitor for API errors"
|
||||
return 0
|
||||
fi
|
||||
|
||||
version=$(echo "$status_response" | \
|
||||
grep -o '"version":"[^"]*"' | \
|
||||
grep -o '[0-9][^"]*' | head -1)
|
||||
|
||||
if [[ -z "$version" ]]; then
|
||||
warn "$arr_name version check failed — could not parse version from response"
|
||||
warn "Proceeding without version verification — monitor for API errors"
|
||||
return 0
|
||||
fi
|
||||
|
||||
major_version="${version%%.*}"
|
||||
|
||||
if [[ "$major_version" == "$expected_major" ]]; then
|
||||
success "$arr_name version: $version (major $major_version — tested ✅)"
|
||||
return 0
|
||||
else
|
||||
error "$arr_name version mismatch — running v${major_version}, tested against v${expected_major}"
|
||||
error "The API endpoint structure may have changed — exiting to protect your library"
|
||||
error "Update ${arr_name^^}_VERSION_MAJOR in Master.conf after verifying the script works with v${major_version}"
|
||||
notify "$arr_name version mismatch on $(hostname) — running v${major_version}, script tested against v${expected_major}" "$arr_name Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
check_api() {
|
||||
local url="$1"
|
||||
local service="${2:-API}"
|
||||
|
||||
Reference in New Issue
Block a user