From 227cdaac19fb5f0d4fa61a0b8bf27c7ea53acb27 Mon Sep 17 00:00:00 2001 From: FailedProxy Date: Mon, 6 Apr 2026 00:58:37 +0000 Subject: [PATCH] added share, rootfs, and disk checks --- Master.conf | 7 ++- Rsync/rsync.sh | 5 +- common.sh | 143 ++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 151 insertions(+), 4 deletions(-) diff --git a/Master.conf b/Master.conf index 9e0f685..bd81bea 100644 --- a/Master.conf +++ b/Master.conf @@ -41,6 +41,12 @@ SSH_PORT=221 # Default global rsync options DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file) +# ------------------- Remote Health Checks ---------------------- +# Abort if remote rootfs usage is at or above this percentage. +# Protects against rsync writing to rootfs when the remote array is down or drives are missing. +# Recommended: 75 — gives headroom before the server becomes unstable + ROOTFS_WARN=75 + # ------------------- Daily Sync Shares ------------------------- # Shares synced once daily by Orchestrators/daily_sync.sh # Add or remove paths here to manage what gets synced. @@ -159,4 +165,3 @@ declare -A PROFILE_EXCLUDE_DIRS=( #----------------------------------------------------------------------------------------------- #---------------------- End Of User Variables, Please adjust above as needed ------------------- #----------------------------------------------------------------------------------------------- - \ No newline at end of file diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index 9eaa5c2..0d3422c 100644 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -37,6 +37,9 @@ echo "--- $ICON_GEAR Setup ---" detect_hosts resolve_remote_ip check_connectivity +check_remote_rootfs +check_remote_share "$DIRECTORY" +check_remote_disks "$DIRECTORY" # Profile is inferred from the directory basename (lowercased) # e.g. /mnt/user/appdata-Failover/Arrs_Stack → arrs_stack @@ -114,7 +117,7 @@ start_containers END=$(date +%s) # ----------------------------------------------------------------------------------------------- -# --- $ICON_SUCCESS Summary --- +# --- Summary --- # ----------------------------------------------------------------------------------------------- echo "" echo "===== SUMMARY =====" diff --git a/common.sh b/common.sh index 7dbf869..e1ba03d 100644 --- a/common.sh +++ b/common.sh @@ -2,7 +2,7 @@ # ----------------------------------------------------------------------------------------------- # ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ---------------------------- # ----------------------------------------------------------------------------------------------- -# Version: 1.2 +# Version: 1.4 # ----------------------------------------------------------------------------------------------- # Changelog: # v1.0 — Initial stable framework @@ -10,8 +10,14 @@ # SSH_KEY collision resolved — gitea key renamed GITEA_SSH_KEY in Master.conf # Version and changelog tracking added # v1.2 — Consistent function header comment blocks across all functions -# check_connectivity friendlier error output with tailscale hint # check_connectivity added as standalone function +# check_connectivity friendlier error output with tailscale hint +# v1.3 — check_remote_rootfs added — aborts if remote rootfs exceeds ROOTFS_WARN threshold +# check_remote_share added — aborts if target directory is missing or empty on remote +# Both protect against rsync running when remote array is down or drives are missing +# v1.4 — check_remote_disks added — verifies all physical disks backing a share are mounted +# Discovers disk layout automatically at runtime, no configuration required +# Aborts if any single disk backing the share is offline or unmounted # ----------------------------------------------------------------------------------------------- # ICONS @@ -24,6 +30,7 @@ ICON_STOP="🛑" ICON_RETRY="🔁" ICON_SYNC="🔄" ICON_GEAR="⚙️" +ICON_DISK="💾" # ----------------------------------------------------------------------------------------------- # OUTPUT HELPERS @@ -176,6 +183,138 @@ check_connectivity() { log "Remote is reachable" } +# ----------------------------------------------------------------------------------------------- +# REMOTE ROOTFS SPACE CHECK +# Checks the remote server's rootfs usage before any rsync runs. +# If the array is down or drives are missing, rsync writes land on rootfs instead of the array — +# this can fill the remote filesystem rapidly and crash the server. +# Threshold is set by ROOTFS_WARN in Master.conf (recommended: 75). +# Aborts cleanly with a clear error showing current usage vs threshold. +# ----------------------------------------------------------------------------------------------- +check_remote_rootfs() { + log "Checking remote rootfs usage..." + + REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null) + + if [[ -z "$REMOTE_USAGE" ]]; then + error "Could not retrieve rootfs usage from $REMOTE_SERVER_NAME" + exit 1 + fi + + if [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then + echo "" + error "Remote rootfs is ${REMOTE_USAGE}% full — threshold is ${ROOTFS_WARN:-75}%" + warn "Array may be down or drives missing on $REMOTE_SERVER_NAME" + info "Hint: Check array status on $REMOTE_SERVER_NAME before retrying" + echo "" + exit 1 + fi + + success "Remote rootfs: ${REMOTE_USAGE}% used (threshold: ${ROOTFS_WARN:-75}%)" +} + +# ----------------------------------------------------------------------------------------------- +# REMOTE SHARE VALIDATION +# Verifies that the target directory exists and is not empty on the remote server. +# Catches the scenario where the array is mounted but drives are not backing the share — +# the path exists as an empty mountpoint, which would cause --delete to wipe the remote. +# Called with the specific directory being synced so each share is checked individually. +# Usage: check_remote_share "/mnt/user/Movies" +# ----------------------------------------------------------------------------------------------- +check_remote_share() { + local dir="$1" + + log "Checking remote share: $dir..." + + SHARE_EXISTS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null) + + if [[ "$SHARE_EXISTS" != "yes" ]]; then + echo "" + error "Remote share does not exist: $dir" + warn "Array may not be started or share is not configured on $REMOTE_SERVER_NAME" + info "Hint: Check shares and array status on $REMOTE_SERVER_NAME before retrying" + echo "" + exit 1 + fi + + SHARE_EMPTY=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null) + + if [[ "$SHARE_EMPTY" == "yes" ]]; then + echo "" + warn "Remote share exists but is empty: $dir" + warn "Drives may not be mounted on $REMOTE_SERVER_NAME — aborting to protect data" + info "Hint: Verify array and drive assignments on $REMOTE_SERVER_NAME before retrying" + echo "" + exit 1 + fi + + success "Remote share verified: $dir" +} + +# ----------------------------------------------------------------------------------------------- +# REMOTE DISK CHECK +# Verifies that all physical disks backing a share are online and mounted on the remote server. +# Discovers disk layout automatically at runtime by finding all /mnt/diskN/sharename paths — +# no configuration required, works for any share regardless of how many disks it spans. +# Aborts if any single disk backing the share is offline — partial disk failure means +# incomplete data which could result in files being deleted by --delete during sync. +# Usage: check_remote_disks "/mnt/user/Movies" +# ----------------------------------------------------------------------------------------------- +check_remote_disks() { + local dir="$1" + local share_name + share_name=$(basename "$dir") + + info "$ICON_DISK Checking disks backing $share_name on $REMOTE_SERVER_NAME..." + + # Find all /mnt/diskN paths that contain this share on the remote + DISK_PATHS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "ls -d /mnt/disk*/$share_name 2>/dev/null" 2>/dev/null) + + if [[ -z "$DISK_PATHS" ]]; then + echo "" + error "No disks found backing share $share_name on $REMOTE_SERVER_NAME" + warn "Share may not exist on any disk or array may not be started" + info "Hint: Check array and share configuration on $REMOTE_SERVER_NAME" + echo "" + exit 1 + fi + + local all_ok=true + + while IFS= read -r disk_share_path; do + # Extract the disk mountpoint — e.g. /mnt/disk8 from /mnt/disk8/Movies + local disk_mount + disk_mount=$(dirname "$disk_share_path") + local disk_name + disk_name=$(basename "$disk_mount") + + MOUNTED=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "mountpoint -q '$disk_mount' && echo yes || echo no" 2>/dev/null) + + if [[ "$MOUNTED" == "yes" ]]; then + success "$ICON_DISK $disk_name online — $share_name present" + else + error "$disk_name is not mounted — $share_name may be incomplete" + all_ok=false + fi + done <<< "$DISK_PATHS" + + if [[ "$all_ok" == false ]]; then + echo "" + error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME" + warn "Aborting to prevent partial or destructive sync" + info "Hint: Check disk assignments and array status on $REMOTE_SERVER_NAME before retrying" + echo "" + exit 1 + fi + + success "All disks backing $share_name are online" +} + # ----------------------------------------------------------------------------------------------- # CONTAINER MANAGEMENT — STOP # Stops all containers listed in CRITICAL_CONTAINER_NAMES on the remote server.