added share, rootfs, and disk checks
This commit is contained in:
+6
-1
@@ -41,6 +41,12 @@ SSH_PORT=221
|
||||
# Default global rsync options
|
||||
DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file)
|
||||
|
||||
# ------------------- Remote Health Checks ----------------------
|
||||
# Abort if remote rootfs usage is at or above this percentage.
|
||||
# Protects against rsync writing to rootfs when the remote array is down or drives are missing.
|
||||
# Recommended: 75 — gives headroom before the server becomes unstable
|
||||
ROOTFS_WARN=75
|
||||
|
||||
# ------------------- Daily Sync Shares -------------------------
|
||||
# Shares synced once daily by Orchestrators/daily_sync.sh
|
||||
# Add or remove paths here to manage what gets synced.
|
||||
@@ -159,4 +165,3 @@ declare -A PROFILE_EXCLUDE_DIRS=(
|
||||
#-----------------------------------------------------------------------------------------------
|
||||
#---------------------- End Of User Variables, Please adjust above as needed -------------------
|
||||
#-----------------------------------------------------------------------------------------------
|
||||
|
||||
+4
-1
@@ -37,6 +37,9 @@ echo "--- $ICON_GEAR Setup ---"
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
check_connectivity
|
||||
check_remote_rootfs
|
||||
check_remote_share "$DIRECTORY"
|
||||
check_remote_disks "$DIRECTORY"
|
||||
|
||||
# Profile is inferred from the directory basename (lowercased)
|
||||
# e.g. /mnt/user/appdata-Failover/Arrs_Stack → arrs_stack
|
||||
@@ -114,7 +117,7 @@ start_containers
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --- $ICON_SUCCESS Summary ---
|
||||
# --- Summary ---
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "===== SUMMARY ====="
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ----------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Version: 1.2
|
||||
# Version: 1.4
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Changelog:
|
||||
# v1.0 — Initial stable framework
|
||||
@@ -10,8 +10,14 @@
|
||||
# SSH_KEY collision resolved — gitea key renamed GITEA_SSH_KEY in Master.conf
|
||||
# Version and changelog tracking added
|
||||
# v1.2 — Consistent function header comment blocks across all functions
|
||||
# check_connectivity friendlier error output with tailscale hint
|
||||
# check_connectivity added as standalone function
|
||||
# check_connectivity friendlier error output with tailscale hint
|
||||
# v1.3 — check_remote_rootfs added — aborts if remote rootfs exceeds ROOTFS_WARN threshold
|
||||
# check_remote_share added — aborts if target directory is missing or empty on remote
|
||||
# Both protect against rsync running when remote array is down or drives are missing
|
||||
# v1.4 — check_remote_disks added — verifies all physical disks backing a share are mounted
|
||||
# Discovers disk layout automatically at runtime, no configuration required
|
||||
# Aborts if any single disk backing the share is offline or unmounted
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# ICONS
|
||||
@@ -24,6 +30,7 @@ ICON_STOP="🛑"
|
||||
ICON_RETRY="🔁"
|
||||
ICON_SYNC="🔄"
|
||||
ICON_GEAR="⚙️"
|
||||
ICON_DISK="💾"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# OUTPUT HELPERS
|
||||
@@ -176,6 +183,138 @@ check_connectivity() {
|
||||
log "Remote is reachable"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# REMOTE ROOTFS SPACE CHECK
|
||||
# Checks the remote server's rootfs usage before any rsync runs.
|
||||
# If the array is down or drives are missing, rsync writes land on rootfs instead of the array —
|
||||
# this can fill the remote filesystem rapidly and crash the server.
|
||||
# Threshold is set by ROOTFS_WARN in Master.conf (recommended: 75).
|
||||
# Aborts cleanly with a clear error showing current usage vs threshold.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_remote_rootfs() {
|
||||
log "Checking remote rootfs usage..."
|
||||
|
||||
REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null)
|
||||
|
||||
if [[ -z "$REMOTE_USAGE" ]]; then
|
||||
error "Could not retrieve rootfs usage from $REMOTE_SERVER_NAME"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then
|
||||
echo ""
|
||||
error "Remote rootfs is ${REMOTE_USAGE}% full — threshold is ${ROOTFS_WARN:-75}%"
|
||||
warn "Array may be down or drives missing on $REMOTE_SERVER_NAME"
|
||||
info "Hint: Check array status on $REMOTE_SERVER_NAME before retrying"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Remote rootfs: ${REMOTE_USAGE}% used (threshold: ${ROOTFS_WARN:-75}%)"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# REMOTE SHARE VALIDATION
|
||||
# Verifies that the target directory exists and is not empty on the remote server.
|
||||
# Catches the scenario where the array is mounted but drives are not backing the share —
|
||||
# the path exists as an empty mountpoint, which would cause --delete to wipe the remote.
|
||||
# Called with the specific directory being synced so each share is checked individually.
|
||||
# Usage: check_remote_share "/mnt/user/Movies"
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_remote_share() {
|
||||
local dir="$1"
|
||||
|
||||
log "Checking remote share: $dir..."
|
||||
|
||||
SHARE_EXISTS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null)
|
||||
|
||||
if [[ "$SHARE_EXISTS" != "yes" ]]; then
|
||||
echo ""
|
||||
error "Remote share does not exist: $dir"
|
||||
warn "Array may not be started or share is not configured on $REMOTE_SERVER_NAME"
|
||||
info "Hint: Check shares and array status on $REMOTE_SERVER_NAME before retrying"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SHARE_EMPTY=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null)
|
||||
|
||||
if [[ "$SHARE_EMPTY" == "yes" ]]; then
|
||||
echo ""
|
||||
warn "Remote share exists but is empty: $dir"
|
||||
warn "Drives may not be mounted on $REMOTE_SERVER_NAME — aborting to protect data"
|
||||
info "Hint: Verify array and drive assignments on $REMOTE_SERVER_NAME before retrying"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Remote share verified: $dir"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# REMOTE DISK CHECK
|
||||
# Verifies that all physical disks backing a share are online and mounted on the remote server.
|
||||
# Discovers disk layout automatically at runtime by finding all /mnt/diskN/sharename paths —
|
||||
# no configuration required, works for any share regardless of how many disks it spans.
|
||||
# Aborts if any single disk backing the share is offline — partial disk failure means
|
||||
# incomplete data which could result in files being deleted by --delete during sync.
|
||||
# Usage: check_remote_disks "/mnt/user/Movies"
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_remote_disks() {
|
||||
local dir="$1"
|
||||
local share_name
|
||||
share_name=$(basename "$dir")
|
||||
|
||||
info "$ICON_DISK Checking disks backing $share_name on $REMOTE_SERVER_NAME..."
|
||||
|
||||
# Find all /mnt/diskN paths that contain this share on the remote
|
||||
DISK_PATHS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"ls -d /mnt/disk*/$share_name 2>/dev/null" 2>/dev/null)
|
||||
|
||||
if [[ -z "$DISK_PATHS" ]]; then
|
||||
echo ""
|
||||
error "No disks found backing share $share_name on $REMOTE_SERVER_NAME"
|
||||
warn "Share may not exist on any disk or array may not be started"
|
||||
info "Hint: Check array and share configuration on $REMOTE_SERVER_NAME"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local all_ok=true
|
||||
|
||||
while IFS= read -r disk_share_path; do
|
||||
# Extract the disk mountpoint — e.g. /mnt/disk8 from /mnt/disk8/Movies
|
||||
local disk_mount
|
||||
disk_mount=$(dirname "$disk_share_path")
|
||||
local disk_name
|
||||
disk_name=$(basename "$disk_mount")
|
||||
|
||||
MOUNTED=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"mountpoint -q '$disk_mount' && echo yes || echo no" 2>/dev/null)
|
||||
|
||||
if [[ "$MOUNTED" == "yes" ]]; then
|
||||
success "$ICON_DISK $disk_name online — $share_name present"
|
||||
else
|
||||
error "$disk_name is not mounted — $share_name may be incomplete"
|
||||
all_ok=false
|
||||
fi
|
||||
done <<< "$DISK_PATHS"
|
||||
|
||||
if [[ "$all_ok" == false ]]; then
|
||||
echo ""
|
||||
error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME"
|
||||
warn "Aborting to prevent partial or destructive sync"
|
||||
info "Hint: Check disk assignments and array status on $REMOTE_SERVER_NAME before retrying"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "All disks backing $share_name are online"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# CONTAINER MANAGEMENT — STOP
|
||||
# Stops all containers listed in CRITICAL_CONTAINER_NAMES on the remote server.
|
||||
|
||||
Reference in New Issue
Block a user