added share, rootfs, and disk checks
This commit is contained in:
@@ -2,7 +2,7 @@
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ----------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Version: 1.2
|
||||
# Version: 1.4
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Changelog:
|
||||
# v1.0 — Initial stable framework
|
||||
@@ -10,8 +10,14 @@
|
||||
# SSH_KEY collision resolved — gitea key renamed GITEA_SSH_KEY in Master.conf
|
||||
# Version and changelog tracking added
|
||||
# v1.2 — Consistent function header comment blocks across all functions
|
||||
# check_connectivity friendlier error output with tailscale hint
|
||||
# check_connectivity added as standalone function
|
||||
# check_connectivity friendlier error output with tailscale hint
|
||||
# v1.3 — check_remote_rootfs added — aborts if remote rootfs exceeds ROOTFS_WARN threshold
|
||||
# check_remote_share added — aborts if target directory is missing or empty on remote
|
||||
# Both protect against rsync running when remote array is down or drives are missing
|
||||
# v1.4 — check_remote_disks added — verifies all physical disks backing a share are mounted
|
||||
# Discovers disk layout automatically at runtime, no configuration required
|
||||
# Aborts if any single disk backing the share is offline or unmounted
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# ICONS
|
||||
@@ -24,6 +30,7 @@ ICON_STOP="🛑"
|
||||
ICON_RETRY="🔁"
|
||||
ICON_SYNC="🔄"
|
||||
ICON_GEAR="⚙️"
|
||||
ICON_DISK="💾"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# OUTPUT HELPERS
|
||||
@@ -176,6 +183,138 @@ check_connectivity() {
|
||||
log "Remote is reachable"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# REMOTE ROOTFS SPACE CHECK
|
||||
# Checks the remote server's rootfs usage before any rsync runs.
|
||||
# If the array is down or drives are missing, rsync writes land on rootfs instead of the array —
|
||||
# this can fill the remote filesystem rapidly and crash the server.
|
||||
# Threshold is set by ROOTFS_WARN in Master.conf (recommended: 75).
|
||||
# Aborts cleanly with a clear error showing current usage vs threshold.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_remote_rootfs() {
|
||||
log "Checking remote rootfs usage..."
|
||||
|
||||
REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null)
|
||||
|
||||
if [[ -z "$REMOTE_USAGE" ]]; then
|
||||
error "Could not retrieve rootfs usage from $REMOTE_SERVER_NAME"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then
|
||||
echo ""
|
||||
error "Remote rootfs is ${REMOTE_USAGE}% full — threshold is ${ROOTFS_WARN:-75}%"
|
||||
warn "Array may be down or drives missing on $REMOTE_SERVER_NAME"
|
||||
info "Hint: Check array status on $REMOTE_SERVER_NAME before retrying"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Remote rootfs: ${REMOTE_USAGE}% used (threshold: ${ROOTFS_WARN:-75}%)"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# REMOTE SHARE VALIDATION
|
||||
# Verifies that the target directory exists and is not empty on the remote server.
|
||||
# Catches the scenario where the array is mounted but drives are not backing the share —
|
||||
# the path exists as an empty mountpoint, which would cause --delete to wipe the remote.
|
||||
# Called with the specific directory being synced so each share is checked individually.
|
||||
# Usage: check_remote_share "/mnt/user/Movies"
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_remote_share() {
|
||||
local dir="$1"
|
||||
|
||||
log "Checking remote share: $dir..."
|
||||
|
||||
SHARE_EXISTS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null)
|
||||
|
||||
if [[ "$SHARE_EXISTS" != "yes" ]]; then
|
||||
echo ""
|
||||
error "Remote share does not exist: $dir"
|
||||
warn "Array may not be started or share is not configured on $REMOTE_SERVER_NAME"
|
||||
info "Hint: Check shares and array status on $REMOTE_SERVER_NAME before retrying"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SHARE_EMPTY=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null)
|
||||
|
||||
if [[ "$SHARE_EMPTY" == "yes" ]]; then
|
||||
echo ""
|
||||
warn "Remote share exists but is empty: $dir"
|
||||
warn "Drives may not be mounted on $REMOTE_SERVER_NAME — aborting to protect data"
|
||||
info "Hint: Verify array and drive assignments on $REMOTE_SERVER_NAME before retrying"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Remote share verified: $dir"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# REMOTE DISK CHECK
|
||||
# Verifies that all physical disks backing a share are online and mounted on the remote server.
|
||||
# Discovers disk layout automatically at runtime by finding all /mnt/diskN/sharename paths —
|
||||
# no configuration required, works for any share regardless of how many disks it spans.
|
||||
# Aborts if any single disk backing the share is offline — partial disk failure means
|
||||
# incomplete data which could result in files being deleted by --delete during sync.
|
||||
# Usage: check_remote_disks "/mnt/user/Movies"
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_remote_disks() {
|
||||
local dir="$1"
|
||||
local share_name
|
||||
share_name=$(basename "$dir")
|
||||
|
||||
info "$ICON_DISK Checking disks backing $share_name on $REMOTE_SERVER_NAME..."
|
||||
|
||||
# Find all /mnt/diskN paths that contain this share on the remote
|
||||
DISK_PATHS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"ls -d /mnt/disk*/$share_name 2>/dev/null" 2>/dev/null)
|
||||
|
||||
if [[ -z "$DISK_PATHS" ]]; then
|
||||
echo ""
|
||||
error "No disks found backing share $share_name on $REMOTE_SERVER_NAME"
|
||||
warn "Share may not exist on any disk or array may not be started"
|
||||
info "Hint: Check array and share configuration on $REMOTE_SERVER_NAME"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local all_ok=true
|
||||
|
||||
while IFS= read -r disk_share_path; do
|
||||
# Extract the disk mountpoint — e.g. /mnt/disk8 from /mnt/disk8/Movies
|
||||
local disk_mount
|
||||
disk_mount=$(dirname "$disk_share_path")
|
||||
local disk_name
|
||||
disk_name=$(basename "$disk_mount")
|
||||
|
||||
MOUNTED=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"mountpoint -q '$disk_mount' && echo yes || echo no" 2>/dev/null)
|
||||
|
||||
if [[ "$MOUNTED" == "yes" ]]; then
|
||||
success "$ICON_DISK $disk_name online — $share_name present"
|
||||
else
|
||||
error "$disk_name is not mounted — $share_name may be incomplete"
|
||||
all_ok=false
|
||||
fi
|
||||
done <<< "$DISK_PATHS"
|
||||
|
||||
if [[ "$all_ok" == false ]]; then
|
||||
echo ""
|
||||
error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME"
|
||||
warn "Aborting to prevent partial or destructive sync"
|
||||
info "Hint: Check disk assignments and array status on $REMOTE_SERVER_NAME before retrying"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "All disks backing $share_name are online"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# CONTAINER MANAGEMENT — STOP
|
||||
# Stops all containers listed in CRITICAL_CONTAINER_NAMES on the remote server.
|
||||
|
||||
Reference in New Issue
Block a user