Warn before a transfer when a disk backing the remote share is nearly full

Online is not writable: check_remote_disks confirmed each backing disk was mounted and healthy but
never how full it was, and rsync answers a full destination by stalling rather than failing.
This commit is contained in:
Gmer4Lfe
2026-08-23 10:15:54 -04:00
parent fd1ab58598
commit a392108562
2 changed files with 46 additions and 0 deletions
+43
View File
@@ -1526,6 +1526,49 @@ check_remote_disks() {
error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME"
exit 1
fi
# ── Free space, per backing disk ──────────────────────────────────────────────────────
# Online is not the same as writable. A share pinned by its allocation method to one disk
# reports plenty free at the share level while that disk is full, and rsync answers a full
# destination by STALLING rather than failing — it holds its lock and every later run of
# that tier declines to start. RSYNC_MAX_RUNTIME_HOURS bounds how long that lasts; this is
# the part that says so before the transfer rather than after.
#
# Warn-and-notify, never abort: rsync still moves whatever fits, and refusing every
# transfer because one backing disk is low is a worse outage than a bounded partial run.
local _min_free_gb="${RSYNC_MIN_FREE_GB:-50}"
if [[ "$_min_free_gb" -gt 0 ]] 2>/dev/null; then
local _paths=""
while IFS= read -r disk_name; do
[[ -n "$disk_name" ]] && _paths+=" /mnt/$disk_name/$share_name"
done <<< "$backing_disks"
while IFS= read -r pool_name; do
[[ -n "$pool_name" ]] && _paths+=" /mnt/$pool_name/$share_name"
done <<< "$zfs_pool_paths"
if [[ -n "${_paths// /}" ]]; then
# One round trip for every path — this runs before each transfer and the mesh link
# is the slow part. -BG so the unit is fixed regardless of the remote's df default.
local _df_out
_df_out=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
"for p in $_paths; do [ -d \"\$p\" ] && echo \"\$p \$(df -BG --output=avail \"\$p\" 2>/dev/null | tail -1 | tr -dc '0-9')\"; done" 2>/dev/null)
local _low=""
while IFS=' ' read -r _p _avail; do
[[ -z "$_p" || -z "$_avail" ]] && continue
if [[ "$_avail" -lt "$_min_free_gb" ]] 2>/dev/null; then
warn "$ICON_DISK $_p has ${_avail}G free — below ${_min_free_gb}G"
_low+="$_p (${_avail}G) "
fi
done <<< "$_df_out"
if [[ -n "$_low" ]]; then
notify "Low free space on $REMOTE_SERVER_NAME backing $share_name: $_low— rsync stalls rather than failing when a destination disk fills" \
"Varaverk: remote disk nearly full" "warning"
fi
fi
fi
success "All disks backing $share_name are online ✅"
}