Warn before a transfer when a disk backing the remote share is nearly full
Online is not writable: check_remote_disks confirmed each backing disk was mounted and healthy but never how full it was, and rsync answers a full destination by stalling rather than failing.
This commit is contained in:
@@ -693,6 +693,9 @@
|
||||
# lowered from 23 on 2026-07-16 — daily maintenance jobs alone now
|
||||
# take ~4h; 19h cap leaves room for them to still run same-day
|
||||
# before the next 1am fire even if a share hits the cap
|
||||
RSYNC_MIN_FREE_GB=50 # warn if any disk backing the remote share is under this
|
||||
# (0 disables). Online is not writable: a full destination
|
||||
# makes rsync stall rather than fail. Warns, never aborts.
|
||||
CRITICAL_CONTAINER_NAMES=() # containers stopped on REMOTE before rsync — profiles override
|
||||
DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override
|
||||
CONTAINER_DELAY=5 # seconds before starting delayed containers
|
||||
|
||||
@@ -1526,6 +1526,49 @@ check_remote_disks() {
|
||||
error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Free space, per backing disk ──────────────────────────────────────────────────────
|
||||
# Online is not the same as writable. A share pinned by its allocation method to one disk
|
||||
# reports plenty free at the share level while that disk is full, and rsync answers a full
|
||||
# destination by STALLING rather than failing — it holds its lock and every later run of
|
||||
# that tier declines to start. RSYNC_MAX_RUNTIME_HOURS bounds how long that lasts; this is
|
||||
# the part that says so before the transfer rather than after.
|
||||
#
|
||||
# Warn-and-notify, never abort: rsync still moves whatever fits, and refusing every
|
||||
# transfer because one backing disk is low is a worse outage than a bounded partial run.
|
||||
local _min_free_gb="${RSYNC_MIN_FREE_GB:-50}"
|
||||
if [[ "$_min_free_gb" -gt 0 ]] 2>/dev/null; then
|
||||
local _paths=""
|
||||
while IFS= read -r disk_name; do
|
||||
[[ -n "$disk_name" ]] && _paths+=" /mnt/$disk_name/$share_name"
|
||||
done <<< "$backing_disks"
|
||||
while IFS= read -r pool_name; do
|
||||
[[ -n "$pool_name" ]] && _paths+=" /mnt/$pool_name/$share_name"
|
||||
done <<< "$zfs_pool_paths"
|
||||
|
||||
if [[ -n "${_paths// /}" ]]; then
|
||||
# One round trip for every path — this runs before each transfer and the mesh link
|
||||
# is the slow part. -BG so the unit is fixed regardless of the remote's df default.
|
||||
local _df_out
|
||||
_df_out=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
||||
"for p in $_paths; do [ -d \"\$p\" ] && echo \"\$p \$(df -BG --output=avail \"\$p\" 2>/dev/null | tail -1 | tr -dc '0-9')\"; done" 2>/dev/null)
|
||||
|
||||
local _low=""
|
||||
while IFS=' ' read -r _p _avail; do
|
||||
[[ -z "$_p" || -z "$_avail" ]] && continue
|
||||
if [[ "$_avail" -lt "$_min_free_gb" ]] 2>/dev/null; then
|
||||
warn "$ICON_DISK $_p has ${_avail}G free — below ${_min_free_gb}G"
|
||||
_low+="$_p (${_avail}G) "
|
||||
fi
|
||||
done <<< "$_df_out"
|
||||
|
||||
if [[ -n "$_low" ]]; then
|
||||
notify "Low free space on $REMOTE_SERVER_NAME backing $share_name: $_low— rsync stalls rather than failing when a destination disk fills" \
|
||||
"Varaverk: remote disk nearly full" "warning"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
success "All disks backing $share_name are online ✅"
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user