Warn before a transfer when a disk backing the remote share is nearly full
Online is not writable: check_remote_disks confirmed each backing disk was mounted and healthy but never how full it was, and rsync answers a full destination by stalling rather than failing.
This commit is contained in:
@@ -693,6 +693,9 @@
|
|||||||
# lowered from 23 on 2026-07-16 — daily maintenance jobs alone now
|
# lowered from 23 on 2026-07-16 — daily maintenance jobs alone now
|
||||||
# take ~4h; 19h cap leaves room for them to still run same-day
|
# take ~4h; 19h cap leaves room for them to still run same-day
|
||||||
# before the next 1am fire even if a share hits the cap
|
# before the next 1am fire even if a share hits the cap
|
||||||
|
RSYNC_MIN_FREE_GB=50 # warn if any disk backing the remote share is under this
|
||||||
|
# (0 disables). Online is not writable: a full destination
|
||||||
|
# makes rsync stall rather than fail. Warns, never aborts.
|
||||||
CRITICAL_CONTAINER_NAMES=() # containers stopped on REMOTE before rsync — profiles override
|
CRITICAL_CONTAINER_NAMES=() # containers stopped on REMOTE before rsync — profiles override
|
||||||
DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override
|
DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override
|
||||||
CONTAINER_DELAY=5 # seconds before starting delayed containers
|
CONTAINER_DELAY=5 # seconds before starting delayed containers
|
||||||
|
|||||||
@@ -1526,6 +1526,49 @@ check_remote_disks() {
|
|||||||
error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME"
|
error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# ── Free space, per backing disk ──────────────────────────────────────────────────────
|
||||||
|
# Online is not the same as writable. A share pinned by its allocation method to one disk
|
||||||
|
# reports plenty free at the share level while that disk is full, and rsync answers a full
|
||||||
|
# destination by STALLING rather than failing — it holds its lock and every later run of
|
||||||
|
# that tier declines to start. RSYNC_MAX_RUNTIME_HOURS bounds how long that lasts; this is
|
||||||
|
# the part that says so before the transfer rather than after.
|
||||||
|
#
|
||||||
|
# Warn-and-notify, never abort: rsync still moves whatever fits, and refusing every
|
||||||
|
# transfer because one backing disk is low is a worse outage than a bounded partial run.
|
||||||
|
local _min_free_gb="${RSYNC_MIN_FREE_GB:-50}"
|
||||||
|
if [[ "$_min_free_gb" -gt 0 ]] 2>/dev/null; then
|
||||||
|
local _paths=""
|
||||||
|
while IFS= read -r disk_name; do
|
||||||
|
[[ -n "$disk_name" ]] && _paths+=" /mnt/$disk_name/$share_name"
|
||||||
|
done <<< "$backing_disks"
|
||||||
|
while IFS= read -r pool_name; do
|
||||||
|
[[ -n "$pool_name" ]] && _paths+=" /mnt/$pool_name/$share_name"
|
||||||
|
done <<< "$zfs_pool_paths"
|
||||||
|
|
||||||
|
if [[ -n "${_paths// /}" ]]; then
|
||||||
|
# One round trip for every path — this runs before each transfer and the mesh link
|
||||||
|
# is the slow part. -BG so the unit is fixed regardless of the remote's df default.
|
||||||
|
local _df_out
|
||||||
|
_df_out=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
||||||
|
"for p in $_paths; do [ -d \"\$p\" ] && echo \"\$p \$(df -BG --output=avail \"\$p\" 2>/dev/null | tail -1 | tr -dc '0-9')\"; done" 2>/dev/null)
|
||||||
|
|
||||||
|
local _low=""
|
||||||
|
while IFS=' ' read -r _p _avail; do
|
||||||
|
[[ -z "$_p" || -z "$_avail" ]] && continue
|
||||||
|
if [[ "$_avail" -lt "$_min_free_gb" ]] 2>/dev/null; then
|
||||||
|
warn "$ICON_DISK $_p has ${_avail}G free — below ${_min_free_gb}G"
|
||||||
|
_low+="$_p (${_avail}G) "
|
||||||
|
fi
|
||||||
|
done <<< "$_df_out"
|
||||||
|
|
||||||
|
if [[ -n "$_low" ]]; then
|
||||||
|
notify "Low free space on $REMOTE_SERVER_NAME backing $share_name: $_low— rsync stalls rather than failing when a destination disk fills" \
|
||||||
|
"Varaverk: remote disk nearly full" "warning"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
success "All disks backing $share_name are online ✅"
|
success "All disks backing $share_name are online ✅"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user