diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index fba158b..ae0900d 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -693,6 +693,9 @@ # lowered from 23 on 2026-07-16 — daily maintenance jobs alone now # take ~4h; 19h cap leaves room for them to still run same-day # before the next 1am fire even if a share hits the cap + RSYNC_MIN_FREE_GB=50 # warn if any disk backing the remote share is under this + # (0 disables). Online is not writable: a full destination + # makes rsync stall rather than fail. Warns, never aborts. CRITICAL_CONTAINER_NAMES=() # containers stopped on REMOTE before rsync — profiles override DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override CONTAINER_DELAY=5 # seconds before starting delayed containers diff --git a/common.sh b/common.sh index ef8c2e7..ad2847d 100755 --- a/common.sh +++ b/common.sh @@ -1526,6 +1526,49 @@ check_remote_disks() { error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME" exit 1 fi + + # ── Free space, per backing disk ────────────────────────────────────────────────────── + # Online is not the same as writable. A share pinned by its allocation method to one disk + # reports plenty free at the share level while that disk is full, and rsync answers a full + # destination by STALLING rather than failing — it holds its lock and every later run of + # that tier declines to start. RSYNC_MAX_RUNTIME_HOURS bounds how long that lasts; this is + # the part that says so before the transfer rather than after. + # + # Warn-and-notify, never abort: rsync still moves whatever fits, and refusing every + # transfer because one backing disk is low is a worse outage than a bounded partial run. + local _min_free_gb="${RSYNC_MIN_FREE_GB:-50}" + if [[ "$_min_free_gb" -gt 0 ]] 2>/dev/null; then + local _paths="" + while IFS= read -r disk_name; do + [[ -n "$disk_name" ]] && _paths+=" /mnt/$disk_name/$share_name" + done <<< "$backing_disks" + while IFS= read -r pool_name; do + [[ -n "$pool_name" ]] && _paths+=" /mnt/$pool_name/$share_name" + done <<< "$zfs_pool_paths" + + if [[ -n "${_paths// /}" ]]; then + # One round trip for every path — this runs before each transfer and the mesh link + # is the slow part. -BG so the unit is fixed regardless of the remote's df default. + local _df_out + _df_out=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ + "for p in $_paths; do [ -d \"\$p\" ] && echo \"\$p \$(df -BG --output=avail \"\$p\" 2>/dev/null | tail -1 | tr -dc '0-9')\"; done" 2>/dev/null) + + local _low="" + while IFS=' ' read -r _p _avail; do + [[ -z "$_p" || -z "$_avail" ]] && continue + if [[ "$_avail" -lt "$_min_free_gb" ]] 2>/dev/null; then + warn "$ICON_DISK $_p has ${_avail}G free — below ${_min_free_gb}G" + _low+="$_p (${_avail}G) " + fi + done <<< "$_df_out" + + if [[ -n "$_low" ]]; then + notify "Low free space on $REMOTE_SERVER_NAME backing $share_name: $_low— rsync stalls rather than failing when a destination disk fills" \ + "Varaverk: remote disk nearly full" "warning" + fi + fi + fi + success "All disks backing $share_name are online ✅" }