Fix rsync.sh global lock bug and add a max-runtime cap

acquire_lock (no args) ran before profile inference, so every rsync.sh
invocation — regardless of share — fought over one generic, unparameterized
lock. The per-profile acquire_rsync_lock() further down (with
RSYNC_MAX_CONCURRENT) never got a chance to matter: a single slow transfer
(e.g. Movies during the HOST2 rebuild) monopolized the lock and starved
every other profile, including Critical-Data's 30-minute sync, for days.

Removed the generic acquire_lock call; acquire_rsync_lock "$PROFILE_NAME"
already provides correct per-profile locking on its own.

Also added RSYNC_MAX_RUNTIME_HOURS (default 23): any single transfer
attempt exceeding it is terminated via timeout, logged as paused rather
than failed, and resumes from where it left off next scheduled run
(safe because --partial is already in DEFAULT_RSYNC_OPTS). Bounds the
worst case for one huge/stuck share instead of letting it hold its lock
indefinitely.

Deployment/master.conf.template

@@ -531,6 +531,7 @@
     BW_LIMIT=12500                  # KB/s — 12500 ≈ 100Mbit
     RETRY_COUNT=3                   # retry attempts before giving up
     SLEEP=300                       # seconds between retry attempts
+    RSYNC_MAX_RUNTIME_HOURS=23      # cap per transfer attempt — pauses and resumes next scheduled run
     CRITICAL_CONTAINER_NAMES=()     # containers stopped on REMOTE before rsync — profiles override
     DELAYED_CONTAINERS=()           # containers needing delay before starting — profiles override
     CONTAINER_DELAY=5               # seconds before starting delayed containers

Rsync/rsync.sh

@@ -98,6 +98,13 @@
 #   SLEEP
 #       Default seconds between retry attempts. (default: 60)
 #
+#   RSYNC_MAX_RUNTIME_HOURS
+#       Max hours a single transfer attempt may run before it's terminated and paused
+#       for the next scheduled run. Protects the per-profile lock from being held
+#       indefinitely by one huge/stuck transfer, starving other profiles of a turn.
+#       Safe because DEFAULT_RSYNC_OPTS includes --partial — a paused transfer resumes
+#       from where it left off, not from scratch. (default: 23)
+#
 #   ROOTFS_WARN
 #       Abort threshold for remote rootfs percentage full. (default: 75)
 #
@@ -177,8 +184,6 @@ if [[ "$EUID" -ne 0 ]]; then
     exit 1
 fi
 
-acquire_lock
-
 if ! command -v docker &>/dev/null; then
     error "Docker command not found"
     exit 1
@@ -232,6 +237,8 @@ BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
 RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
 SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
 CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY}
+RSYNC_MAX_RUNTIME_HOURS=${RSYNC_MAX_RUNTIME_HOURS:-23}
+RSYNC_MAX_RUNTIME_SECONDS=$(( RSYNC_MAX_RUNTIME_HOURS * 3600 ))
 
 read -r -a CRITICAL_CONTAINER_NAMES       <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]:-}"
 read -r -a DELAYED_CONTAINERS             <<< "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]:-}"
@@ -284,13 +291,15 @@ if [[ "$MERGE_RUN" == false && "$SEED" == false \
     && -z "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]+x}" ]]; then
     _remote_top=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
         root@"$REMOTE_SERVER" "ls -1A '$DIRECTORY' 2>/dev/null | sort" 2>/dev/null)
-    _remote_count=$(echo "$_remote_top" | grep -c . 2>/dev/null || echo 0)
+    _remote_count=$(echo "$_remote_top" | grep -c .)
+    _remote_count=${_remote_count:-0}
 
     if [[ "$_remote_count" -gt 0 ]]; then
         _local_top=$(ls -1A "$DIRECTORY" 2>/dev/null | sort)
         _overlap=$(comm -12 \
             <(echo "$_local_top") \
-            <(echo "$_remote_top") | grep -c . 2>/dev/null || echo 0)
+            <(echo "$_remote_top") | grep -c .)
+        _overlap=${_overlap:-0}
         _overlap_pct=$(( _overlap * 100 / _remote_count ))
 
         if [[ "$_overlap_pct" -ge 75 ]]; then
@@ -375,6 +384,9 @@ if [[ "$MERGE_RUN" == true ]]; then
         echo "$ICON_DONE Merge pass 1 complete — ${_pull_bytes:-0} bytes pulled"
     else
         warn "Merge pass 1 failed (exit $_pull_exit) — continuing with push"
+        echo "$_pull_output" | grep -iE "error|rsync:|permission denied" | while read -r _line; do
+            warn "  $_line"
+        done
     fi
     unset _pull_opts _pull_output _pull_exit _pull_bytes
     # Pass 2: push with --delete — local is now the authoritative superset
@@ -383,14 +395,15 @@ fi
 
 START=$(date +%s)
 RSYNC_SUCCESS=false
+RSYNC_TIMED_OUT=false
 BYTES_TRANSFERRED=0
 ATTEMPT=0
 
 for (( ATTEMPT=1; ATTEMPT<=RETRY_COUNT; ATTEMPT++ )); do
     log "$ICON_RETRY Attempt $ATTEMPT of $RETRY_COUNT..."
-    echo "$ICON_SYNC  Rsync running — this may take a while..."
+    echo "$ICON_SYNC  Rsync running — this may take a while (capped at ${RSYNC_MAX_RUNTIME_HOURS}h)..."
 
-    RSYNC_OUTPUT=$(rsync "${RSYNC_OPTS[@]}" \
+    RSYNC_OUTPUT=$(timeout "${RSYNC_MAX_RUNTIME_SECONDS}s" rsync "${RSYNC_OPTS[@]}" \
         -e "ssh -i $SSH_KEY -T -o Compression=no -o IPQoS=throughput" \
         "$DIRECTORY" "root@${REMOTE_SERVER}:$(dirname "$DIRECTORY")/" 2>&1)
 
@@ -405,6 +418,12 @@ for (( ATTEMPT=1; ATTEMPT<=RETRY_COUNT; ATTEMPT++ )); do
         echo "$ICON_DONE Rsync complete — $BYTES_TRANSFERRED bytes transferred"
         RSYNC_SUCCESS=true
         break
+    elif [[ "$RSYNC_EXIT" -eq 124 ]]; then
+        # Hit the runtime cap, not a failure — --partial means next run resumes from here.
+        # No retry: retrying an already-huge transfer 2 more times just wastes the window.
+        warn "$ICON_RETRY Rsync exceeded ${RSYNC_MAX_RUNTIME_HOURS}h cap — pausing, will resume next scheduled run"
+        RSYNC_TIMED_OUT=true
+        break
     else
         warn "$ICON_RETRY Rsync failed (attempt $ATTEMPT/$RETRY_COUNT)"
         log "Exit code: $RSYNC_EXIT"
@@ -491,6 +510,7 @@ BANDWIDTH_MONITOR="$SCRIPT_DIR/../Monitors/bandwidth_monitor.sh"
 if [[ "$DRY_RUN" == false ]] && [[ -f "$BANDWIDTH_MONITOR" ]]; then
     STATUS="success"
     [[ "$RSYNC_SUCCESS" == false ]] && STATUS="failed"
+    [[ "$RSYNC_TIMED_OUT" == true ]] && STATUS="timeout"
     bash "$BANDWIDTH_MONITOR" --log-transfer \
         "$PROFILE_NAME" "$DURATION" "$STATUS" "$BYTES_TRANSFERRED"
     log "$ICON_BANDWIDTH Transfer logged to bandwidth monitor ($BYTES_TRANSFERRED bytes)"
@@ -512,6 +532,8 @@ if [[ "$DRY_RUN" == true ]]; then
     warn "DRY RUN — no changes made"
 elif [[ "$RSYNC_SUCCESS" == true ]]; then
     echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
+elif [[ "$RSYNC_TIMED_OUT" == true ]]; then
+    echo "$ICON_DONE Status: PAUSED — exceeded ${RSYNC_MAX_RUNTIME_HOURS}h cap, resumes next scheduled run"
 else
     echo "$ICON_ERROR Status: FAILED after $RETRY_COUNT attempts"
     notify "Rsync FAILED — $DIRECTORY ($PROFILE_NAME) after $RETRY_COUNT attempts on $(hostname)" \
@@ -524,5 +546,5 @@ exec 1>&- 2>&-; wait
 cp "$VV_LIVE_LOG" "$VV_LAST_LOG" 2>/dev/null
 rm -f "$VV_LIVE_LOG"
 
-[[ "$RSYNC_SUCCESS" == false ]] && [[ "$DRY_RUN" == false ]] && exit 1
+[[ "$RSYNC_SUCCESS" == false ]] && [[ "$RSYNC_TIMED_OUT" == false ]] && [[ "$DRY_RUN" == false ]] && exit 1
 exit 0