diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index db19f48..8cc76ce 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -531,6 +531,7 @@ BW_LIMIT=12500 # KB/s — 12500 ≈ 100Mbit RETRY_COUNT=3 # retry attempts before giving up SLEEP=300 # seconds between retry attempts + RSYNC_MAX_RUNTIME_HOURS=23 # cap per transfer attempt — pauses and resumes next scheduled run CRITICAL_CONTAINER_NAMES=() # containers stopped on REMOTE before rsync — profiles override DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override CONTAINER_DELAY=5 # seconds before starting delayed containers diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index fe223bf..7cd9e30 100755 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -98,6 +98,13 @@ # SLEEP # Default seconds between retry attempts. (default: 60) # +# RSYNC_MAX_RUNTIME_HOURS +# Max hours a single transfer attempt may run before it's terminated and paused +# for the next scheduled run. Protects the per-profile lock from being held +# indefinitely by one huge/stuck transfer, starving other profiles of a turn. +# Safe because DEFAULT_RSYNC_OPTS includes --partial — a paused transfer resumes +# from where it left off, not from scratch. (default: 23) +# # ROOTFS_WARN # Abort threshold for remote rootfs percentage full. (default: 75) # @@ -177,8 +184,6 @@ if [[ "$EUID" -ne 0 ]]; then exit 1 fi -acquire_lock - if ! command -v docker &>/dev/null; then error "Docker command not found" exit 1 @@ -232,6 +237,8 @@ BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT} RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT} SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP} CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY} +RSYNC_MAX_RUNTIME_HOURS=${RSYNC_MAX_RUNTIME_HOURS:-23} +RSYNC_MAX_RUNTIME_SECONDS=$(( RSYNC_MAX_RUNTIME_HOURS * 3600 )) read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]:-}" read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]:-}" @@ -284,13 +291,15 @@ if [[ "$MERGE_RUN" == false && "$SEED" == false \ && -z "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]+x}" ]]; then _remote_top=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \ root@"$REMOTE_SERVER" "ls -1A '$DIRECTORY' 2>/dev/null | sort" 2>/dev/null) - _remote_count=$(echo "$_remote_top" | grep -c . 2>/dev/null || echo 0) + _remote_count=$(echo "$_remote_top" | grep -c .) + _remote_count=${_remote_count:-0} if [[ "$_remote_count" -gt 0 ]]; then _local_top=$(ls -1A "$DIRECTORY" 2>/dev/null | sort) _overlap=$(comm -12 \ <(echo "$_local_top") \ - <(echo "$_remote_top") | grep -c . 2>/dev/null || echo 0) + <(echo "$_remote_top") | grep -c .) + _overlap=${_overlap:-0} _overlap_pct=$(( _overlap * 100 / _remote_count )) if [[ "$_overlap_pct" -ge 75 ]]; then @@ -375,6 +384,9 @@ if [[ "$MERGE_RUN" == true ]]; then echo "$ICON_DONE Merge pass 1 complete — ${_pull_bytes:-0} bytes pulled" else warn "Merge pass 1 failed (exit $_pull_exit) — continuing with push" + echo "$_pull_output" | grep -iE "error|rsync:|permission denied" | while read -r _line; do + warn " $_line" + done fi unset _pull_opts _pull_output _pull_exit _pull_bytes # Pass 2: push with --delete — local is now the authoritative superset @@ -383,14 +395,15 @@ fi START=$(date +%s) RSYNC_SUCCESS=false +RSYNC_TIMED_OUT=false BYTES_TRANSFERRED=0 ATTEMPT=0 for (( ATTEMPT=1; ATTEMPT<=RETRY_COUNT; ATTEMPT++ )); do log "$ICON_RETRY Attempt $ATTEMPT of $RETRY_COUNT..." - echo "$ICON_SYNC Rsync running — this may take a while..." + echo "$ICON_SYNC Rsync running — this may take a while (capped at ${RSYNC_MAX_RUNTIME_HOURS}h)..." - RSYNC_OUTPUT=$(rsync "${RSYNC_OPTS[@]}" \ + RSYNC_OUTPUT=$(timeout "${RSYNC_MAX_RUNTIME_SECONDS}s" rsync "${RSYNC_OPTS[@]}" \ -e "ssh -i $SSH_KEY -T -o Compression=no -o IPQoS=throughput" \ "$DIRECTORY" "root@${REMOTE_SERVER}:$(dirname "$DIRECTORY")/" 2>&1) @@ -405,6 +418,12 @@ for (( ATTEMPT=1; ATTEMPT<=RETRY_COUNT; ATTEMPT++ )); do echo "$ICON_DONE Rsync complete — $BYTES_TRANSFERRED bytes transferred" RSYNC_SUCCESS=true break + elif [[ "$RSYNC_EXIT" -eq 124 ]]; then + # Hit the runtime cap, not a failure — --partial means next run resumes from here. + # No retry: retrying an already-huge transfer 2 more times just wastes the window. + warn "$ICON_RETRY Rsync exceeded ${RSYNC_MAX_RUNTIME_HOURS}h cap — pausing, will resume next scheduled run" + RSYNC_TIMED_OUT=true + break else warn "$ICON_RETRY Rsync failed (attempt $ATTEMPT/$RETRY_COUNT)" log "Exit code: $RSYNC_EXIT" @@ -491,6 +510,7 @@ BANDWIDTH_MONITOR="$SCRIPT_DIR/../Monitors/bandwidth_monitor.sh" if [[ "$DRY_RUN" == false ]] && [[ -f "$BANDWIDTH_MONITOR" ]]; then STATUS="success" [[ "$RSYNC_SUCCESS" == false ]] && STATUS="failed" + [[ "$RSYNC_TIMED_OUT" == true ]] && STATUS="timeout" bash "$BANDWIDTH_MONITOR" --log-transfer \ "$PROFILE_NAME" "$DURATION" "$STATUS" "$BYTES_TRANSFERRED" log "$ICON_BANDWIDTH Transfer logged to bandwidth monitor ($BYTES_TRANSFERRED bytes)" @@ -512,6 +532,8 @@ if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" elif [[ "$RSYNC_SUCCESS" == true ]]; then echo "$ICON_DONE Status: $ICON_SUCCESS DONE" +elif [[ "$RSYNC_TIMED_OUT" == true ]]; then + echo "$ICON_DONE Status: PAUSED — exceeded ${RSYNC_MAX_RUNTIME_HOURS}h cap, resumes next scheduled run" else echo "$ICON_ERROR Status: FAILED after $RETRY_COUNT attempts" notify "Rsync FAILED — $DIRECTORY ($PROFILE_NAME) after $RETRY_COUNT attempts on $(hostname)" \ @@ -524,5 +546,5 @@ exec 1>&- 2>&-; wait cp "$VV_LIVE_LOG" "$VV_LAST_LOG" 2>/dev/null rm -f "$VV_LIVE_LOG" -[[ "$RSYNC_SUCCESS" == false ]] && [[ "$DRY_RUN" == false ]] && exit 1 +[[ "$RSYNC_SUCCESS" == false ]] && [[ "$RSYNC_TIMED_OUT" == false ]] && [[ "$DRY_RUN" == false ]] && exit 1 exit 0 \ No newline at end of file