diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 07a2b76..0379904 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -1656,6 +1656,10 @@ # HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES BACKUP_VERIFY_SAMPLE=10 # random files to check per share BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample + BACKUP_VERIFY_MD5_TIMEOUT_MAX=600 # ceiling for one remote checksum. The per-file budget + # scales with size (~50MB/s); this caps it. A fixed + # connect-sized timeout killed multi-GB checksums and + # the empty result was then reported as MISSING. # ━━━ SMART Health ━━━ # Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*. diff --git a/Monitors/backup_verify.sh b/Monitors/backup_verify.sh old mode 100755 new mode 100644 index d5a4ebf..c43b17f --- a/Monitors/backup_verify.sh +++ b/Monitors/backup_verify.sh @@ -122,6 +122,7 @@ source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" SSH_TIMEOUT=15 +BACKUP_VERIFY_MD5_TIMEOUT_MAX="${BACKUP_VERIFY_MD5_TIMEOUT_MAX:-600}" # ============================================================================================== # ━━━ Setup ━━━ @@ -228,6 +229,7 @@ TOTAL_CHECKED=0 TOTAL_MATCH=0 TOTAL_MISMATCH=0 TOTAL_MISSING=0 +TOTAL_UNVERIFIED=0 SHARES_WITH_ISSUES=() for share in "${VERIFY_SHARES[@]}"; do @@ -265,6 +267,7 @@ for share in "${VERIFY_SHARES[@]}"; do SHARE_MATCH=0 SHARE_MISMATCH=0 SHARE_MISSING=0 + SHARE_UNVERIFIED=0 for local_file in "${SAMPLE_FILES[@]}"; do [[ -z "$local_file" ]] && continue @@ -276,19 +279,49 @@ for share in "${VERIFY_SHARES[@]}"; do continue fi - # Remote checksum via SSH — timeout protected - remote_md5=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + # The path is interpolated into a remote shell command, so it must be escaped for + # reuse as one word. A bare '$local_file' inside single quotes breaks on the first + # apostrophe — "Frieren - Beyond Journey's End" ended the quote early, md5sum fell + # back to reading stdin, and the empty-input hash d41d8cd9... was reported as a + # MISMATCH against a file that is byte-identical on the remote. + printf -v remote_q '%q' "$local_file" + + # Existence and content are separate questions. Asking them together means a slow + # checksum is indistinguishable from an absent file. + remote_exists=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" \ -o StrictHostKeyChecking=no \ root@"$REMOTE_SERVER" \ - "md5sum '$local_file' 2>/dev/null | awk '{print \$1}'" 2>/dev/null) + "test -f $remote_q && echo yes" 2>/dev/null /dev/null || echo 0) + md5_timeout=$(( local_size / 52428800 + SSH_TIMEOUT )) + (( md5_timeout > BACKUP_VERIFY_MD5_TIMEOUT_MAX )) && md5_timeout=$BACKUP_VERIFY_MD5_TIMEOUT_MAX + + remote_md5=$(timeout "$md5_timeout" ssh -i "$SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" \ + -o StrictHostKeyChecking=no \ + root@"$REMOTE_SERVER" \ + "md5sum $remote_q 2>/dev/null | awk '{print \$1}'" 2>/dev/null