#!/bin/bash # ============================================================================================== # ============================= Backup Verify ================================================== # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # rsync mirror integrity verification via independent MD5 checksums. Scheduled # weekly (Sunday 10am). Randomly samples BACKUP_VERIFY_SAMPLE files per share # above BACKUP_VERIFY_MIN_SIZE, computes checksums locally, then computes the # same checksums on the remote via SSH and compares. # # Per file: MATCH (checksums identical) | MISMATCH (file exists on both but # checksums differ — sync failure or corruption) | MISSING (file exists locally # but not on remote). All MISMATCHes and significant MISSINGs trigger notification. # rsync exit code 0 is not trusted — this script verifies actual content. # # Share list from HOST*_BACKUP_VERIFY_SHARES if defined, otherwise falls back # to HOST*_DAILY_SYNC_SHARES. Both aliased by detect_hosts(). # # ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # # 1. Pre-flight — connectivity to the remote, remote array mounted, version parity # 2. Resolve the share list (BACKUP_VERIFY_SHARES, else DAILY_SYNC_SHARES) # 3. Per share: # a. Randomly sample BACKUP_VERIFY_SAMPLE files above BACKUP_VERIFY_MIN_SIZE # b. Compute each file's MD5 locally # c. Compute the same file's MD5 on the remote over SSH # d. Classify: MATCH | MISMATCH | MISSING # 4. Report per-share and overall counts; notify on any MISMATCH and on # significant MISSING counts # # Sampling rather than full verification is deliberate — a complete checksum of every # mirrored file would take longer than the interval between runs. Random sampling over # a weekly cadence surfaces systematic corruption without ever reading the whole library. # # ============================================================================================== # DESIGN PRINCIPLES # ============================================================================================== # # Independent Verification # rsync reports success when the transfer completed without network errors and # file sizes and modification times match. It does not detect silent corruption # during transfer (bitflip in transit), corruption written to storage at rest # (faulty drive sector), or files that matched size/mtime but had wrong content. # All of these produce exit code 0. This script checks whether "done" means "correct." # # Intentionally Small Sample # 10 files per share (default) — a spot check, not an exhaustive verify. # Catches systematic problems and hardware issues while running in minutes, not # hours. Full verification would take longer than the rsync itself. # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # Single Instance Lock # acquire_lock prevents concurrent runs producing conflicting results. # # Remote Connectivity Check # check_connectivity() verifies the remote Tailscale IP is reachable before # any SSH calls. Without this, all files show as MISSING on a network hiccup. # # Remote Array Check # check_remote_array() verifies /mnt/user is mounted on the remote before # computing checksums. Array not started = all files "missing" = false alarm. # # Version Parity # Refuses to run if remote unRAID version doesn't match local. A mismatch # may mean the remote is in an unexpected state. # # SSH Timeout # SSH_TIMEOUT caps all SSH calls. One hung connection does not block the run. # # ============================================================================================== # CONFIGURATION # ============================================================================================== # # host*.conf # # HOST*_BACKUP_VERIFY_SHARES # Shares to verify. Leave empty to use HOST*_DAILY_SYNC_SHARES automatically. # Aliased by detect_hosts() → BACKUP_VERIFY_SHARES. # # HOST*_DAILY_SYNC_SHARES # Fallback share list if BACKUP_VERIFY_SHARES is empty. Aliased by detect_hosts(). # # master.conf # # BACKUP_VERIFY_SAMPLE # Random files checked per share per run. (default: 10) # # BACKUP_VERIFY_MIN_SIZE # Minimum file size to include in sample — tiny files have low corruption # risk and slow checksums. (default: 1M) # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # backup_verify.sh # Sample files from all shares and compare checksums. Notify on MISMATCH # or significant MISSING count. Silent when all samples match. # # backup_verify.sh --dry-run # Show which files would be sampled. No checksums computed, no notifications. # # backup_verify.sh --status # Show share list, sample size, and min file size configuration. Then exit. # # backup_verify.sh --log # Verbose per-file checksum comparison output during the run. # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" SSH_TIMEOUT=15 BACKUP_VERIFY_MD5_TIMEOUT_MAX="${BACKUP_VERIFY_MD5_TIMEOUT_MAX:-600}" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi acquire_lock # detect_hosts() sets MY_ID and aliases BACKUP_VERIFY_SHARES + DAILY_SYNC_SHARES detect_hosts require_partnership # Share selection — configured list or fallback to daily sync shares if [[ ${#BACKUP_VERIFY_SHARES[@]} -gt 0 ]]; then VERIFY_SHARES=("${BACKUP_VERIFY_SHARES[@]}") log "Using BACKUP_VERIFY_SHARES (${#VERIFY_SHARES[@]} shares)" else VERIFY_SHARES=("${DAILY_SYNC_SHARES[@]}") log "BACKUP_VERIFY_SHARES not set — using DAILY_SYNC_SHARES (${#VERIFY_SHARES[@]} shares)" fi if [[ ${#VERIFY_SHARES[@]} -eq 0 ]]; then warn "No shares configured for $MY_ID — nothing to verify" warn "Check HOST*_BACKUP_VERIFY_SHARES or HOST*_DAILY_SYNC_SHARES in host*.conf" exit 0 fi log "$ICON_GEAR Config: sample=${BACKUP_VERIFY_SAMPLE} min-size=${BACKUP_VERIFY_MIN_SIZE} ssh-timeout=${SSH_TIMEOUT}s" log "$ICON_GEAR Remote: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)" log "$ICON_GEAR Shares: ${VERIFY_SHARES[*]}" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing sample selection only, no checksums computed" # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_HOST Remote: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)" echo "$ICON_VERIFY Shares: ${#VERIFY_SHARES[@]}" echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share" echo "$ICON_VERIFY Min size: $BACKUP_VERIFY_MIN_SIZE" echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "" echo " Shares to verify:" for share in "${VERIFY_SHARES[@]}"; do echo " $share" done echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ━━━ Pre-flight ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_SHIELD Pre-flight ━━━" resolve_remote_ip # Connectivity — no point making 100+ SSH calls if remote is unreachable check_connectivity echo "Connectivity to $REMOTE_SERVER_NAME ✅" # Version parity — mismatched unRAID could cause md5sum path differences check_os_version_parity || { warn "Version parity check failed — proceeding with caution" warn "Checksum results may be unreliable if md5sum path changed between versions" } echo "Version parity with $REMOTE_SERVER_NAME ✅" # Remote array — if array is down all files appear "missing" = false alarm if ! check_remote_array; then error "Remote array not mounted on $REMOTE_SERVER_NAME" error "All files would appear as MISSING — aborting to prevent false alarm" notify "Backup verify aborted on $(hostname) — remote array not mounted on $REMOTE_SERVER_NAME" \ "Backup Verify" "warning" exit 1 fi echo "Remote array mounted on $REMOTE_SERVER_NAME ✅" echo "Pre-flight passed ✅" # ============================================================================================== # ━━━ Backup Verification ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_VERIFY Backup Verification — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → $REMOTE_ID ($REMOTE_SERVER_NAME)" echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share (min: $BACKUP_VERIFY_MIN_SIZE)" echo "" START=$(date +%s) TOTAL_CHECKED=0 TOTAL_MATCH=0 TOTAL_MISMATCH=0 TOTAL_MISSING=0 TOTAL_UNVERIFIED=0 SHARES_WITH_ISSUES=() for share in "${VERIFY_SHARES[@]}"; do SHARE_NAME=$(basename "$share") echo "━━━ $ICON_VERIFY $SHARE_NAME ━━━" if [[ ! -d "$share" ]]; then warn "$SHARE_NAME not found locally — skipping" echo "" continue fi # Sample random files above minimum size mapfile -t SAMPLE_FILES < <( find "$share" -type f -size +"$BACKUP_VERIFY_MIN_SIZE" 2>/dev/null | \ shuf | head -n "$BACKUP_VERIFY_SAMPLE" ) if [[ ${#SAMPLE_FILES[@]} -eq 0 ]]; then log "$SHARE_NAME — no files found above $BACKUP_VERIFY_MIN_SIZE" echo "" continue fi log "$SHARE_NAME — sampled ${#SAMPLE_FILES[@]} files" if [[ "$DRY_RUN" == true ]]; then for f in "${SAMPLE_FILES[@]}"; do warn "DRY RUN — would check: $(basename "$f")" done echo "" continue fi SHARE_MATCH=0 SHARE_MISMATCH=0 SHARE_MISSING=0 SHARE_UNVERIFIED=0 for local_file in "${SAMPLE_FILES[@]}"; do [[ -z "$local_file" ]] && continue # Local checksum local_md5=$(md5sum "$local_file" 2>/dev/null | awk '{print $1}') if [[ -z "$local_md5" ]]; then warn "Could not checksum locally: $(basename "$local_file") — skipping" continue fi # The path is interpolated into a remote shell command, so it must be escaped for # reuse as one word. A bare '$local_file' inside single quotes breaks on the first # apostrophe — "Frieren - Beyond Journey's End" ended the quote early, md5sum fell # back to reading stdin, and the empty-input hash d41d8cd9... was reported as a # MISMATCH against a file that is byte-identical on the remote. printf -v remote_q '%q' "$local_file" # Existence and content are separate questions. Asking them together means a slow # checksum is indistinguishable from an absent file. remote_exists=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" \ -o StrictHostKeyChecking=no \ root@"$REMOTE_SERVER" \ "test -f $remote_q && echo yes" 2>/dev/null /dev/null || echo 0) md5_timeout=$(( local_size / 52428800 + SSH_TIMEOUT )) (( md5_timeout > BACKUP_VERIFY_MD5_TIMEOUT_MAX )) && md5_timeout=$BACKUP_VERIFY_MD5_TIMEOUT_MAX remote_md5=$(timeout "$md5_timeout" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" \ -o StrictHostKeyChecking=no \ root@"$REMOTE_SERVER" \ "md5sum $remote_q 2>/dev/null | awk '{print \$1}'" 2>/dev/null