187 lines
7.4 KiB
Bash
187 lines
7.4 KiB
Bash
#!/bin/bash
|
|
# -----------------------------------------------------------------------------------------------
|
|
# --------------------------------- Backup Verify ----------------------------------------------
|
|
# -----------------------------------------------------------------------------------------------
|
|
# Verifies the rsync mirror is healthy by comparing random file samples between
|
|
# local and remote servers using MD5 checksums.
|
|
#
|
|
# Randomly samples BACKUP_VERIFY_SAMPLE files per share, computes checksums locally,
|
|
# then computes the same checksums on the remote via SSH and compares results.
|
|
#
|
|
# Results per file:
|
|
# MATCH — checksums identical, file is correctly mirrored
|
|
# MISMATCH — file exists on both but checksums differ — sync may have failed
|
|
# MISSING — file exists locally but not on remote — not yet synced or deleted
|
|
#
|
|
# Silent when all files match. Notifies on any mismatch or missing file.
|
|
# Uses existing SSH keys — no additional configuration needed beyond share list.
|
|
#
|
|
# All configuration in Master.conf under Backup Verify section.
|
|
# Supports --dry-run to show what would be checked without running checksums.
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../Master.conf"
|
|
source "$SCRIPT_DIR/../common.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_GEAR Setup ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
|
|
|
detect_hosts
|
|
resolve_remote_ip
|
|
|
|
# Use BACKUP_VERIFY_SHARES if defined, fall back to DAILY_SYNC_SHARES
|
|
if [[ ${#BACKUP_VERIFY_SHARES[@]} -gt 0 ]]; then
|
|
VERIFY_SHARES=("${BACKUP_VERIFY_SHARES[@]}")
|
|
info "Using BACKUP_VERIFY_SHARES (${#VERIFY_SHARES[@]} shares)"
|
|
else
|
|
VERIFY_SHARES=("${DAILY_SYNC_SHARES[@]}")
|
|
info "BACKUP_VERIFY_SHARES not set — using DAILY_SYNC_SHARES (${#VERIFY_SHARES[@]} shares)"
|
|
fi
|
|
|
|
if [[ ${#VERIFY_SHARES[@]} -eq 0 ]]; then
|
|
warn "No shares configured — nothing to verify"
|
|
exit 0
|
|
fi
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SUMMARY Status ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_VERIFY Shares: ${#VERIFY_SHARES[@]}"
|
|
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share"
|
|
echo "$ICON_VERIFY Min size: $BACKUP_VERIFY_MIN_SIZE"
|
|
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
|
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing sample selection only, no checksums computed"
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_VERIFY Backup Verification ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_VERIFY Backup Verification — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
|
|
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share (min size: $BACKUP_VERIFY_MIN_SIZE)"
|
|
echo ""
|
|
|
|
START=$(date +%s)
|
|
TOTAL_CHECKED=0
|
|
TOTAL_MATCH=0
|
|
TOTAL_MISMATCH=0
|
|
TOTAL_MISSING=0
|
|
SHARES_WITH_ISSUES=()
|
|
|
|
for share in "${VERIFY_SHARES[@]}"; do
|
|
SHARE_NAME=$(basename "$share")
|
|
echo "━━━ $ICON_VERIFY $SHARE_NAME ━━━"
|
|
|
|
if [[ ! -d "$share" ]]; then
|
|
warn "$SHARE_NAME not found locally — skipping"
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
# Find files above minimum size and randomly sample
|
|
SAMPLE_FILES=$(find "$share" -type f -size +"$BACKUP_VERIFY_MIN_SIZE" 2>/dev/null | \
|
|
shuf | head -n "$BACKUP_VERIFY_SAMPLE")
|
|
|
|
SAMPLE_COUNT=$(echo "$SAMPLE_FILES" | grep -c "." 2>/dev/null || echo 0)
|
|
|
|
if [[ "$SAMPLE_COUNT" -eq 0 ]]; then
|
|
info "No files found above $BACKUP_VERIFY_MIN_SIZE — skipping"
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
info "Sampled $SAMPLE_COUNT files"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo "$SAMPLE_FILES" | while IFS= read -r f; do
|
|
warn "DRY RUN — would check: $(basename "$f")"
|
|
done
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
SHARE_MISMATCH=0
|
|
SHARE_MISSING=0
|
|
SHARE_MATCH=0
|
|
|
|
while IFS= read -r local_file; do
|
|
[[ -z "$local_file" ]] && continue
|
|
|
|
# Compute local checksum
|
|
local_md5=$(md5sum "$local_file" 2>/dev/null | awk '{print $1}')
|
|
|
|
if [[ -z "$local_md5" ]]; then
|
|
warn "Could not checksum: $local_file — skipping"
|
|
continue
|
|
fi
|
|
|
|
# Compute remote checksum via SSH
|
|
remote_md5=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
|
"md5sum '$local_file' 2>/dev/null | awk '{print \$1}'" 2>/dev/null)
|
|
|
|
((TOTAL_CHECKED++))
|
|
|
|
if [[ -z "$remote_md5" ]]; then
|
|
warn "$ICON_ERROR MISSING: $(basename "$local_file")"
|
|
((SHARE_MISSING++))
|
|
((TOTAL_MISSING++))
|
|
elif [[ "$local_md5" == "$remote_md5" ]]; then
|
|
log "MATCH: $(basename "$local_file")"
|
|
((SHARE_MATCH++))
|
|
((TOTAL_MATCH++))
|
|
else
|
|
error "$ICON_ERROR MISMATCH: $(basename "$local_file")"
|
|
((SHARE_MISMATCH++))
|
|
((TOTAL_MISMATCH++))
|
|
fi
|
|
|
|
done <<< "$SAMPLE_FILES"
|
|
|
|
echo " $ICON_SUCCESS Match: $SHARE_MATCH $ICON_WARN Missing: $SHARE_MISSING $ICON_ERROR Mismatch: $SHARE_MISMATCH"
|
|
|
|
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 ]]; then
|
|
SHARES_WITH_ISSUES+=("$SHARE_NAME")
|
|
fi
|
|
|
|
echo ""
|
|
done
|
|
|
|
END=$(date +%s)
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SUMMARY Summary ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo "━━━━━ $ICON_SUMMARY BACKUP VERIFY SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME"
|
|
echo "$ICON_VERIFY Checked: $TOTAL_CHECKED files"
|
|
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
|
|
echo "$ICON_WARN Missing: $TOTAL_MISSING"
|
|
echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
|
|
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo "$ICON_WARN Status: DRY RUN — no checksums computed"
|
|
elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
|
echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention"
|
|
notify "Backup verify failed on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" "Backup Verify" "warning"
|
|
else
|
|
echo "$ICON_DONE Status: $ICON_SUCCESS ALL FILES MATCH"
|
|
notify "Backup verify passed on $(hostname) → $REMOTE_SERVER_NAME — $TOTAL_CHECKED files checked across ${#VERIFY_SHARES[@]} shares" "Backup Verify" "normal"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" |