330 lines
13 KiB
Bash
Executable File
330 lines
13 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Backup Verify ==================================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# rsync mirror integrity verification via independent MD5 checksums. Scheduled
|
|
# weekly (Sunday 10am). Randomly samples BACKUP_VERIFY_SAMPLE files per share
|
|
# above BACKUP_VERIFY_MIN_SIZE, computes checksums locally, then computes the
|
|
# same checksums on the remote via SSH and compares.
|
|
#
|
|
# Per file: MATCH (checksums identical) | MISMATCH (file exists on both but
|
|
# checksums differ — sync failure or corruption) | MISSING (file exists locally
|
|
# but not on remote). All MISMATCHes and significant MISSINGs trigger notification.
|
|
# rsync exit code 0 is not trusted — this script verifies actual content.
|
|
#
|
|
# Share list from HOST*_BACKUP_VERIFY_SHARES if defined, otherwise falls back
|
|
# to HOST*_DAILY_SYNC_SHARES. Both aliased by detect_hosts().
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Independent Verification
|
|
# rsync reports success when the transfer completed without network errors and
|
|
# file sizes and modification times match. It does not detect silent corruption
|
|
# during transfer (bitflip in transit), corruption written to storage at rest
|
|
# (faulty drive sector), or files that matched size/mtime but had wrong content.
|
|
# All of these produce exit code 0. This script checks whether "done" means "correct."
|
|
#
|
|
# Intentionally Small Sample
|
|
# 10 files per share (default) — a spot check, not an exhaustive verify.
|
|
# Catches systematic problems and hardware issues while running in minutes, not
|
|
# hours. Full verification would take longer than the rsync itself.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Single Instance Lock
|
|
# acquire_lock prevents concurrent runs producing conflicting results.
|
|
#
|
|
# Remote Connectivity Check
|
|
# check_connectivity() verifies the remote Tailscale IP is reachable before
|
|
# any SSH calls. Without this, all files show as MISSING on a network hiccup.
|
|
#
|
|
# Remote Array Check
|
|
# check_remote_array() verifies /mnt/user is mounted on the remote before
|
|
# computing checksums. Array not started = all files "missing" = false alarm.
|
|
#
|
|
# Version Parity
|
|
# Refuses to run if remote unRAID version doesn't match local. A mismatch
|
|
# may mean the remote is in an unexpected state.
|
|
#
|
|
# SSH Timeout
|
|
# SSH_TIMEOUT caps all SSH calls. One hung connection does not block the run.
|
|
#
|
|
# Notification Validated
|
|
# platform_require_cmd confirms the notify script is present before use.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# host*.conf
|
|
#
|
|
# HOST*_BACKUP_VERIFY_SHARES
|
|
# Shares to verify. Leave empty to use HOST*_DAILY_SYNC_SHARES automatically.
|
|
# Aliased by detect_hosts() → BACKUP_VERIFY_SHARES.
|
|
#
|
|
# HOST*_DAILY_SYNC_SHARES
|
|
# Fallback share list if BACKUP_VERIFY_SHARES is empty. Aliased by detect_hosts().
|
|
#
|
|
# master.conf
|
|
#
|
|
# BACKUP_VERIFY_SAMPLE
|
|
# Random files checked per share per run. (default: 10)
|
|
#
|
|
# BACKUP_VERIFY_MIN_SIZE
|
|
# Minimum file size to include in sample — tiny files have low corruption
|
|
# risk and slow checksums. (default: 1M)
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# backup_verify.sh
|
|
# Sample files from all shares and compare checksums. Notify on MISMATCH
|
|
# or significant MISSING count. Silent when all samples match.
|
|
#
|
|
# backup_verify.sh --dry-run
|
|
# Show which files would be sampled. No checksums computed, no notifications.
|
|
#
|
|
# backup_verify.sh --status
|
|
# Show share list, sample size, and min file size configuration. Then exit.
|
|
#
|
|
# backup_verify.sh --log
|
|
# Verbose per-file checksum comparison output during the run.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
SSH_TIMEOUT=15
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
|
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
|
|
acquire_lock
|
|
|
|
# detect_hosts() sets MY_ID and aliases BACKUP_VERIFY_SHARES + DAILY_SYNC_SHARES
|
|
detect_hosts
|
|
|
|
# Share selection — configured list or fallback to daily sync shares
|
|
if [[ ${#BACKUP_VERIFY_SHARES[@]} -gt 0 ]]; then
|
|
VERIFY_SHARES=("${BACKUP_VERIFY_SHARES[@]}")
|
|
log "Using BACKUP_VERIFY_SHARES (${#VERIFY_SHARES[@]} shares)"
|
|
else
|
|
VERIFY_SHARES=("${DAILY_SYNC_SHARES[@]}")
|
|
log "BACKUP_VERIFY_SHARES not set — using DAILY_SYNC_SHARES (${#VERIFY_SHARES[@]} shares)"
|
|
fi
|
|
|
|
if [[ ${#VERIFY_SHARES[@]} -eq 0 ]]; then
|
|
warn "No shares configured for $MY_ID — nothing to verify"
|
|
warn "Check HOST*_BACKUP_VERIFY_SHARES or HOST*_DAILY_SYNC_SHARES in host*.conf"
|
|
exit 0
|
|
fi
|
|
|
|
log "$ICON_GEAR Config: sample=${BACKUP_VERIFY_SAMPLE} min-size=${BACKUP_VERIFY_MIN_SIZE} ssh-timeout=${SSH_TIMEOUT}s"
|
|
log "$ICON_GEAR Remote: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
|
|
log "$ICON_GEAR Shares: ${VERIFY_SHARES[*]}"
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing sample selection only, no checksums computed"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_HOST Remote: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
|
|
echo "$ICON_VERIFY Shares: ${#VERIFY_SHARES[@]}"
|
|
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share"
|
|
echo "$ICON_VERIFY Min size: $BACKUP_VERIFY_MIN_SIZE"
|
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
|
echo ""
|
|
echo " Shares to verify:"
|
|
for share in "${VERIFY_SHARES[@]}"; do
|
|
echo " $share"
|
|
done
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Pre-flight ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
|
|
|
resolve_remote_ip
|
|
|
|
# Connectivity — no point making 100+ SSH calls if remote is unreachable
|
|
check_connectivity
|
|
log "Connectivity to $REMOTE_SERVER_NAME ✅"
|
|
|
|
# Version parity — mismatched unRAID could cause md5sum path differences
|
|
check_unraid_version_parity || {
|
|
warn "Version parity check failed — proceeding with caution"
|
|
warn "Checksum results may be unreliable if md5sum path changed between versions"
|
|
}
|
|
log "Version parity with $REMOTE_SERVER_NAME ✅"
|
|
|
|
# Remote array — if array is down all files appear "missing" = false alarm
|
|
if ! check_remote_array; then
|
|
error "Remote array not mounted on $REMOTE_SERVER_NAME"
|
|
error "All files would appear as MISSING — aborting to prevent false alarm"
|
|
notify "Backup verify aborted on $(hostname) — remote array not mounted on $REMOTE_SERVER_NAME" \
|
|
"Backup Verify" "warning"
|
|
exit 1
|
|
fi
|
|
log "Remote array mounted on $REMOTE_SERVER_NAME ✅"
|
|
|
|
echo "Pre-flight passed ✅"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Backup Verification ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_VERIFY Backup Verification — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
|
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share (min: $BACKUP_VERIFY_MIN_SIZE)"
|
|
echo ""
|
|
|
|
START=$(date +%s)
|
|
TOTAL_CHECKED=0
|
|
TOTAL_MATCH=0
|
|
TOTAL_MISMATCH=0
|
|
TOTAL_MISSING=0
|
|
SHARES_WITH_ISSUES=()
|
|
|
|
for share in "${VERIFY_SHARES[@]}"; do
|
|
SHARE_NAME=$(basename "$share")
|
|
echo "━━━ $ICON_VERIFY $SHARE_NAME ━━━"
|
|
|
|
if [[ ! -d "$share" ]]; then
|
|
warn "$SHARE_NAME not found locally — skipping"
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
# Sample random files above minimum size
|
|
mapfile -t SAMPLE_FILES < <(
|
|
find "$share" -type f -size +"$BACKUP_VERIFY_MIN_SIZE" 2>/dev/null | \
|
|
shuf | head -n "$BACKUP_VERIFY_SAMPLE"
|
|
)
|
|
|
|
if [[ ${#SAMPLE_FILES[@]} -eq 0 ]]; then
|
|
log "$SHARE_NAME — no files found above $BACKUP_VERIFY_MIN_SIZE"
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
log "$SHARE_NAME — sampled ${#SAMPLE_FILES[@]} files"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
for f in "${SAMPLE_FILES[@]}"; do
|
|
warn "DRY RUN — would check: $(basename "$f")"
|
|
done
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
SHARE_MATCH=0
|
|
SHARE_MISMATCH=0
|
|
SHARE_MISSING=0
|
|
|
|
for local_file in "${SAMPLE_FILES[@]}"; do
|
|
[[ -z "$local_file" ]] && continue
|
|
|
|
# Local checksum
|
|
local_md5=$(md5sum "$local_file" 2>/dev/null | awk '{print $1}')
|
|
if [[ -z "$local_md5" ]]; then
|
|
warn "Could not checksum locally: $(basename "$local_file") — skipping"
|
|
continue
|
|
fi
|
|
|
|
# Remote checksum via SSH — timeout protected
|
|
remote_md5=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" \
|
|
-o StrictHostKeyChecking=no \
|
|
root@"$REMOTE_SERVER" \
|
|
"md5sum '$local_file' 2>/dev/null | awk '{print \$1}'" 2>/dev/null)
|
|
|
|
(( TOTAL_CHECKED++ ))
|
|
|
|
if [[ -z "$remote_md5" ]]; then
|
|
warn "$ICON_ERROR MISSING: $(basename "$local_file")"
|
|
(( SHARE_MISSING++ ))
|
|
(( TOTAL_MISSING++ ))
|
|
elif [[ "$local_md5" == "$remote_md5" ]]; then
|
|
log "MATCH: $(basename "$local_file")"
|
|
(( SHARE_MATCH++ ))
|
|
(( TOTAL_MATCH++ ))
|
|
else
|
|
error "MISMATCH: $(basename "$local_file")"
|
|
error " local: $local_md5"
|
|
error " remote: $remote_md5"
|
|
(( SHARE_MISMATCH++ ))
|
|
(( TOTAL_MISMATCH++ ))
|
|
fi
|
|
done
|
|
|
|
# Per-share result — only visible if issues found
|
|
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 ]]; then
|
|
warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH"
|
|
SHARES_WITH_ISSUES+=("$SHARE_NAME")
|
|
else
|
|
log "$SHARE_NAME — all $SHARE_MATCH files match ✅"
|
|
fi
|
|
|
|
echo ""
|
|
done
|
|
|
|
END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo "━━━━━ $ICON_SUMMARY BACKUP VERIFY SUMMARY ━━━━━"
|
|
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_HOST Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
|
echo "$ICON_VERIFY Checked: $TOTAL_CHECKED files"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
echo ""
|
|
|
|
if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
|
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
|
|
warn "Missing: $TOTAL_MISSING"
|
|
[[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no checksums computed"
|
|
elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
|
echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention: ${SHARES_WITH_ISSUES[*]}"
|
|
notify "Backup verify FAILED on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" \
|
|
"Backup Verify" "warning"
|
|
else
|
|
echo "$ICON_DONE Status: all $TOTAL_CHECKED files match across ${#VERIFY_SHARES[@]} shares ✅"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ "$TOTAL_MISMATCH" -gt 0 ]] && exit 1
|
|
exit 0 |