massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2
This commit is contained in:
+173
-88
@@ -1,79 +1,152 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Backup Verify ----------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ================================= Backup Verify ==============================================
|
||||
# ==============================================================================================
|
||||
# Verifies the rsync mirror is healthy by comparing random file samples between
|
||||
# local and remote servers using MD5 checksums.
|
||||
#
|
||||
# Randomly samples BACKUP_VERIFY_SAMPLE files per share, computes checksums locally,
|
||||
# then computes the same checksums on the remote via SSH and compares results.
|
||||
# ── WHAT IT DOES ──────────────────────────────────────────────────────────────────────────────
|
||||
# Randomly samples BACKUP_VERIFY_SAMPLE files per share above BACKUP_VERIFY_MIN_SIZE,
|
||||
# computes MD5 checksums locally, then computes the same checksums on the remote via SSH
|
||||
# and compares results. Catches silent corruption or incomplete syncs that rsync itself
|
||||
# would not detect.
|
||||
#
|
||||
# Results per file:
|
||||
# MATCH — checksums identical, file is correctly mirrored
|
||||
# MISMATCH — file exists on both but checksums differ — sync may have failed
|
||||
# MISSING — file exists locally but not on remote — not yet synced or deleted
|
||||
# ── RESULTS PER FILE ──────────────────────────────────────────────────────────────────────────
|
||||
# MATCH — checksums identical, file is correctly mirrored ✅
|
||||
# MISMATCH — file exists on both but checksums differ — sync may have partially failed
|
||||
# MISSING — file exists locally but not on remote — not yet synced or deleted on remote
|
||||
#
|
||||
# Silent when all files match. Notifies on any mismatch or missing file.
|
||||
# Uses existing SSH keys — no additional configuration needed beyond share list.
|
||||
# ── SHARE SELECTION ───────────────────────────────────────────────────────────────────────────
|
||||
# Uses HOST*_BACKUP_VERIFY_SHARES if defined, falls back to HOST*_DAILY_SYNC_SHARES.
|
||||
# Both aliased by detect_hosts() — no manual HOST1/HOST2 selection needed.
|
||||
#
|
||||
# All configuration in Master.conf under Backup Verify section.
|
||||
# Supports --dry-run to show what would be checked without running checksums.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — prevents concurrent runs producing duplicate/conflicting results
|
||||
# check_connectivity() — verifies remote reachable before attempting SSH calls
|
||||
# check_remote_array() — verifies remote array mounted before checksums
|
||||
# remote array down = all files "missing" = false alarm ✅
|
||||
# version parity — verifies both servers on compatible unRAID before trusting results
|
||||
# SSH_TIMEOUT — all SSH calls protected against hangs
|
||||
# validate_unraid_cmd — notify script validated before use
|
||||
# Silent by default — only issues produce output, all-match runs are silent
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_BACKUP_VERIFY_SHARES — override share list (empty = use DAILY_SYNC_SHARES)
|
||||
# HOST*_DAILY_SYNC_SHARES — fallback share list
|
||||
# All aliased by detect_hosts()
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# BACKUP_VERIFY_SAMPLE — random files to check per share (default 10)
|
||||
# BACKUP_VERIFY_MIN_SIZE — minimum file size to include in sample (default 1M)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# backup_verify.sh — normal run
|
||||
# backup_verify.sh --dry-run — show sample selection only, no checksums
|
||||
# backup_verify.sh --log — verbose output
|
||||
# backup_verify.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
SSH_TIMEOUT=15
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases BACKUP_VERIFY_SHARES + DAILY_SYNC_SHARES
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
# Use BACKUP_VERIFY_SHARES if defined, fall back to DAILY_SYNC_SHARES
|
||||
# Share selection — configured list or fallback to daily sync shares
|
||||
if [[ ${#BACKUP_VERIFY_SHARES[@]} -gt 0 ]]; then
|
||||
VERIFY_SHARES=("${BACKUP_VERIFY_SHARES[@]}")
|
||||
info "Using BACKUP_VERIFY_SHARES (${#VERIFY_SHARES[@]} shares)"
|
||||
log "Using BACKUP_VERIFY_SHARES (${#VERIFY_SHARES[@]} shares)"
|
||||
else
|
||||
VERIFY_SHARES=("${DAILY_SYNC_SHARES[@]}")
|
||||
info "BACKUP_VERIFY_SHARES not set — using DAILY_SYNC_SHARES (${#VERIFY_SHARES[@]} shares)"
|
||||
log "BACKUP_VERIFY_SHARES not set — using DAILY_SYNC_SHARES (${#VERIFY_SHARES[@]} shares)"
|
||||
fi
|
||||
|
||||
if [[ ${#VERIFY_SHARES[@]} -eq 0 ]]; then
|
||||
warn "No shares configured — nothing to verify"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_VERIFY Shares: ${#VERIFY_SHARES[@]}"
|
||||
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share"
|
||||
echo "$ICON_VERIFY Min size: $BACKUP_VERIFY_MIN_SIZE"
|
||||
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
warn "No shares configured for $MY_ID — nothing to verify"
|
||||
warn "Check HOST*_BACKUP_VERIFY_SHARES or HOST*_DAILY_SYNC_SHARES in master_host*.conf"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing sample selection only, no checksums computed"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_VERIFY Backup Verification ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HOST Remote: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
|
||||
echo "$ICON_VERIFY Shares: ${#VERIFY_SHARES[@]}"
|
||||
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share"
|
||||
echo "$ICON_VERIFY Min size: $BACKUP_VERIFY_MIN_SIZE"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo ""
|
||||
echo " Shares to verify:"
|
||||
for share in "${VERIFY_SHARES[@]}"; do
|
||||
echo " $share"
|
||||
done
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||||
|
||||
# Connectivity — no point making 100+ SSH calls if remote is unreachable
|
||||
check_connectivity
|
||||
|
||||
# Version parity — mismatched unRAID could cause md5sum path differences
|
||||
check_unraid_version_parity || {
|
||||
warn "Version parity check failed — proceeding with caution"
|
||||
warn "Checksum results may be unreliable if md5sum path changed between versions"
|
||||
}
|
||||
|
||||
# Remote array — if array is down all files appear "missing" = false alarm
|
||||
if ! check_remote_array; then
|
||||
error "Remote array not mounted on $REMOTE_SERVER_NAME"
|
||||
error "All files would appear as MISSING — aborting to prevent false alarm"
|
||||
notify "Backup verify aborted on $(hostname) — remote array not mounted on $REMOTE_SERVER_NAME" \
|
||||
"Backup Verify" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Pre-flight passed ✅"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Backup Verification ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_VERIFY Backup Verification — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
|
||||
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share (min size: $BACKUP_VERIFY_MIN_SIZE)"
|
||||
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share (min: $BACKUP_VERIFY_MIN_SIZE)"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
@@ -93,69 +166,74 @@ for share in "${VERIFY_SHARES[@]}"; do
|
||||
continue
|
||||
fi
|
||||
|
||||
# Find files above minimum size and randomly sample
|
||||
SAMPLE_FILES=$(find "$share" -type f -size +"$BACKUP_VERIFY_MIN_SIZE" 2>/dev/null | \
|
||||
shuf | head -n "$BACKUP_VERIFY_SAMPLE")
|
||||
# Sample random files above minimum size
|
||||
mapfile -t SAMPLE_FILES < <(
|
||||
find "$share" -type f -size +"$BACKUP_VERIFY_MIN_SIZE" 2>/dev/null | \
|
||||
shuf | head -n "$BACKUP_VERIFY_SAMPLE"
|
||||
)
|
||||
|
||||
SAMPLE_COUNT=$(echo "$SAMPLE_FILES" | grep -c "." 2>/dev/null || echo 0)
|
||||
|
||||
if [[ "$SAMPLE_COUNT" -eq 0 ]]; then
|
||||
info "No files found above $BACKUP_VERIFY_MIN_SIZE — skipping"
|
||||
if [[ ${#SAMPLE_FILES[@]} -eq 0 ]]; then
|
||||
log "$SHARE_NAME — no files found above $BACKUP_VERIFY_MIN_SIZE"
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
info "Sampled $SAMPLE_COUNT files"
|
||||
log "$SHARE_NAME — sampled ${#SAMPLE_FILES[@]} files"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$SAMPLE_FILES" | while IFS= read -r f; do
|
||||
for f in "${SAMPLE_FILES[@]}"; do
|
||||
warn "DRY RUN — would check: $(basename "$f")"
|
||||
done
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
SHARE_MATCH=0
|
||||
SHARE_MISMATCH=0
|
||||
SHARE_MISSING=0
|
||||
SHARE_MATCH=0
|
||||
|
||||
while IFS= read -r local_file; do
|
||||
for local_file in "${SAMPLE_FILES[@]}"; do
|
||||
[[ -z "$local_file" ]] && continue
|
||||
|
||||
# Compute local checksum
|
||||
# Local checksum
|
||||
local_md5=$(md5sum "$local_file" 2>/dev/null | awk '{print $1}')
|
||||
|
||||
if [[ -z "$local_md5" ]]; then
|
||||
warn "Could not checksum: $local_file — skipping"
|
||||
warn "Could not checksum locally: $(basename "$local_file") — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Compute remote checksum via SSH
|
||||
remote_md5=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
||||
# Remote checksum via SSH — timeout protected
|
||||
remote_md5=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o StrictHostKeyChecking=no \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"md5sum '$local_file' 2>/dev/null | awk '{print \$1}'" 2>/dev/null)
|
||||
|
||||
((TOTAL_CHECKED++))
|
||||
(( TOTAL_CHECKED++ ))
|
||||
|
||||
if [[ -z "$remote_md5" ]]; then
|
||||
warn "$ICON_ERROR MISSING: $(basename "$local_file")"
|
||||
((SHARE_MISSING++))
|
||||
((TOTAL_MISSING++))
|
||||
(( SHARE_MISSING++ ))
|
||||
(( TOTAL_MISSING++ ))
|
||||
elif [[ "$local_md5" == "$remote_md5" ]]; then
|
||||
log "MATCH: $(basename "$local_file")"
|
||||
((SHARE_MATCH++))
|
||||
((TOTAL_MATCH++))
|
||||
(( SHARE_MATCH++ ))
|
||||
(( TOTAL_MATCH++ ))
|
||||
else
|
||||
error "$ICON_ERROR MISMATCH: $(basename "$local_file")"
|
||||
((SHARE_MISMATCH++))
|
||||
((TOTAL_MISMATCH++))
|
||||
error "MISMATCH: $(basename "$local_file")"
|
||||
error " local: $local_md5"
|
||||
error " remote: $remote_md5"
|
||||
(( SHARE_MISMATCH++ ))
|
||||
(( TOTAL_MISMATCH++ ))
|
||||
fi
|
||||
done
|
||||
|
||||
done <<< "$SAMPLE_FILES"
|
||||
|
||||
echo " $ICON_SUCCESS Match: $SHARE_MATCH $ICON_WARN Missing: $SHARE_MISSING $ICON_ERROR Mismatch: $SHARE_MISMATCH"
|
||||
|
||||
# Per-share result — only visible if issues found
|
||||
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 ]]; then
|
||||
warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH"
|
||||
SHARES_WITH_ISSUES+=("$SHARE_NAME")
|
||||
else
|
||||
log "$SHARE_NAME — all $SHARE_MATCH files match ✅"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
@@ -163,25 +241,32 @@ done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo "━━━━━ $ICON_SUMMARY BACKUP VERIFY SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME"
|
||||
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HOST Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_VERIFY Checked: $TOTAL_CHECKED files"
|
||||
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
|
||||
echo "$ICON_WARN Missing: $TOTAL_MISSING"
|
||||
echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no checksums computed"
|
||||
elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention"
|
||||
notify "Backup verify failed on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" "Backup Verify" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL FILES MATCH"
|
||||
notify "Backup verify passed on $(hostname) → $REMOTE_SERVER_NAME — $TOTAL_CHECKED files checked across ${#VERIFY_SHARES[@]} shares" "Backup Verify" "normal"
|
||||
if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
||||
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
|
||||
warn "Missing: $TOTAL_MISSING"
|
||||
[[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no checksums computed"
|
||||
elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention: ${SHARES_WITH_ISSUES[*]}"
|
||||
notify "Backup verify FAILED on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" \
|
||||
"Backup Verify" "warning"
|
||||
else
|
||||
log "$ICON_DONE Status: all $TOTAL_CHECKED files match across ${#VERIFY_SHARES[@]} shares ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$TOTAL_MISMATCH" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
Reference in New Issue
Block a user