Varaverk/Monitors/backup_verify.sh

#!/bin/bash
# ==============================================================================================
# ============================= Backup Verify ==================================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# rsync mirror integrity verification via independent MD5 checksums. Scheduled
# weekly (Sunday 10am). Randomly samples BACKUP_VERIFY_SAMPLE files per share
# above BACKUP_VERIFY_MIN_SIZE, computes checksums locally, then computes the
# same checksums on the remote via SSH and compares.
#
# Per file: MATCH (checksums identical) | MISMATCH (file exists on both but
# checksums differ — sync failure or corruption) | MISSING (file exists locally
# but not on remote). All MISMATCHes and significant MISSINGs trigger notification.
# rsync exit code 0 is not trusted — this script verifies actual content.
#
# Share list from HOST*_BACKUP_VERIFY_SHARES if defined, otherwise falls back
# to HOST*_DAILY_SYNC_SHARES. Both aliased by detect_hosts().
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Independent Verification
#   rsync reports success when the transfer completed without network errors and
#   file sizes and modification times match. It does not detect silent corruption
#   during transfer (bitflip in transit), corruption written to storage at rest
#   (faulty drive sector), or files that matched size/mtime but had wrong content.
#   All of these produce exit code 0. This script checks whether "done" means "correct."
#
# Intentionally Small Sample
#   10 files per share (default) — a spot check, not an exhaustive verify.
#   Catches systematic problems and hardware issues while running in minutes, not
#   hours. Full verification would take longer than the rsync itself.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Single Instance Lock
#   acquire_lock prevents concurrent runs producing conflicting results.
#
# Remote Connectivity Check
#   check_connectivity() verifies the remote Tailscale IP is reachable before
#   any SSH calls. Without this, all files show as MISSING on a network hiccup.
#
# Remote Array Check
#   check_remote_array() verifies /mnt/user is mounted on the remote before
#   computing checksums. Array not started = all files "missing" = false alarm.
#
# Version Parity
#   Refuses to run if remote unRAID version doesn't match local. A mismatch
#   may mean the remote is in an unexpected state.
#
# SSH Timeout
#   SSH_TIMEOUT caps all SSH calls. One hung connection does not block the run.
#
# Notification Validated
#   platform_require_cmd confirms the notify script is present before use.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# host*.conf
#
#   HOST*_BACKUP_VERIFY_SHARES
#       Shares to verify. Leave empty to use HOST*_DAILY_SYNC_SHARES automatically.
#       Aliased by detect_hosts() → BACKUP_VERIFY_SHARES.
#
#   HOST*_DAILY_SYNC_SHARES
#       Fallback share list if BACKUP_VERIFY_SHARES is empty. Aliased by detect_hosts().
#
# master.conf
#
#   BACKUP_VERIFY_SAMPLE
#       Random files checked per share per run. (default: 10)
#
#   BACKUP_VERIFY_MIN_SIZE
#       Minimum file size to include in sample — tiny files have low corruption
#       risk and slow checksums. (default: 1M)
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# backup_verify.sh
#     Sample files from all shares and compare checksums. Notify on MISMATCH
#     or significant MISSING count. Silent when all samples match.
#
# backup_verify.sh --dry-run
#     Show which files would be sampled. No checksums computed, no notifications.
#
# backup_verify.sh --status
#     Show share list, sample size, and min file size configuration. Then exit.
#
# backup_verify.sh --log
#     Verbose per-file checksum comparison output during the run.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

SSH_TIMEOUT=15

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"

if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

platform_require_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

acquire_lock

# detect_hosts() sets MY_ID and aliases BACKUP_VERIFY_SHARES + DAILY_SYNC_SHARES
detect_hosts

# Share selection — configured list or fallback to daily sync shares
if [[ ${#BACKUP_VERIFY_SHARES[@]} -gt 0 ]]; then
    VERIFY_SHARES=("${BACKUP_VERIFY_SHARES[@]}")
    log "Using BACKUP_VERIFY_SHARES (${#VERIFY_SHARES[@]} shares)"
else
    VERIFY_SHARES=("${DAILY_SYNC_SHARES[@]}")
    log "BACKUP_VERIFY_SHARES not set — using DAILY_SYNC_SHARES (${#VERIFY_SHARES[@]} shares)"
fi

if [[ ${#VERIFY_SHARES[@]} -eq 0 ]]; then
    warn "No shares configured for $MY_ID — nothing to verify"
    warn "Check HOST*_BACKUP_VERIFY_SHARES or HOST*_DAILY_SYNC_SHARES in host*.conf"
    exit 0
fi

log "$ICON_GEAR Config: sample=${BACKUP_VERIFY_SAMPLE} min-size=${BACKUP_VERIFY_MIN_SIZE} ssh-timeout=${SSH_TIMEOUT}s"
log "$ICON_GEAR Remote: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
log "$ICON_GEAR Shares: ${VERIFY_SHARES[*]}"

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing sample selection only, no checksums computed"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST    My ID:      $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_HOST    Remote:     $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
    echo "$ICON_VERIFY  Shares:     ${#VERIFY_SHARES[@]}"
    echo "$ICON_VERIFY  Sample:     $BACKUP_VERIFY_SAMPLE files per share"
    echo "$ICON_VERIFY  Min size:   $BACKUP_VERIFY_MIN_SIZE"
    echo "$ICON_GEAR    Dry Run:    $DRY_RUN"
    echo ""
    echo "  Shares to verify:"
    for share in "${VERIFY_SHARES[@]}"; do
        echo "    $share"
    done
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Pre-flight ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SHIELD Pre-flight ━━━"

resolve_remote_ip

# Connectivity — no point making 100+ SSH calls if remote is unreachable
check_connectivity
log "Connectivity to $REMOTE_SERVER_NAME ✅"

# Version parity — mismatched unRAID could cause md5sum path differences
check_unraid_version_parity || {
    warn "Version parity check failed — proceeding with caution"
    warn "Checksum results may be unreliable if md5sum path changed between versions"
}
log "Version parity with $REMOTE_SERVER_NAME ✅"

# Remote array — if array is down all files appear "missing" = false alarm
if ! check_remote_array; then
    error "Remote array not mounted on $REMOTE_SERVER_NAME"
    error "All files would appear as MISSING — aborting to prevent false alarm"
    notify "Backup verify aborted on $(hostname) — remote array not mounted on $REMOTE_SERVER_NAME" \
        "Backup Verify" "warning"
    exit 1
fi
log "Remote array mounted on $REMOTE_SERVER_NAME ✅"

echo "Pre-flight passed ✅"

# ==============================================================================================
# ━━━ Backup Verification ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_VERIFY Backup Verification — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_HOST    $MY_ID ($LOCAL_SERVER_NAME) → $REMOTE_ID ($REMOTE_SERVER_NAME)"
echo "$ICON_VERIFY  Sample: $BACKUP_VERIFY_SAMPLE files per share (min: $BACKUP_VERIFY_MIN_SIZE)"
echo ""

START=$(date +%s)
TOTAL_CHECKED=0
TOTAL_MATCH=0
TOTAL_MISMATCH=0
TOTAL_MISSING=0
SHARES_WITH_ISSUES=()

for share in "${VERIFY_SHARES[@]}"; do
    SHARE_NAME=$(basename "$share")
    echo "━━━ $ICON_VERIFY $SHARE_NAME ━━━"

    if [[ ! -d "$share" ]]; then
        warn "$SHARE_NAME not found locally — skipping"
        echo ""
        continue
    fi

    # Sample random files above minimum size
    mapfile -t SAMPLE_FILES < <(
        find "$share" -type f -size +"$BACKUP_VERIFY_MIN_SIZE" 2>/dev/null | \
        shuf | head -n "$BACKUP_VERIFY_SAMPLE"
    )

    if [[ ${#SAMPLE_FILES[@]} -eq 0 ]]; then
        log "$SHARE_NAME — no files found above $BACKUP_VERIFY_MIN_SIZE"
        echo ""
        continue
    fi

    log "$SHARE_NAME — sampled ${#SAMPLE_FILES[@]} files"

    if [[ "$DRY_RUN" == true ]]; then
        for f in "${SAMPLE_FILES[@]}"; do
            warn "DRY RUN — would check: $(basename "$f")"
        done
        echo ""
        continue
    fi

    SHARE_MATCH=0
    SHARE_MISMATCH=0
    SHARE_MISSING=0

    for local_file in "${SAMPLE_FILES[@]}"; do
        [[ -z "$local_file" ]] && continue

        # Local checksum
        local_md5=$(md5sum "$local_file" 2>/dev/null | awk '{print $1}')
        if [[ -z "$local_md5" ]]; then
            warn "Could not checksum locally: $(basename "$local_file") — skipping"
            continue
        fi

        # Remote checksum via SSH — timeout protected
        remote_md5=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
            -o ConnectTimeout="$SSH_TIMEOUT" \
            -o StrictHostKeyChecking=no \
            root@"$REMOTE_SERVER" \
            "md5sum '$local_file' 2>/dev/null | awk '{print \$1}'" 2>/dev/null)

        (( TOTAL_CHECKED++ ))

        if [[ -z "$remote_md5" ]]; then
            warn "$ICON_ERROR MISSING: $(basename "$local_file")"
            (( SHARE_MISSING++ ))
            (( TOTAL_MISSING++ ))
        elif [[ "$local_md5" == "$remote_md5" ]]; then
            log "MATCH: $(basename "$local_file")"
            (( SHARE_MATCH++ ))
            (( TOTAL_MATCH++ ))
        else
            error "MISMATCH: $(basename "$local_file")"
            error "  local:  $local_md5"
            error "  remote: $remote_md5"
            (( SHARE_MISMATCH++ ))
            (( TOTAL_MISMATCH++ ))
        fi
    done

    # Per-share result — only visible if issues found
    if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 ]]; then
        warn "$SHARE_NAME — match: $SHARE_MATCH  missing: $SHARE_MISSING  mismatch: $SHARE_MISMATCH"
        SHARES_WITH_ISSUES+=("$SHARE_NAME")
    else
        log "$SHARE_NAME — all $SHARE_MATCH files match ✅"
    fi

    echo ""
done

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo "━━━━━ $ICON_SUMMARY BACKUP VERIFY SUMMARY ━━━━━"
echo "$ICON_HOST    My ID:     $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_HOST    Remote:    $REMOTE_ID ($REMOTE_SERVER_NAME)"
echo "$ICON_VERIFY  Checked:   $TOTAL_CHECKED files"
echo "$ICON_TIME    Duration:  $(format_duration $(( END - START )))"
echo ""

if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
    echo "$ICON_SUCCESS Match:     $TOTAL_MATCH"
    warn "Missing:   $TOTAL_MISSING"
    [[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR   Mismatch:  $TOTAL_MISMATCH"
fi

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no checksums computed"
elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
    echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention: ${SHARES_WITH_ISSUES[*]}"
    notify "Backup verify FAILED on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" \
        "Backup Verify" "warning"
else
    echo "$ICON_DONE Status: all $TOTAL_CHECKED files match across ${#VERIFY_SHARES[@]} shares ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ "$TOTAL_MISMATCH" -gt 0 ]] && exit 1
exit 0