Varaverk/Rsync/rsync.sh

#!/bin/bash
# ==============================================================================================
# ================================= Rsync Core Script ==========================================
# ==============================================================================================
# Core rsync script — called per share or per appdata profile.
# Called by orchestrators (daily/weekly/critical sync) and directly for manual syncs.
#
# ── PROFILE SYSTEM ────────────────────────────────────────────────────────────────────────────
# Profile is inferred from the directory basename (lowercased).
# Override with --profile=name for explicit profile selection.
# If no profile match found → all settings fall back to global defaults in master.conf.
#
# Profiles define:
#   PROFILE_RSYNC_OPTS               — rsync flags (does NOT inherit DEFAULT_RSYNC_OPTS)
#   PROFILE_BW_LIMIT                 — bandwidth limit in KB/s
#   PROFILE_RETRY_COUNT              — retry attempts before giving up
#   PROFILE_SLEEP                    — seconds between retry attempts
#   PROFILE_CRITICAL_CONTAINER_NAMES — containers stopped both sides before sync
#   PROFILE_DELAYED_CONTAINERS       — containers needing delay before starting after sync
#   PROFILE_CONTAINER_DELAY          — seconds before starting delayed containers
#   PROFILE_EXCLUDE_DIRS             — paths excluded from transfer
#   PROFILE_REMOTE_RESTART_CONTAINERS — containers restarted on remote after dirty sync
#                                       (critical-fallback, emby-fallback profiles)
#                                       Was running → restart. Was stopped → leave stopped.
#
# ── RSYNC ENABLE/DISABLE ──────────────────────────────────────────────────────────────────────
# Two-tier toggle system — checked at entry:
#   Tier 1: RSYNC_ENABLED=false → all rsync stops
#   Tier 2: Per-orchestrator flag (DAILY_RSYNC_ENABLED etc.) — checked by caller
#   Direct calls to rsync.sh only check Tier 1
#
# ── BANDWIDTH LOGGING ─────────────────────────────────────────────────────────────────────────
# After each sync logs to bandwidth_monitor.sh --log-transfer:
#   profile | duration_seconds | status | bytes_transferred
# Bytes captured from rsync --stats output — version-proof parsing.
# bandwidth_monitor.sh flags syncs exceeding BANDWIDTH_WARN_GB.
#
# ── DIRTY SYNC REMOTE RESTART ─────────────────────────────────────────────────────────────────
# Profiles using dirty sync (critical-fallback, emby-fallback) define
# PROFILE_REMOTE_RESTART_CONTAINERS — containers restarted on remote after sync completes.
# This ensures the remote picks up config changes synced during the dirty window.
# Was running → restart. Was stopped → leave stopped.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
#   check_rsync_enabled()        — Tier 1 gate before any operation
#   check_unraid_version_parity  — refuses if servers on incompatible unRAID versions
#   check_remote_docker_daemon   — verifies remote daemon before container operations
#   check_local_disk_temps()     — temp check before transfer (exit 1=skip, 2=abort all)
#   check_connectivity()         — verifies remote reachable
#   check_remote_rootfs()        — aborts if remote rootfs nearly full
#   check_remote_share()         — aborts if target directory missing or empty
#   check_remote_disks()         — verifies all backing disks online on remote
#   acquire_rsync_lock()         — per-profile lock + global concurrent limit
#   validate_unraid_cmd          — notify validated before use
#   Silent by default            — only failures produce visible output
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   rsync.sh /mnt/user/Movies                             — media share, global defaults
#   rsync.sh /mnt/user/appdata-Fallback/Arrs_Stack        — matched to [arrs_stack] profile
#   rsync.sh /mnt/user/appdata-Fallback/Arrs_Stack --dry-run --log
#   rsync.sh /mnt/user/appdata-Fallback/Critical-Data --profile=critical-fallback
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

# ── Separate positional directory arg from flags ───────────────────────────────────────────────
DIRECTORY=""
PROFILE_OVERRIDE=""
RAW_ARGS=()

for ARG in "$@"; do
    case "$ARG" in
        --profile=*) PROFILE_OVERRIDE="${ARG#--profile=}" ;;
        --*|*=*)     RAW_ARGS+=("$ARG") ;;
        *)           [[ -z "$DIRECTORY" ]] && DIRECTORY="$ARG" || RAW_ARGS+=("$ARG") ;;
    esac
done

parse_args "${RAW_ARGS[@]}"

[[ -z "$DIRECTORY" ]] && {
    error "No directory specified"
    error "Usage: rsync.sh <dir> [--dry-run] [--log] [--profile=name]"
    exit 1
}

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

validate_unraid_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

detect_hosts

# Tier 1 global gate — Tier 2 (per-orchestrator) checked by caller
if ! check_rsync_enabled; then
    warn "RSYNC_ENABLED=false — exiting cleanly"
    exit 0
fi

# Blocklist gate — refuse to sync with a partner blocked after offboard
BLOCKLIST_FILE="/boot/config/partnership_blocklist.db"
if [[ -f "$BLOCKLIST_FILE" ]] && grep -q "^${REMOTE_SERVER_NAME}|" "$BLOCKLIST_FILE" 2>/dev/null; then
    error "Rsync blocked — $REMOTE_SERVER_NAME is on the partnership blocklist"
    error "Re-onboard the partnership to restore access: partnership_manager.sh --onboard"
    exit 1
fi

resolve_remote_ip

# ── Profile inference ─────────────────────────────────────────────────────────────────────────
if [[ -n "$PROFILE_OVERRIDE" ]]; then
    PROFILE_NAME="$PROFILE_OVERRIDE"
    log "Profile override: $PROFILE_NAME"
else
    PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]')
    log "Profile inferred: $PROFILE_NAME"
fi

# Acquire per-profile lock and check global concurrent limit
acquire_rsync_lock "$PROFILE_NAME"

# ── Load profile settings ─────────────────────────────────────────────────────────────────────
BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY}

read -r -a CRITICAL_CONTAINER_NAMES       <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]:-}"
read -r -a DELAYED_CONTAINERS             <<< "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]:-}"
read -r -a EXCLUDE_DIRS                   <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-}"
read -r -a REMOTE_RESTART_CONTAINERS      <<< "${PROFILE_REMOTE_RESTART_CONTAINERS[$PROFILE_NAME]:-}"

# Local containers use same names as remote (mirrored naming scheme)
LOCAL_CRITICAL_CONTAINER_NAMES=("${CRITICAL_CONTAINER_NAMES[@]}")

[[ "$SHOW_STATUS" == true ]] && show_status && exit 0
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"

# ==============================================================================================
# ━━━ Pre-flight Checks ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SHIELD Pre-flight ━━━"

# Disk temp — before touching remote or moving data
# Exit 1 = skip this profile | Exit 2 = abort all remaining profiles
check_local_disk_temps
TEMP_RESULT=$?
if [[ "$TEMP_RESULT" -eq 2 ]]; then
    error "Drive temps CRITICAL — aborting all remaining syncs"
    exit 2
elif [[ "$TEMP_RESULT" -eq 1 ]]; then
    warn "Drive temps high — skipping profile [$PROFILE_NAME]"
    exit 1
else
    log "Drive temps OK — $TEMP_CHECK_RESULT"
fi

# Version parity — refuse if servers on incompatible unRAID versions
check_unraid_version_parity || exit 1

check_connectivity
check_remote_rootfs
check_remote_share "$DIRECTORY"
check_remote_disks "$DIRECTORY"

# Remote Docker daemon — check before attempting container operations
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]] || [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then
    check_remote_docker_daemon || {
        warn "Remote Docker daemon unresponsive — skipping container operations"
        warn "Proceeding with rsync only — containers will not be stopped or restarted"
        CRITICAL_CONTAINER_NAMES=()
        LOCAL_CRITICAL_CONTAINER_NAMES=()
        REMOTE_RESTART_CONTAINERS=()
    }
fi

# ==============================================================================================
# ━━━ Stop Containers ━━━
# ==============================================================================================
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
    # Local first — flush local databases before pushing
    stop_local_containers
    # Remote next — prevent writes while receiving
    stop_containers
fi

# ==============================================================================================
# ━━━ Transfer ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Transfer ━━━"
echo "$ICON_RUN  Source:   $DIRECTORY"
echo "$ICON_NET  Remote:   $REMOTE_SERVER:$DIRECTORY"
echo "$ICON_GEAR Profile:  $PROFILE_NAME"
echo "$ICON_HOST Identity: $MY_ID → $REMOTE_ID"
echo ""

get_rsync_opts

# Append profile excludes
for ex in "${EXCLUDE_DIRS[@]:-}"; do
    [[ -n "$ex" ]] && RSYNC_OPTS+=(--exclude="$ex")
done

# Add --stats to capture bytes transferred for bandwidth logging
RSYNC_OPTS+=(--stats)

[[ "$DRY_RUN" == true ]] && RSYNC_OPTS+=("--dry-run")

START=$(date +%s)
RSYNC_SUCCESS=false
BYTES_TRANSFERRED=0
ATTEMPT=0

for (( ATTEMPT=1; ATTEMPT<=RETRY_COUNT; ATTEMPT++ )); do
    log "$ICON_RETRY Attempt $ATTEMPT of $RETRY_COUNT..."
    echo "$ICON_SYNC  Rsync running — this may take a while..."

    RSYNC_OUTPUT=$(rsync "${RSYNC_OPTS[@]}" \
        -e "ssh -i $SSH_KEY -T -o Compression=no -o IPQoS=throughput" \
        "$DIRECTORY" "root@${REMOTE_SERVER}:$(dirname "$DIRECTORY")/" 2>&1)

    RSYNC_EXIT=$?

    if [[ "$RSYNC_EXIT" -eq 0 ]]; then
        # Parse bytes transferred from --stats output
        BYTES_TRANSFERRED=$(echo "$RSYNC_OUTPUT" | \
            awk '/Total transferred file size:/{gsub(/,/,"",$NF); gsub(/[^0-9]/,"",$NF); print $NF+0}')
        BYTES_TRANSFERRED="${BYTES_TRANSFERRED:-0}"

        log "$ICON_DONE Rsync complete — $BYTES_TRANSFERRED bytes transferred"
        RSYNC_SUCCESS=true
        break
    else
        warn "$ICON_RETRY Rsync failed (attempt $ATTEMPT/$RETRY_COUNT)"
        log "Exit code: $RSYNC_EXIT"
        if [[ "$ATTEMPT" -lt "$RETRY_COUNT" ]]; then
            log "Retrying in ${SLEEP}s..."
            sleep "$SLEEP"
        fi
    fi
done

# ==============================================================================================
# ━━━ Start Containers ━━━
# ==============================================================================================
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
    # Remote first — can be coming up while local restarts
    start_containers
    # Local next
    start_local_containers
fi

# ==============================================================================================
# ━━━ Remote Restart (dirty sync profiles) ━━━
# ==============================================================================================
# For dirty sync profiles (critical-fallback, emby-fallback) — restart containers on remote
# that were running before sync so they pick up config changes from the dirty sync window.
# Was running → restart. Was stopped → leave stopped.
if [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_START $ICON_CONTAINERS Remote Restart (post dirty sync) ━━━"
    log "Restarting configured containers on $REMOTE_SERVER_NAME..."

    for container in "${REMOTE_RESTART_CONTAINERS[@]}"; do
        [[ -z "$container" ]] && continue

        # Check if container was running before sync (still tracked via RUNNING_CONTAINERS)
        WAS_RUNNING=false
        for prev in "${RUNNING_CONTAINERS[@]:-}"; do
            [[ "$prev" == "$container" ]] && WAS_RUNNING=true && break
        done

        if [[ "$WAS_RUNNING" == false ]]; then
            # Not in stop list — check current remote state
            REMOTE_STATUS=$(timeout 15 ssh -i "$SSH_KEY" \
                -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
                "docker inspect -f '{{.State.Running}}' $container 2>/dev/null" 2>/dev/null)
            [[ "$REMOTE_STATUS" != "true" ]] && \
                log "$container not running on $REMOTE_SERVER_NAME — skipping remote restart" && \
                continue
        fi

        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would restart $container on $REMOTE_SERVER_NAME"
            continue
        fi

        timeout 15 ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
            "docker restart $container" >/dev/null 2>&1 && \
            log "$ICON_STARTED $container restarted on $REMOTE_SERVER_NAME ✅" || \
            warn "Failed to restart $container on $REMOTE_SERVER_NAME"
    done
fi

END=$(date +%s)
DURATION=$(( END - START ))

# ==============================================================================================
# ━━━ Bandwidth Logging ━━━
# ==============================================================================================
# Logs to bandwidth_monitor.sh — new format includes bytes transferred and warn flag.
# Only logs on actual runs (not dry-run) and only when bandwidth_monitor.sh exists.
BANDWIDTH_MONITOR="$SCRIPT_DIR/../Monitors/bandwidth_monitor.sh"

if [[ "$DRY_RUN" == false ]] && [[ -f "$BANDWIDTH_MONITOR" ]]; then
    STATUS="success"
    [[ "$RSYNC_SUCCESS" == false ]] && STATUS="failed"
    bash "$BANDWIDTH_MONITOR" --log-transfer \
        "$PROFILE_NAME" "$DURATION" "$STATUS" "$BYTES_TRANSFERRED"
    log "$ICON_BANDWIDTH Transfer logged to bandwidth monitor ($BYTES_TRANSFERRED bytes)"
fi

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY RSYNC SUMMARY ━━━━━"
echo "$ICON_HOST    Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_RUN     Directory: $DIRECTORY"
echo "$ICON_GEAR    Profile:   $PROFILE_NAME"
echo "$ICON_TIME    Duration:  $(format_duration $DURATION)"
[[ "$BYTES_TRANSFERRED" -gt 0 ]] && \
    echo "$ICON_BANDWIDTH Transferred: $(awk "BEGIN {printf \"%.2fGB\", $BYTES_TRANSFERRED / 1073741824}")"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no changes made"
elif [[ "$RSYNC_SUCCESS" == true ]]; then
    log "$ICON_DONE Status: $ICON_SUCCESS DONE"
else
    echo "$ICON_ERROR Status: FAILED after $RETRY_COUNT attempts"
    notify "Rsync FAILED — $DIRECTORY ($PROFILE_NAME) after $RETRY_COUNT attempts on $(hostname)" \
        "Rsync" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"

[[ "$RSYNC_SUCCESS" == false ]] && [[ "$DRY_RUN" == false ]] && exit 1
exit 0