Varaverk/Rsync/rsync.sh

#!/bin/bash
# ==============================================================================================
# ============================= Rsync Core Script ==============================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Core rsync engine for the two-server ecosystem. Called per share or per
# appdata profile by orchestrators (daily_sync_maintenance, weekly_sync_maintenance,
# critical_sync_maintenance) and directly for manual or scheduled dirty syncs.
#
# Profile is inferred from the directory basename (lowercased). Override with
# --profile=name for explicit selection. If no profile matches, global defaults
# from master.conf apply and no containers are stopped.
#
# After each sync, logs transfer data to bandwidth_monitor.sh for the weekly
# bandwidth report. Silent on success — only failures produce visible output.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Profiles define per-share behavior:
#   PROFILE_CRITICAL_CONTAINER_NAMES  — containers stopped on both servers before sync
#   PROFILE_DELAYED_CONTAINERS        — containers with a delay before restart after sync
#   PROFILE_CONTAINER_DELAY           — seconds before delayed containers start
#   PROFILE_RSYNC_OPTS                — rsync flags (does not inherit DEFAULT_RSYNC_OPTS)
#   PROFILE_BW_LIMIT                  — bandwidth limit in KB/s
#   PROFILE_RETRY_COUNT               — retry attempts on failure
#   PROFILE_SLEEP                     — seconds between retry attempts
#   PROFILE_EXCLUDE_DIRS              — paths excluded from transfer
#   PROFILE_REMOTE_RESTART_CONTAINERS — containers restarted on remote after dirty sync
#                                       Was running → restart. Was stopped → leave stopped.
#
# Two-tier rsync enable/disable:
#   Tier 1: RSYNC_ENABLED=false → all rsync stops immediately (checked by this script)
#   Tier 2: per-orchestrator flag (DAILY_RSYNC_ENABLED etc.) → checked by caller
#
# Bandwidth logging: after each sync, logs profile/duration/status/bytes to
# bandwidth_monitor.sh --log-transfer. Bytes captured from rsync --stats via awk
# using version-stable field names.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Global Rsync Gate
#   check_rsync_enabled() — RSYNC_ENABLED=false exits cleanly before any operation.
#
# Partnership Blocklist
#   Refuses to sync if REMOTE_SERVER_NAME appears in the partnership blocklist.
#   Written at offboard — prevents stale access after a partnership ends.
#
# Version Parity
#   check_unraid_version_parity — refuses sync if servers on incompatible unRAID versions.
#
# Remote Health Pre-flights
#   check_connectivity() — Tailscale IP reachable before any SSH
#   check_remote_rootfs() — aborts if remote rootfs exceeds ROOTFS_WARN_PCT
#   check_remote_share() — aborts if target directory missing or empty on remote
#   check_remote_disks() — verifies all backing disks online on remote
#
# Drive Temperature Check
#   check_local_disk_temps() — runs before any transfer. Exit 1 = skip this profile,
#   exit 2 = abort all remaining syncs (CRITICAL temperature).
#
# Remote Docker Daemon Check
#   check_remote_docker_daemon — verified before any container stop/start operations.
#   If daemon unresponsive: container operations skipped, rsync proceeds without stopping.
#
# Per-Profile Concurrency Lock
#   acquire_rsync_lock() — per-profile lock prevents parallel runs of the same profile.
#   Global concurrent limit prevents too many simultaneous rsync processes.
#
# Notification Validated
#   platform_require_cmd confirms the notify script is present before use.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   RSYNC_ENABLED
#       Global on/off toggle for all rsync operations. (default: true)
#
#   DEFAULT_RSYNC_OPTS
#       Base rsync flags for unproiled shares. Note: --delete is intentionally absent —
#       media shares spread files only, arr cleanup scripts own deletions. Profile-specific
#       opts set --delete explicitly where needed.
#
#   BW_LIMIT
#       Default bandwidth cap in KB/s when no PROFILE_BW_LIMIT is set. (default: 0 = unlimited)
#
#   RETRY_COUNT
#       Default retry attempts on rsync failure. (default: 3)
#
#   SLEEP
#       Default seconds between retry attempts. (default: 60)
#
#   ROOTFS_WARN_PCT
#       Abort threshold for remote rootfs percentage full. (default: 75)
#
#   PROFILES["profile_KEY"]
#       Profile definitions — one entry per PROFILE_* key per profile name.
#       See OPERATIONAL MODEL above for all supported keys.
#
#   BANDWIDTH_LOG / BANDWIDTH_WARN_GB
#       Shared with bandwidth_monitor.sh — set once, used by both.
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# rsync.sh /path/to/share
#     Sync the given path to the remote. Profile inferred from directory basename.
#
# rsync.sh /path/to/share --profile=name
#     Sync with explicit profile override — bypasses basename inference.
#
# rsync.sh /path/to/share --dry-run
#     Run all pre-flight checks and show what rsync would transfer. No transfer,
#     no container stops.
#
# rsync.sh /path/to/share --status
#     Show resolved profile, remote identity, and configuration. Then exit.
#
# rsync.sh /path/to/share --log
#     Verbose output throughout — every decision logged.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

# ── Separate positional directory arg from flags ───────────────────────────────────────────────
DIRECTORY=""
PROFILE_OVERRIDE=""
RAW_ARGS=()

for ARG in "$@"; do
    case "$ARG" in
        --profile=*) PROFILE_OVERRIDE="${ARG#--profile=}" ;;
        --*|*=*)     RAW_ARGS+=("$ARG") ;;
        *)           [[ -z "$DIRECTORY" ]] && DIRECTORY="$ARG" || RAW_ARGS+=("$ARG") ;;
    esac
done

parse_args "${RAW_ARGS[@]}"

[[ -z "$DIRECTORY" ]] && {
    error "No directory specified"
    error "Usage: rsync.sh <dir> [--dry-run] [--log] [--profile=name]"
    exit 1
}

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

acquire_lock

if ! command -v docker &>/dev/null; then
    error "Docker command not found"
    exit 1
fi

platform_require_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

detect_hosts

# Tier 1 global gate — Tier 2 (per-orchestrator) checked by caller
if ! check_rsync_enabled; then
    warn "RSYNC_ENABLED=false — exiting cleanly"
    exit 0
fi

# Blocklist gate — refuse to sync with a partner blocked after offboard
BLOCKLIST_FILE="${PARTNERSHIP_BLOCKLIST_FILE:-${STATE_DIR:-/boot/config}/partnership_blocklist.db}"
if [[ -f "$BLOCKLIST_FILE" ]] && grep -q "^${REMOTE_SERVER_NAME}|" "$BLOCKLIST_FILE" 2>/dev/null; then
    error "Rsync blocked — $REMOTE_SERVER_NAME is on the partnership blocklist"
    error "Re-onboard the partnership to restore access: partnership_manager.sh --onboard"
    exit 1
fi

resolve_remote_ip

# ── Profile inference ─────────────────────────────────────────────────────────────────────────
if [[ -n "$PROFILE_OVERRIDE" ]]; then
    PROFILE_NAME="$PROFILE_OVERRIDE"
    log "Profile override: $PROFILE_NAME"
else
    PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]')
    log "Profile inferred: $PROFILE_NAME"
fi

# Acquire per-profile lock and check global concurrent limit
acquire_rsync_lock "$PROFILE_NAME"

# ── Load profile settings ─────────────────────────────────────────────────────────────────────
BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY}

read -r -a CRITICAL_CONTAINER_NAMES       <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]:-}"
read -r -a DELAYED_CONTAINERS             <<< "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]:-}"
read -r -a EXCLUDE_DIRS                   <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-}"
read -r -a REMOTE_RESTART_CONTAINERS      <<< "${PROFILE_REMOTE_RESTART_CONTAINERS[$PROFILE_NAME]:-}"

# Local containers use same names as remote (mirrored naming scheme)
LOCAL_CRITICAL_CONTAINER_NAMES=("${CRITICAL_CONTAINER_NAMES[@]}")

[[ "$SHOW_STATUS" == true ]] && show_status && exit 0
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"

# ==============================================================================================
# ━━━ Pre-flight Checks ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SHIELD Pre-flight ━━━"

# Disk temp — before touching remote or moving data
# Exit 1 = skip this profile | Exit 2 = abort all remaining profiles
check_local_disk_temps
TEMP_RESULT=$?
if [[ "$TEMP_RESULT" -eq 2 ]]; then
    error "Drive temps CRITICAL — aborting all remaining syncs"
    exit 2
elif [[ "$TEMP_RESULT" -eq 1 ]]; then
    warn "Drive temps high — skipping profile [$PROFILE_NAME]"
    exit 1
else
    log "Drive temps OK — $TEMP_CHECK_RESULT"
fi

# Version parity — refuse if servers on incompatible unRAID versions
check_unraid_version_parity || exit 1

check_connectivity
check_remote_rootfs
check_remote_share "$DIRECTORY"
check_remote_disks "$DIRECTORY"

# Remote Docker daemon — check before attempting container operations
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]] || [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then
    check_remote_docker_daemon || {
        warn "Remote Docker daemon unresponsive — skipping container operations"
        warn "Proceeding with rsync only — containers will not be stopped or restarted"
        CRITICAL_CONTAINER_NAMES=()
        LOCAL_CRITICAL_CONTAINER_NAMES=()
        REMOTE_RESTART_CONTAINERS=()
    }
fi

# ==============================================================================================
# ━━━ Stop Containers ━━━
# ==============================================================================================
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
    if [[ "$DRY_RUN" == true ]]; then
        for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]:-}"; do
            [[ -n "$c" ]] && warn "DRY RUN — would stop local: $c"
        done
        for c in "${CRITICAL_CONTAINER_NAMES[@]:-}"; do
            [[ -n "$c" ]] && warn "DRY RUN — would stop remote: $c"
        done
    else
        # Local first — flush local databases before pushing
        stop_local_containers
        # Remote next — prevent writes while receiving
        stop_containers
    fi
fi

# ==============================================================================================
# ━━━ Transfer ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Transfer ━━━"
echo "$ICON_RUN  Source:   $DIRECTORY"
echo "$ICON_NET  Remote:   $REMOTE_SERVER:$DIRECTORY"
echo "$ICON_GEAR Profile:  $PROFILE_NAME"
echo "$ICON_HOST Identity: $MY_ID → $REMOTE_ID"
echo ""

get_rsync_opts

# Append profile excludes
for ex in "${EXCLUDE_DIRS[@]:-}"; do
    [[ -n "$ex" ]] && RSYNC_OPTS+=(--exclude="$ex")
done

# Add --stats to capture bytes transferred for bandwidth logging
RSYNC_OPTS+=(--stats)

[[ "$DRY_RUN" == true ]] && RSYNC_OPTS+=("--dry-run")

START=$(date +%s)
RSYNC_SUCCESS=false
BYTES_TRANSFERRED=0
ATTEMPT=0

for (( ATTEMPT=1; ATTEMPT<=RETRY_COUNT; ATTEMPT++ )); do
    log "$ICON_RETRY Attempt $ATTEMPT of $RETRY_COUNT..."
    echo "$ICON_SYNC  Rsync running — this may take a while..."

    RSYNC_OUTPUT=$(rsync "${RSYNC_OPTS[@]}" \
        -e "ssh -i $SSH_KEY -T -o Compression=no -o IPQoS=throughput" \
        "$DIRECTORY" "root@${REMOTE_SERVER}:$(dirname "$DIRECTORY")/" 2>&1)

    RSYNC_EXIT=$?

    if [[ "$RSYNC_EXIT" -eq 0 ]]; then
        # Parse bytes transferred from --stats output
        BYTES_TRANSFERRED=$(echo "$RSYNC_OUTPUT" | \
            awk '/Total transferred file size:/{gsub(/,/,"",$NF); gsub(/[^0-9]/,"",$NF); print $NF+0}')
        BYTES_TRANSFERRED="${BYTES_TRANSFERRED:-0}"

        log "$ICON_DONE Rsync complete — $BYTES_TRANSFERRED bytes transferred"
        RSYNC_SUCCESS=true
        break
    else
        warn "$ICON_RETRY Rsync failed (attempt $ATTEMPT/$RETRY_COUNT)"
        log "Exit code: $RSYNC_EXIT"
        if [[ "$ATTEMPT" -lt "$RETRY_COUNT" ]]; then
            log "Retrying in ${SLEEP}s..."
            sleep "$SLEEP"
        fi
    fi
done

# ==============================================================================================
# ━━━ Start Containers ━━━
# ==============================================================================================
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
    if [[ "$DRY_RUN" == true ]]; then
        for c in "${CRITICAL_CONTAINER_NAMES[@]:-}"; do
            [[ -n "$c" ]] && warn "DRY RUN — would start remote: $c"
        done
        for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]:-}"; do
            [[ -n "$c" ]] && warn "DRY RUN — would start local: $c"
        done
    else
        # Remote first — can be coming up while local restarts
        start_containers
        # Local next
        start_local_containers
    fi
fi

# ==============================================================================================
# ━━━ Remote Restart (dirty sync profiles) ━━━
# ==============================================================================================
# For dirty sync profiles (critical-fallback, emby-fallback) — restart containers on remote
# that were running before sync so they pick up config changes from the dirty sync window.
# Was running → restart. Was stopped → leave stopped.
if [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_START $ICON_CONTAINERS Remote Restart (post dirty sync) ━━━"
    log "Restarting configured containers on $REMOTE_SERVER_NAME..."

    for container in "${REMOTE_RESTART_CONTAINERS[@]}"; do
        [[ -z "$container" ]] && continue

        # Check if container was running before sync (still tracked via RUNNING_CONTAINERS)
        WAS_RUNNING=false
        for prev in "${RUNNING_CONTAINERS[@]:-}"; do
            [[ "$prev" == "$container" ]] && WAS_RUNNING=true && break
        done

        if [[ "$WAS_RUNNING" == false ]]; then
            # Not in stop list — check current remote state
            REMOTE_STATUS=$(timeout 15 ssh -i "$SSH_KEY" \
                -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
                "docker inspect -f '{{.State.Running}}' $container 2>/dev/null" 2>/dev/null)
            [[ "$REMOTE_STATUS" != "true" ]] && \
                log "$container not running on $REMOTE_SERVER_NAME — skipping remote restart" && \
                continue
        fi

        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would restart $container on $REMOTE_SERVER_NAME"
            continue
        fi

        timeout 15 ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
            "docker restart $container" >/dev/null 2>&1 && \
            log "$ICON_STARTED $container restarted on $REMOTE_SERVER_NAME ✅" || \
            warn "Failed to restart $container on $REMOTE_SERVER_NAME"
    done
fi

END=$(date +%s)
DURATION=$(( END - START ))

# ==============================================================================================
# ━━━ Bandwidth Logging ━━━
# ==============================================================================================
# Logs to bandwidth_monitor.sh — new format includes bytes transferred and warn flag.
# Only logs on actual runs (not dry-run) and only when bandwidth_monitor.sh exists.
BANDWIDTH_MONITOR="$SCRIPT_DIR/../Monitors/bandwidth_monitor.sh"

if [[ "$DRY_RUN" == false ]] && [[ -f "$BANDWIDTH_MONITOR" ]]; then
    STATUS="success"
    [[ "$RSYNC_SUCCESS" == false ]] && STATUS="failed"
    bash "$BANDWIDTH_MONITOR" --log-transfer \
        "$PROFILE_NAME" "$DURATION" "$STATUS" "$BYTES_TRANSFERRED"
    log "$ICON_BANDWIDTH Transfer logged to bandwidth monitor ($BYTES_TRANSFERRED bytes)"
fi

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY RSYNC SUMMARY ━━━━━"
echo "$ICON_HOST    Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_RUN     Directory: $DIRECTORY"
echo "$ICON_GEAR    Profile:   $PROFILE_NAME"
echo "$ICON_TIME    Duration:  $(format_duration $DURATION)"
[[ "$BYTES_TRANSFERRED" -gt 0 ]] && \
    echo "$ICON_BANDWIDTH Transferred: $(awk "BEGIN {printf \"%.2fGB\", $BYTES_TRANSFERRED / 1073741824}")"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no changes made"
elif [[ "$RSYNC_SUCCESS" == true ]]; then
    echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
else
    echo "$ICON_ERROR Status: FAILED after $RETRY_COUNT attempts"
    notify "Rsync FAILED — $DIRECTORY ($PROFILE_NAME) after $RETRY_COUNT attempts on $(hostname)" \
        "Rsync" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"

[[ "$RSYNC_SUCCESS" == false ]] && [[ "$DRY_RUN" == false ]] && exit 1
exit 0