This commit is contained in:
2026-03-27 17:50:01 +00:00
parent be849ed684
commit f3abddddc2
2 changed files with 170 additions and 182 deletions
+59 -112
View File
@@ -20,195 +20,142 @@ set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Core System # Load configs & helpers
source "$SCRIPT_DIR/../Master.conf" source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh" source "$SCRIPT_DIR/../common.sh"
parse_args "$@" # ----------------------------- Input -----------------------------
DIRECTORY="$1"
# Input
if [[ -z "$DIRECTORY" ]]; then if [[ -z "$DIRECTORY" ]]; then
echo "Usage: $0 <directory-to-sync> [--dry-run] [--status] [VAR=value ...]" echo "Usage: $0 <directory-to-sync> [--dry-run] [VAR=value ...]"
exit 1 exit 1
fi fi
shift shift
parse_args "$@"
# Host # ----------------------------- Host & IP -----------------------------
detect_hosts detect_hosts
resolve_remote resolve_remote_ip
# Profile # ----------------------------- Profile -----------------------------
PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]') PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]')
log "️ Detected profile: $PROFILE_NAME"
# Safe array normalization
normalize_list() {
local input="$1"
[[ -z "$input" ]] && return
read -r -a __out <<< "$input"
echo "${__out[@]}"
}
CRITICAL_CONTAINER_NAMES=($(normalize_list "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]}"))
DELAYED_CONTAINERS=($(normalize_list "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]}"))
EXCLUDE_DIRS=($(normalize_list "${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]}"))
# Profile overrides
MAX_RSYNC_PROCS=${PROFILE_MAX_PROCS[$PROFILE_NAME]:-$MAX_RSYNC_PROCS} MAX_RSYNC_PROCS=${PROFILE_MAX_PROCS[$PROFILE_NAME]:-$MAX_RSYNC_PROCS}
BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT} BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
CRITICAL_CONTAINER_NAMES=(${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]})
DELAYED_CONTAINERS=(${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]})
CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY}
RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT} RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP} SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY} EXCLUDE_DIRS=(${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-${EXCLUDE_DIRS[@]}})
# Status Mode # ----------------------------- Status Mode -----------------------------
if [[ "$SHOW_STATUS" == true ]]; then if [[ "$SHOW_STATUS" == true ]]; then
echo "==================================================" show_status
echo " ️ RSYNC STATUS"
echo "--------------------------------------------------"
echo " 📁 Source: $DIRECTORY"
echo " 🌐 Remote: $REMOTE_SERVER"
echo " 🧩 Profile: $PROFILE_NAME"
echo " 🚦 BW_LIMIT: $BW_LIMIT"
echo " 🔁 Max Procs: $MAX_RSYNC_PROCS"
echo " 🔁 Retries: $RETRY_COUNT"
echo " 🧪 Dry Run: $DRY_RUN"
echo " 📦 Containers: ${CRITICAL_CONTAINER_NAMES[*]}"
echo " 🚫 Excludes: ${EXCLUDE_DIRS[*]}"
echo "=================================================="
exit 0 exit 0
fi fi
# Header
echo # ----------------------------- User Info -----------------------------
echo "============================================================" echo "============================================================"
echo " 🚀 SYNC STARTING" echo " Sync Job Starting"
echo "------------------------------------------------------------" echo "------------------------------------------------------------"
echo " 📁 Source: $DIRECTORY" echo " Source: $DIRECTORY"
echo " 🌐 Destination: $REMOTE_SERVER:$DIRECTORY" echo " Destination: $REMOTE_SERVER:$DIRECTORY"
echo " 🧩 Profile: $PROFILE_NAME" echo " Profile: $PROFILE_NAME"
echo " 🧪 Dry Run: $DRY_RUN" echo " Dry Run: $DRY_RUN"
echo "============================================================" echo "============================================================"
log " MAX_RSYNC_PROCS=$MAX_RSYNC_PROCS BW_LIMIT=$BW_LIMIT" log "Settings: MAX_RSYNC_PROCS=$MAX_RSYNC_PROCS, BW_LIMIT=$BW_LIMIT"
log "Containers: ${CRITICAL_CONTAINER_NAMES[*]}" log "Containers: ${CRITICAL_CONTAINER_NAMES[*]}"
log "Excludes: ${EXCLUDE_DIRS[*]}" log "Excludes: ${EXCLUDE_DIRS[*]}"
# RSYNC SLOT CONTROL # ----------------------------- Rsync Wait -----------------------------
wait_for_rsync() { wait_for_rsync() {
for attempt in $(seq 1 "$RETRY_COUNT"); do for attempt in $(seq 1 "$RETRY_COUNT"); do
RSYNC_COUNT=$(pgrep -fc "rsync.*root@${REMOTE_SERVER}" || true) RSYNC_COUNT=$(pgrep -fc "rsync.*root@${REMOTE_SERVER}" || true)
if [ "$RSYNC_COUNT" -ge "$MAX_RSYNC_PROCS" ]; then
if [[ "$RSYNC_COUNT" -ge "$MAX_RSYNC_PROCS" ]]; then echo "Waiting for available rsync slot... ($RSYNC_COUNT running)"
warn "🟡 Waiting for rsync slot ($RSYNC_COUNT active)" log "RSYNC_COUNT=$RSYNC_COUNT >= MAX=$MAX_RSYNC_PROCS"
sleep "$SLEEP" sleep "$SLEEP"
else else
return 0 return 0
fi fi
done done
echo "Too many rsync processes, exiting."
error "🔴 Too many rsync processes running" exit 1
} }
# MAIN # ----------------------------- Main -----------------------------
main() { main() {
echo ""
info " Checking remote connectivity..." echo "🔍 Checking remote connectivity..."
for i in $(seq 1 "$RETRY_COUNT"); do for i in $(seq 1 "$RETRY_COUNT"); do
if ping -c 1 "$REMOTE_SERVER" &>/dev/null; then if ping -c 1 "$REMOTE_SERVER" &>/dev/null; then
info "🟢 Remote reachable" echo " Remote server reachable"
break break
fi fi
echo "⚠️ Remote down, retrying in $SLEEP seconds..."
warn "🟡 Remote unreachable (attempt $i/$RETRY_COUNT)"
sleep "$SLEEP" sleep "$SLEEP"
done done
if ! ping -c 1 "$REMOTE_SERVER" &>/dev/null; then if ! ping -c 1 "$REMOTE_SERVER" &>/dev/null; then
error "🔴 Remote server unreachable" echo " Remote server unreachable. Exiting."
exit 1
fi fi
wait_for_rsync wait_for_rsync
info "🟡 Stopping containers on remote..." echo ""
echo "🛑 Stopping containers on remote..."
stop_containers stop_containers
# Build RSYNC OPTIONS echo ""
echo "📦 Building rsync options..."
get_rsync_opts get_rsync_opts
for ex in "${EXCLUDE_DIRS[@]}"; do for ex in "${EXCLUDE_DIRS[@]}"; do
RSYNC_OPTS+=(--exclude="$ex") RSYNC_OPTS+=(--exclude="$ex")
done done
if [[ "$DRY_RUN" == true ]]; then [ "$DRY_RUN" = true ] && RSYNC_OPTS+=("--dry-run")
RSYNC_OPTS+=(--dry-run)
info "🧪 Dry-run enabled"
fi
log "Final rsync opts: ${RSYNC_OPTS[*]}" log "Final rsync opts: ${RSYNC_OPTS[*]}"
# EXECUTION echo ""
info "🚀 Starting rsync..." echo "🚀 Starting rsync..."
START_TIME=$(date +%s)
for attempt in $(seq 1 "$RETRY_COUNT"); do for attempt in $(seq 1 "$RETRY_COUNT"); do
info "Attempt $attempt/$RETRY_COUNT" echo "Attempt $attempt/$RETRY_COUNT"
if rsync "${RSYNC_OPTS[@]}" \ if rsync "${RSYNC_OPTS[@]}" \
-e "ssh -i $SSH_KEY -T -o Compression=no -o IPQoS=throughput" \ -e "ssh -i \"$SSH_KEY\" -T -o Compression=no -o IPQoS=throughput" \
"$DIRECTORY" "root@${REMOTE_SERVER}:$DIRECTORY"; then "$DIRECTORY" "root@${REMOTE_SERVER}:$DIRECTORY"; then
info "🟢 Rsync completed successfully" echo " Rsync completed successfully"
break break
else
warn "❌ Rsync failed"
if [[ "$attempt" -lt "$RETRY_COUNT" ]]; then else
warn "🟡 Retrying in $SLEEP seconds..." echo "❌ Rsync failed"
if [ "$attempt" -lt "$RETRY_COUNT" ]; then
echo "Retrying in $SLEEP seconds..."
sleep "$SLEEP" sleep "$SLEEP"
else else
error "🔴 All rsync attempts failed" echo "💥 All retries failed"
start_containers start_containers
exit 1
fi fi
fi fi
done done
# FINALIZE echo ""
info "🟡 Restarting containers..." echo "🔄 Restarting containers..."
start_containers start_containers
END_TIME=$(date +%s) echo ""
DURATION=$((END_TIME - START_TIME)) echo "🎉 Sync complete!"
# SUMMARY
echo "============================================================"
if [[ "${PIPESTATUS[0]}" -eq 0 ]]; then
echo " 🟢 SYNC SUMMARY"
echo "------------------------------------------------------------"
echo " 📁 Directory: $DIRECTORY"
echo " 🧩 Profile: $PROFILE_NAME"
echo " 🌐 Remote: $REMOTE_SERVER"
echo " ⏱ Duration: ${DURATION}s"
echo " 📡 Status: 🟢 SUCCESS"
echo " 🧪 Dry Run: $DRY_RUN"
else
echo " 🔴 SYNC SUMMARY"
echo "------------------------------------------------------------"
echo " 📁 Directory: $DIRECTORY"
echo " 🧩 Profile: $PROFILE_NAME"
echo " 🌐 Remote: $REMOTE_SERVER"
echo " ⏱ Duration: ${DURATION}s"
echo " 📡 Status: 🔴 FAILED"
echo " 🧪 Dry Run: $DRY_RUN"
fi
echo "============================================================"
info "️ Sync complete"
} }
main main
+111 -70
View File
@@ -3,38 +3,50 @@
# ----------------- UNRAID OPS COMMON LIBRARY (CORE FRAMEWORK v1) ------------------------------- # ----------------- UNRAID OPS COMMON LIBRARY (CORE FRAMEWORK v1) -------------------------------
# ----------------------------------------------------------------------------------------------- # -----------------------------------------------------------------------------------------------
# OUTPUT HELPERS # ----------------------------- Output Helpers -----------------------------
info() { echo "[INFO] $*"; } info() {
warn() { echo "[WARN] $*"; } echo "[INFO] $*"
error() { echo "[ERROR] $*"; exit 1; } }
log() { [[ "${ENABLE_LOGGING:-false}" == true ]] && echo "[LOG] $*"; }
# DEFAULT FLAGS warn() {
DRY_RUN=${DRY_RUN:-false} echo "[WARN] $*"
ENABLE_LOGGING=${ENABLE_LOGGING:-false} }
SHOW_STATUS=${SHOW_STATUS:-false}
# ARGUMENT PARSER error() {
echo "[ERROR] $*"
}
log() {
[[ "$ENABLE_LOGGING" == true ]] && echo "[LOG] $*"
}
# ----------------------------- Argument Parsing -----------------------------
parse_args() { parse_args() {
DRY_RUN=${DRY_RUN:-false}
ENABLE_LOGGING=${ENABLE_LOGGING:-false}
for ARG in "$@"; do for ARG in "$@"; do
if [[ "$ARG" == *=* ]]; then if [[ "$ARG" == *=* ]]; then
VAR_NAME="${ARG%%=*}" VAR_NAME="${ARG%%=*}"
VAR_VALUE="${ARG#*=}" VAR_VALUE="${ARG#*=}"
case "$VAR_NAME" in case "$VAR_NAME" in
LOG) LOG)
[[ "$VAR_VALUE" == "true" ]] && ENABLE_LOGGING=true if [[ "$VAR_VALUE" == "true" ]]; then
[[ "$VAR_VALUE" == "false" ]] && ENABLE_LOGGING=false ENABLE_LOGGING=true
log "Logging set to $ENABLE_LOGGING" info "Verbose logging enabled"
elif [[ "$VAR_VALUE" == "false" ]]; then
ENABLE_LOGGING=false
else
warn "Invalid value for LOG: $VAR_VALUE (use true/false)"
fi
;; ;;
*) *)
if declare -p "$VAR_NAME" &>/dev/null; then if declare -p "$VAR_NAME" &>/dev/null; then
printf -v "$VAR_NAME" '%s' "$VAR_VALUE" printf -v "$VAR_NAME" '%s' "$VAR_VALUE"
log "Override $VAR_NAME -> $VAR_VALUE" log "Overriding $VAR_NAME -> $VAR_VALUE"
else else
warn "Unknown variable: $VAR_NAME" warn "Unknown variable $VAR_NAME, ignoring."
fi fi
;; ;;
esac esac
@@ -43,11 +55,11 @@ parse_args() {
case "$ARG" in case "$ARG" in
--dry-run|-n) --dry-run|-n)
DRY_RUN=true DRY_RUN=true
info "Dry-run enabled" info "Dry-run mode enabled"
;; ;;
--log) --log)
ENABLE_LOGGING=true ENABLE_LOGGING=true
info "Logging enabled" info "Verbose logging enabled"
;; ;;
--no-log) --no-log)
ENABLE_LOGGING=false ENABLE_LOGGING=false
@@ -55,34 +67,40 @@ parse_args() {
--status|--summary) --status|--summary)
SHOW_STATUS=true SHOW_STATUS=true
;; ;;
;;
--help|-h) --help|-h)
echo "Usage: script [--dry-run|-n] [--log] [--status] [VAR=value]" echo "Usage: script <dir> [--dry-run|-n] [--log] [LOG=true] [VAR=value ...]"
exit 0 exit 0
;; ;;
*) *)
warn "Unknown argument: $ARG" warn "Unknown argument $ARG"
;; ;;
esac esac
fi fi
done done
} }
# VALIDATION # ----------------------------- Validation -----------------------------
require_var() { require_var() {
local var="$1" local var="$1"
[[ -z "${!var:-}" ]] && error "Missing required variable: $var" if [[ -z "${!var:-}" ]]; then
error "Required variable $var is not set."
exit 1
fi
} }
validate_int() { validate_int() {
local name="$1" local name="$1"
local value="$2" local value="$2"
[[ ! "${value:-}" =~ ^[0-9]+$ ]] && error "$name must be an integer" if ! [[ "${value:-}" =~ ^[0-9]+$ ]]; then
error "$name must be a non-negative integer."
exit 1
fi
} }
# HOST DETECTION # ----------------------------- Host Detection -----------------------------
detect_hosts() { detect_hosts() {
LOCAL_HOSTNAME="$(hostname)" LOCAL_HOSTNAME="$(hostname)"
if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then
@@ -92,104 +110,127 @@ detect_hosts() {
LOCAL_SERVER_NAME="$HOST2" LOCAL_SERVER_NAME="$HOST2"
REMOTE_SERVER_NAME="$HOST1" REMOTE_SERVER_NAME="$HOST1"
else else
error "Unknown host: $LOCAL_HOSTNAME" error "Local hostname ($LOCAL_HOSTNAME) not recognized."
exit 1
fi fi
declare -gA SSH_KEYS declare -A SSH_KEYS
SSH_KEYS["$HOST1|$HOST2"]="$HOST1_SSH_KEY" SSH_KEYS["$HOST1|$HOST2"]="$HOST1_SSH_KEY"
SSH_KEYS["$HOST2|$HOST1"]="$HOST2_SSH_KEY" SSH_KEYS["$HOST2|$HOST1"]="$HOST2_SSH_KEY"
KEY_ID="$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME" KEY_ID="$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME"
SSH_KEY="${SSH_KEYS[$KEY_ID]:-}" SSH_KEY="${SSH_KEYS[$KEY_ID]}"
[[ -z "$SSH_KEY" ]] && error "Missing SSH key for $KEY_ID" if [[ -z "$SSH_KEY" ]]; then
error "No SSH key defined for $LOCAL_SERVER_NAME$REMOTE_SERVER_NAME"
exit 1
fi
log "Host mapping: $LOCAL_SERVER_NAME -> $REMOTE_SERVER_NAME" info "Local: $LOCAL_SERVER_NAME → Remote: $REMOTE_SERVER_NAME"
log "SSH key: $SSH_KEY"
} }
# REMOTE RESOLUTION # ----------------------------- Remote IP -----------------------------
resolve_remote_ip() { resolve_remote_ip() {
log "Resolving Tailscale IP for $REMOTE_SERVER_NAME..."
log "Resolving remote IP..."
REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)
[[ -z "$REMOTE_SERVER" ]] && error "Failed to resolve IP for $REMOTE_SERVER_NAME" if [[ -z "$REMOTE_SERVER" ]]; then
error "Could not resolve Tailscale IP for $REMOTE_SERVER_NAME"
exit 1
fi
log "Remote IP: $REMOTE_SERVER" info "Remote IP: $REMOTE_SERVER"
} }
# ----------------------------- Connectivity -----------------------------
check_remote_online() { check_remote_online() {
log "Pinging $REMOTE_SERVER..."
ping -c 1 "$REMOTE_SERVER" &>/dev/null ping -c 1 "$REMOTE_SERVER" &>/dev/null
} }
# SAFE SSH WRAPPER # ----------------------------- Containers -----------------------------
ssh_exec() { declare -a RUNNING_CONTAINERS=()
ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "$@"
}
# CONTAINER CONTROL
RUNNING_CONTAINERS=()
stop_containers() { stop_containers() {
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]]; then
log "No containers defined to stop"
return
fi
[[ ${#CRITICAL_CONTAINER_NAMES[@]:-0} -eq 0 ]] && return info "Stopping containers on $REMOTE_SERVER_NAME..."
RUNNING_CONTAINERS=() RUNNING_CONTAINERS=()
for container in "${CRITICAL_CONTAINER_NAMES[@]}"; do for container in "${CRITICAL_CONTAINER_NAMES[@]}"; do
log "Checking container: $container"
STATUS=$(ssh_exec "docker inspect -f '{{.State.Running}}' $container 2>/dev/null" || echo "false") STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"docker inspect -f '{{.State.Running}}' $container 2>/dev/null" || echo "false")
if [[ "$STATUS" == "true" ]]; then if [[ "$STATUS" == "true" ]]; then
info "Stopping $container" info "Stopping $container"
RUNNING_CONTAINERS+=("$container") RUNNING_CONTAINERS+=("$container")
ssh_exec "docker stop $container" || warn "Failed to stop $container" ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $container"
else
log "$container already stopped"
fi fi
done done
} }
start_containers() { start_containers() {
if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then
log "No containers to restart"
return
fi
[[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]] && return info "Starting containers on $REMOTE_SERVER_NAME..."
for container in "${RUNNING_CONTAINERS[@]}"; do for container in "${RUNNING_CONTAINERS[@]}"; do
if [[ " ${DELAYED_CONTAINERS[*]} " == *" $container "* ]]; then
if [[ " ${DELAYED_CONTAINERS[*]:-} " == *" $container "* ]]; then info "Delaying $container (${CONTAINER_DELAY}s)"
info "Delaying $container ($CONTAINER_DELAY s)"
sleep "$CONTAINER_DELAY" sleep "$CONTAINER_DELAY"
fi fi
info "Starting $container" info "Starting $container"
ssh_exec "docker start $container" || warn "Failed to start $container" ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $container"
done done
} }
# RSYNC OPTS # ----------------------------- Rsync Options -----------------------------
get_rsync_opts() { get_rsync_opts() {
if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then
IFS=' ' read -r -a RSYNC_OPTS <<< "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]}" read -r -a RSYNC_OPTS <<< "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]}"
log "Using profile rsync opts" log "Using profile rsync options: ${PROFILE_RSYNC_OPTS[$PROFILE_NAME]}"
else else
RSYNC_OPTS=("${DEFAULT_RSYNC_OPTS[@]:-}") RSYNC_OPTS=("${DEFAULT_RSYNC_OPTS[@]}")
log "Using default rsync opts" log "Using default rsync options: ${DEFAULT_RSYNC_OPTS[*]}"
fi fi
} }
# STATUS OUTPUT # ----------------------------- Status Output -----------------------------
show_status() { show_status() {
echo "===== RSYNC STATUS ====="
echo "Local: $LOCAL_SERVER_NAME"
echo "Remote: $REMOTE_SERVER_NAME"
echo "Remote IP: $REMOTE_SERVER"
echo "SSH Key: $SSH_KEY"
echo echo
echo "==================================================" echo "Directory: $DIRECTORY"
echo " RSYNC OPS STATUS" echo "Profile: $PROFILE_NAME"
echo "--------------------------------------------------" echo
echo " Local: $LOCAL_SERVER_NAME" echo "Max Procs: $MAX_RSYNC_PROCS"
echo " Remote: $REMOTE_SERVER_NAME" echo "Bandwidth: $BW_LIMIT KB/s"
echo " IP: ${REMOTE_SERVER:-unset}" echo "Retries: $RETRY_COUNT"
echo " Dry Run: $DRY_RUN" echo "Sleep: $SLEEP"
echo " Logging: $ENABLE_LOGGING" echo "Dry Run: $DRY_RUN"
echo " Profile: ${PROFILE_NAME:-default}" echo
echo "==================================================" echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]:-(none)}"
echo "Delayed: ${DELAYED_CONTAINERS[*]:-(none)}"
echo
echo "Excludes: ${EXCLUDE_DIRS[*]:-(none)}"
echo
echo "Rsync Opts:"
printf ' %s\n' "${RSYNC_OPTS[@]}"
echo "========================="
} }