208 lines
8.7 KiB
Bash
208 lines
8.7 KiB
Bash
#!/bin/bash
|
|
# ----------------------------------------------------------------------------------------------
|
|
# --------------------------------- Rsync Core Script ------------------------------------------
|
|
# ----------------------------------------------------------------------------------------------
|
|
# Core rsync script — called per share or per appdata profile.
|
|
# Profile is inferred from the directory basename (lowercased).
|
|
# If no profile match is found all settings fall through to global defaults in Master.conf.
|
|
#
|
|
# After each successful sync, logs the transfer to bandwidth_monitor.sh for weekly reporting.
|
|
# Log entry: date | time | profile | duration | status
|
|
#
|
|
# Usage:
|
|
# rsync.sh /mnt/user/Movies — media share, uses global defaults
|
|
# rsync.sh /mnt/user/appdata-Failover/Arrs_Stack — matched to [arrs_stack] profile
|
|
# rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --dry-run --log
|
|
# ---------------- User Variables, Please adjust in Master.conf as needed ----------------------
|
|
# ----------------------------------------------------------------------------------------------
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../Master.conf"
|
|
source "$SCRIPT_DIR/../common.sh"
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# Separate the positional directory argument from flag/key=value args.
|
|
# Optional --profile=name overrides the basename profile inference.
|
|
# -----------------------------------------------------------------------------------------------
|
|
DIRECTORY=""
|
|
PROFILE_OVERRIDE=""
|
|
RAW_ARGS=()
|
|
|
|
for ARG in "$@"; do
|
|
case "$ARG" in
|
|
--profile=*) PROFILE_OVERRIDE="${ARG#--profile=}" ;;
|
|
--*|*=*) RAW_ARGS+=("$ARG") ;;
|
|
*) [[ -z "$DIRECTORY" ]] && DIRECTORY="$ARG" || RAW_ARGS+=("$ARG") ;;
|
|
esac
|
|
done
|
|
|
|
parse_args "${RAW_ARGS[@]}"
|
|
|
|
[[ -z "$DIRECTORY" ]] && error "No directory specified. Usage: rsync.sh <dir> [--dry-run] [--log] [--profile=name]" && exit 1
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_GEAR Setup ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
|
|
|
detect_hosts
|
|
resolve_remote_ip
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# Profile inference — basename of directory lowercased
|
|
# Optional --profile=name overrides basename inference
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
if [[ -n "$PROFILE_OVERRIDE" ]]; then
|
|
PROFILE_NAME="$PROFILE_OVERRIDE"
|
|
info "$ICON_GEAR Profile override: $PROFILE_NAME"
|
|
else
|
|
PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]')
|
|
info "$ICON_GEAR Loading profile: $PROFILE_NAME"
|
|
fi
|
|
|
|
# Acquire per-profile lock and check global concurrent limit
|
|
acquire_rsync_lock "$PROFILE_NAME"
|
|
|
|
# Scalar overrides
|
|
BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
|
|
RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
|
|
SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
|
|
CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY}
|
|
|
|
# Array overrides
|
|
read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]:-}"
|
|
read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]:-}"
|
|
read -r -a EXCLUDE_DIRS <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-}"
|
|
|
|
# Local and remote use the same container list — same naming scheme on both servers
|
|
LOCAL_CRITICAL_CONTAINER_NAMES=("${CRITICAL_CONTAINER_NAMES[@]}")
|
|
|
|
[[ "$SHOW_STATUS" == true ]] && show_status && exit 0
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SHIELD Pre-flight Checks ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━"
|
|
|
|
# Disk temp check — before touching remote or moving any data
|
|
# Returns: 0=OK 1=warn(skip this profile) 2=crit(abort all remaining)
|
|
check_local_disk_temps
|
|
TEMP_RESULT=$?
|
|
if [[ "$TEMP_RESULT" -eq 2 ]]; then
|
|
error "Drive temps CRITICAL — aborting sync for all remaining profiles"
|
|
exit 2 # caller (daily_sync_maintenance.sh) sees exit 2 → stops all syncs
|
|
elif [[ "$TEMP_RESULT" -eq 1 ]]; then
|
|
warn "Drive temps too high — skipping profile [$PROFILE_NAME]"
|
|
exit 1 # caller sees exit 1 → skips this profile, continues to next
|
|
else
|
|
success "Drive temps OK — $TEMP_CHECK_RESULT"
|
|
fi
|
|
|
|
check_connectivity
|
|
check_remote_rootfs
|
|
check_remote_share "$DIRECTORY"
|
|
check_remote_disks "$DIRECTORY"
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_STOP $ICON_CONTAINERS Containers ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_STOP $ICON_CONTAINERS Containers ━━━"
|
|
|
|
# Stop local containers first — flush local databases before pushing
|
|
stop_local_containers
|
|
|
|
# Stop remote containers — prevent writes while receiving
|
|
stop_containers
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SYNC Transfer ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Transfer ━━━"
|
|
echo "$ICON_RUN Source: $DIRECTORY"
|
|
echo "$ICON_NET Remote: $REMOTE_SERVER:$DIRECTORY"
|
|
echo "$ICON_GEAR Profile: $PROFILE_NAME"
|
|
echo ""
|
|
|
|
get_rsync_opts
|
|
|
|
# Append profile excludes
|
|
for ex in "${EXCLUDE_DIRS[@]}"; do
|
|
[[ -n "$ex" ]] && RSYNC_OPTS+=(--exclude="$ex")
|
|
done
|
|
|
|
[[ "$DRY_RUN" == true ]] && RSYNC_OPTS+=("--dry-run") && warn "DRY RUN — no changes will be made"
|
|
|
|
START=$(date +%s)
|
|
RSYNC_SUCCESS=false
|
|
|
|
for i in $(seq 1 "$RETRY_COUNT"); do
|
|
info "$ICON_RETRY Attempt $i of $RETRY_COUNT..."
|
|
|
|
if rsync "${RSYNC_OPTS[@]}" \
|
|
-e "ssh -i $SSH_KEY -T -o Compression=no -o IPQoS=throughput" \
|
|
"$DIRECTORY" "root@${REMOTE_SERVER}:$(dirname "$DIRECTORY")/"; then
|
|
echo "$ICON_DONE Rsync complete"
|
|
RSYNC_SUCCESS=true
|
|
break
|
|
else
|
|
warn "$ICON_RETRY Rsync failed (attempt $i/$RETRY_COUNT)"
|
|
[[ "$i" -lt "$RETRY_COUNT" ]] && info "Retrying in ${SLEEP}s..." && sleep "$SLEEP"
|
|
fi
|
|
done
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_START $ICON_CONTAINERS Containers ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_START $ICON_CONTAINERS Containers ━━━"
|
|
|
|
# Start remote containers first — they can be coming up while local restarts
|
|
start_containers
|
|
|
|
# Start local containers
|
|
start_local_containers
|
|
|
|
END=$(date +%s)
|
|
DURATION=$((END - START))
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# Log transfer to bandwidth monitor — only on successful non-dry-run syncs
|
|
# Reliable format: date|time|profile|duration|status
|
|
# Does not parse rsync output — version-proof and always works
|
|
# -----------------------------------------------------------------------------------------------
|
|
BANDWIDTH_MONITOR="$SCRIPT_DIR/../Monitors/bandwidth_monitor.sh"
|
|
|
|
if [[ "$DRY_RUN" == false ]] && [[ -f "$BANDWIDTH_MONITOR" ]]; then
|
|
STATUS="success"
|
|
[[ "$RSYNC_SUCCESS" == false ]] && STATUS="failed"
|
|
bash "$BANDWIDTH_MONITOR" --log-transfer "$PROFILE_NAME" "$DURATION" "$STATUS"
|
|
log "$ICON_BANDWIDTH Transfer logged to bandwidth monitor"
|
|
fi
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ Summary ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY SUMMARY ━━━━━"
|
|
echo "$ICON_RUN Directory: $DIRECTORY"
|
|
echo "$ICON_GEAR Profile: $PROFILE_NAME"
|
|
echo "$ICON_DISK Disk check: $([[ "$SKIP_DISK_CHECK" == "true" ]] && echo "skipped (ZFS pool)" || echo "passed")"
|
|
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
|
|
|
if [[ "$RSYNC_SUCCESS" == true ]]; then
|
|
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
|
notify "Rsync complete — $DIRECTORY ($PROFILE_NAME) in $(format_duration $DURATION)" "Rsync" "normal"
|
|
else
|
|
echo "$ICON_ERROR Status: $ICON_ERROR FAILED after $RETRY_COUNT attempts"
|
|
notify "Rsync failed — $DIRECTORY ($PROFILE_NAME) after $RETRY_COUNT attempts" "Rsync" "warning"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ "$RSYNC_SUCCESS" == false ]] && exit 1
|
|
exit 0 |