massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2

This commit is contained in:
2026-05-03 17:16:49 -04:00
parent 2691a35e80
commit ec7de648dc
72 changed files with 25640 additions and 14629 deletions
File diff suppressed because it is too large Load Diff
+232 -104
View File
@@ -1,30 +1,71 @@
#!/bin/bash
# ----------------------------------------------------------------------------------------------
# --------------------------------- Rsync Core Script ------------------------------------------
# ----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= Rsync Core Script ==========================================
# ==============================================================================================
# Core rsync script — called per share or per appdata profile.
# Called by orchestrators (daily/weekly/critical sync) and directly for manual syncs.
#
# ── PROFILE SYSTEM ────────────────────────────────────────────────────────────────────────────
# Profile is inferred from the directory basename (lowercased).
# If no profile match is found all settings fall through to global defaults in Master.conf.
# Override with --profile=name for explicit profile selection.
# If no profile match found → all settings fall back to global defaults in master.conf.
#
# After each successful sync, logs the transfer to bandwidth_monitor.sh for weekly reporting.
# Log entry: date | time | profile | duration | status
# Profiles define:
# PROFILE_RSYNC_OPTS — rsync flags (does NOT inherit DEFAULT_RSYNC_OPTS)
# PROFILE_BW_LIMIT — bandwidth limit in KB/s
# PROFILE_RETRY_COUNT — retry attempts before giving up
# PROFILE_SLEEP — seconds between retry attempts
# PROFILE_CRITICAL_CONTAINER_NAMES — containers stopped both sides before sync
# PROFILE_DELAYED_CONTAINERS — containers needing delay before starting after sync
# PROFILE_CONTAINER_DELAY — seconds before starting delayed containers
# PROFILE_EXCLUDE_DIRS — paths excluded from transfer
# PROFILE_REMOTE_RESTART_CONTAINERS — containers restarted on remote after dirty sync
# (critical-failover, emby-failover profiles)
# Was running → restart. Was stopped → leave stopped.
#
# Usage:
# rsync.sh /mnt/user/Movies — media share, uses global defaults
# rsync.sh /mnt/user/appdata-Failover/Arrs_Stack — matched to [arrs_stack] profile
# ── RSYNC ENABLE/DISABLE ──────────────────────────────────────────────────────────────────────
# Two-tier toggle system — checked at entry:
# Tier 1: RSYNC_ENABLED=false → all rsync stops
# Tier 2: Per-orchestrator flag (DAILY_RSYNC_ENABLED etc.) — checked by caller
# Direct calls to rsync.sh only check Tier 1
#
# ── BANDWIDTH LOGGING ─────────────────────────────────────────────────────────────────────────
# After each sync logs to bandwidth_monitor.sh --log-transfer:
# profile | duration_seconds | status | bytes_transferred
# Bytes captured from rsync --stats output — version-proof parsing.
# bandwidth_monitor.sh flags syncs exceeding BANDWIDTH_WARN_GB.
#
# ── DIRTY SYNC REMOTE RESTART ─────────────────────────────────────────────────────────────────
# Profiles using dirty sync (critical-failover, emby-failover) define
# PROFILE_REMOTE_RESTART_CONTAINERS — containers restarted on remote after sync completes.
# This ensures the remote picks up config changes synced during the dirty window.
# Was running → restart. Was stopped → leave stopped.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# check_rsync_enabled() — Tier 1 gate before any operation
# check_unraid_version_parity — refuses if servers on incompatible unRAID versions
# check_remote_docker_daemon — verifies remote daemon before container operations
# check_local_disk_temps() — temp check before transfer (exit 1=skip, 2=abort all)
# check_connectivity() — verifies remote reachable
# check_remote_rootfs() — aborts if remote rootfs nearly full
# check_remote_share() — aborts if target directory missing or empty
# check_remote_disks() — verifies all backing disks online on remote
# acquire_rsync_lock() — per-profile lock + global concurrent limit
# validate_unraid_cmd — notify validated before use
# Silent by default — only failures produce visible output
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# rsync.sh /mnt/user/Movies — media share, global defaults
# rsync.sh /mnt/user/appdata-Failover/Arrs_Stack — matched to [arrs_stack] profile
# rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --dry-run --log
# ---------------- User Variables, Please adjust in Master.conf as needed ----------------------
# ----------------------------------------------------------------------------------------------
# rsync.sh /mnt/user/appdata-Failover/Critical-Data --profile=critical-failover
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
# -----------------------------------------------------------------------------------------------
# Separate the positional directory argument from flag/key=value args.
# Optional --profile=name overrides the basename profile inference.
# -----------------------------------------------------------------------------------------------
# ── Separate positional directory arg from flags ───────────────────────────────────────────────
DIRECTORY=""
PROFILE_OVERRIDE=""
RAW_ARGS=()
@@ -32,185 +73,272 @@ RAW_ARGS=()
for ARG in "$@"; do
case "$ARG" in
--profile=*) PROFILE_OVERRIDE="${ARG#--profile=}" ;;
--*|*=*) RAW_ARGS+=("$ARG") ;;
*) [[ -z "$DIRECTORY" ]] && DIRECTORY="$ARG" || RAW_ARGS+=("$ARG") ;;
--*|*=*) RAW_ARGS+=("$ARG") ;;
*) [[ -z "$DIRECTORY" ]] && DIRECTORY="$ARG" || RAW_ARGS+=("$ARG") ;;
esac
done
parse_args "${RAW_ARGS[@]}"
[[ -z "$DIRECTORY" ]] && error "No directory specified. Usage: rsync.sh <dir> [--dry-run] [--log] [--profile=name]" && exit 1
[[ -z "$DIRECTORY" ]] && {
error "No directory specified"
error "Usage: rsync.sh <dir> [--dry-run] [--log] [--profile=name]"
exit 1
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
detect_hosts
# Tier 1 global gate — check before doing anything
# Tier 2 (per-orchestrator) is handled by the calling orchestrator
# Direct calls to rsync.sh only check Tier 1
# Tier 1 global gate — Tier 2 (per-orchestrator) checked by caller
if ! check_rsync_enabled; then
warn "RSYNC_ENABLED=false — exiting cleanly"
exit 0
fi
resolve_remote_ip
# -----------------------------------------------------------------------------------------------
# Profile inference — basename of directory lowercased
# Optional --profile=name overrides basename inference
# -----------------------------------------------------------------------------------------------
echo ""
# ── Profile inference ─────────────────────────────────────────────────────────────────────────
if [[ -n "$PROFILE_OVERRIDE" ]]; then
PROFILE_NAME="$PROFILE_OVERRIDE"
info "$ICON_GEAR Profile override: $PROFILE_NAME"
log "Profile override: $PROFILE_NAME"
else
PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]')
info "$ICON_GEAR Loading profile: $PROFILE_NAME"
log "Profile inferred: $PROFILE_NAME"
fi
# Acquire per-profile lock and check global concurrent limit
acquire_rsync_lock "$PROFILE_NAME"
# Scalar overrides
# ── Load profile settings ─────────────────────────────────────────────────────────────────────
BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY}
# Array overrides
read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]:-}"
read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]:-}"
read -r -a EXCLUDE_DIRS <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-}"
read -r -a REMOTE_RESTART_CONTAINERS <<< "${PROFILE_REMOTE_RESTART_CONTAINERS[$PROFILE_NAME]:-}"
# Local and remote use the same container list — same naming scheme on both servers
# Local containers use same names as remote (mirrored naming scheme)
LOCAL_CRITICAL_CONTAINER_NAMES=("${CRITICAL_CONTAINER_NAMES[@]}")
[[ "$SHOW_STATUS" == true ]] && show_status && exit 0
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SHIELD Pre-flight Checks ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Pre-flight Checks ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━"
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
# Disk temp check — before touching remote or moving any data
# Returns: 0=OK 1=warn(skip this profile) 2=crit(abort all remaining)
# Disk temp — before touching remote or moving data
# Exit 1 = skip this profile | Exit 2 = abort all remaining profiles
check_local_disk_temps
TEMP_RESULT=$?
if [[ "$TEMP_RESULT" -eq 2 ]]; then
error "Drive temps CRITICAL — aborting sync for all remaining profiles"
exit 2 # caller (daily_sync_maintenance.sh) sees exit 2 → stops all syncs
error "Drive temps CRITICAL — aborting all remaining syncs"
exit 2
elif [[ "$TEMP_RESULT" -eq 1 ]]; then
warn "Drive temps too high — skipping profile [$PROFILE_NAME]"
exit 1 # caller sees exit 1 → skips this profile, continues to next
warn "Drive temps high — skipping profile [$PROFILE_NAME]"
exit 1
else
success "Drive temps OK — $TEMP_CHECK_RESULT"
log "Drive temps OK — $TEMP_CHECK_RESULT"
fi
# Version parity — refuse if servers on incompatible unRAID versions
check_unraid_version_parity || exit 1
check_connectivity
check_remote_rootfs
check_remote_share "$DIRECTORY"
check_remote_disks "$DIRECTORY"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_STOP $ICON_CONTAINERS Containers ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_STOP $ICON_CONTAINERS Containers ━━━"
# Remote Docker daemon — check before attempting container operations
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]] || [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then
check_remote_docker_daemon || {
warn "Remote Docker daemon unresponsive — skipping container operations"
warn "Proceeding with rsync only — containers will not be stopped or restarted"
CRITICAL_CONTAINER_NAMES=()
LOCAL_CRITICAL_CONTAINER_NAMES=()
REMOTE_RESTART_CONTAINERS=()
}
fi
# Stop local containers first — flush local databases before pushing
stop_local_containers
# ==============================================================================================
# ━━━ Stop Containers ━━━
# ==============================================================================================
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then
echo ""
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
# Local first — flush local databases before pushing
stop_local_containers
# Remote next — prevent writes while receiving
stop_containers
fi
# Stop remote containers — prevent writes while receiving
stop_containers
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SYNC Transfer ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Transfer ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Transfer ━━━"
echo "$ICON_RUN Source: $DIRECTORY"
echo "$ICON_NET Remote: $REMOTE_SERVER:$DIRECTORY"
echo "$ICON_GEAR Profile: $PROFILE_NAME"
echo "$ICON_RUN Source: $DIRECTORY"
echo "$ICON_NET Remote: $REMOTE_SERVER:$DIRECTORY"
echo "$ICON_GEAR Profile: $PROFILE_NAME"
echo "$ICON_HOST Identity: $MY_ID$REMOTE_ID"
echo ""
get_rsync_opts
# Append profile excludes
for ex in "${EXCLUDE_DIRS[@]}"; do
for ex in "${EXCLUDE_DIRS[@]:-}"; do
[[ -n "$ex" ]] && RSYNC_OPTS+=(--exclude="$ex")
done
[[ "$DRY_RUN" == true ]] && RSYNC_OPTS+=("--dry-run") && warn "DRY RUN — no changes will be made"
# Add --stats to capture bytes transferred for bandwidth logging
RSYNC_OPTS+=(--stats)
[[ "$DRY_RUN" == true ]] && RSYNC_OPTS+=("--dry-run")
START=$(date +%s)
RSYNC_SUCCESS=false
BYTES_TRANSFERRED=0
ATTEMPT=0
for i in $(seq 1 "$RETRY_COUNT"); do
info "$ICON_RETRY Attempt $i of $RETRY_COUNT..."
for (( ATTEMPT=1; ATTEMPT<=RETRY_COUNT; ATTEMPT++ )); do
log "$ICON_RETRY Attempt $ATTEMPT of $RETRY_COUNT..."
if rsync "${RSYNC_OPTS[@]}" \
RSYNC_OUTPUT=$(rsync "${RSYNC_OPTS[@]}" \
-e "ssh -i $SSH_KEY -T -o Compression=no -o IPQoS=throughput" \
"$DIRECTORY" "root@${REMOTE_SERVER}:$(dirname "$DIRECTORY")/"; then
echo "$ICON_DONE Rsync complete"
"$DIRECTORY" "root@${REMOTE_SERVER}:$(dirname "$DIRECTORY")/" 2>&1)
RSYNC_EXIT=$?
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
# Parse bytes transferred from --stats output
BYTES_TRANSFERRED=$(echo "$RSYNC_OUTPUT" | \
awk '/Total transferred file size:/{gsub(/,/,"",$NF); gsub(/[^0-9]/,"",$NF); print $NF+0}')
BYTES_TRANSFERRED="${BYTES_TRANSFERRED:-0}"
log "$ICON_DONE Rsync complete — $BYTES_TRANSFERRED bytes transferred"
RSYNC_SUCCESS=true
break
else
warn "$ICON_RETRY Rsync failed (attempt $i/$RETRY_COUNT)"
[[ "$i" -lt "$RETRY_COUNT" ]] && info "Retrying in ${SLEEP}s..." && sleep "$SLEEP"
warn "$ICON_RETRY Rsync failed (attempt $ATTEMPT/$RETRY_COUNT)"
log "Exit code: $RSYNC_EXIT"
if [[ "$ATTEMPT" -lt "$RETRY_COUNT" ]]; then
log "Retrying in ${SLEEP}s..."
sleep "$SLEEP"
fi
fi
done
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_START $ICON_CONTAINERS Containers ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_START $ICON_CONTAINERS Containers ━━━"
# ==============================================================================================
# ━━━ Start Containers ━━━
# ==============================================================================================
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then
echo ""
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
# Remote first — can be coming up while local restarts
start_containers
# Local next
start_local_containers
fi
# Start remote containers first — they can be coming up while local restarts
start_containers
# ==============================================================================================
# ━━━ Remote Restart (dirty sync profiles) ━━━
# ==============================================================================================
# For dirty sync profiles (critical-failover, emby-failover) — restart containers on remote
# that were running before sync so they pick up config changes from the dirty sync window.
# Was running → restart. Was stopped → leave stopped.
if [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then
echo ""
echo "━━━ $ICON_START $ICON_CONTAINERS Remote Restart (post dirty sync) ━━━"
log "Restarting configured containers on $REMOTE_SERVER_NAME..."
# Start local containers
start_local_containers
for container in "${REMOTE_RESTART_CONTAINERS[@]}"; do
[[ -z "$container" ]] && continue
# Check if container was running before sync (still tracked via RUNNING_CONTAINERS)
WAS_RUNNING=false
for prev in "${RUNNING_CONTAINERS[@]:-}"; do
[[ "$prev" == "$container" ]] && WAS_RUNNING=true && break
done
if [[ "$WAS_RUNNING" == false ]]; then
# Not in stop list — check current remote state
REMOTE_STATUS=$(timeout 15 ssh -i "$SSH_KEY" \
-o ConnectTimeout=10 root@"$REMOTE_SERVER" \
"docker inspect -f '{{.State.Running}}' $container 2>/dev/null" 2>/dev/null)
[[ "$REMOTE_STATUS" != "true" ]] && \
log "$container not running on $REMOTE_SERVER_NAME — skipping remote restart" && \
continue
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart $container on $REMOTE_SERVER_NAME"
continue
fi
timeout 15 ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
"docker restart $container" >/dev/null 2>&1 && \
log "$ICON_STARTED $container restarted on $REMOTE_SERVER_NAME" || \
warn "Failed to restart $container on $REMOTE_SERVER_NAME"
done
fi
END=$(date +%s)
DURATION=$((END - START))
DURATION=$(( END - START ))
# -----------------------------------------------------------------------------------------------
# Log transfer to bandwidth monitor — only on successful non-dry-run syncs
# Reliable format: date|time|profile|duration|status
# Does not parse rsync output — version-proof and always works
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Bandwidth Logging ━━━
# ==============================================================================================
# Logs to bandwidth_monitor.sh — new format includes bytes transferred and warn flag.
# Only logs on actual runs (not dry-run) and only when bandwidth_monitor.sh exists.
BANDWIDTH_MONITOR="$SCRIPT_DIR/../Monitors/bandwidth_monitor.sh"
if [[ "$DRY_RUN" == false ]] && [[ -f "$BANDWIDTH_MONITOR" ]]; then
STATUS="success"
[[ "$RSYNC_SUCCESS" == false ]] && STATUS="failed"
bash "$BANDWIDTH_MONITOR" --log-transfer "$PROFILE_NAME" "$DURATION" "$STATUS"
log "$ICON_BANDWIDTH Transfer logged to bandwidth monitor"
bash "$BANDWIDTH_MONITOR" --log-transfer \
"$PROFILE_NAME" "$DURATION" "$STATUS" "$BYTES_TRANSFERRED"
log "$ICON_BANDWIDTH Transfer logged to bandwidth monitor ($BYTES_TRANSFERRED bytes)"
fi
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY SUMMARY ━━━━━"
echo "$ICON_RUN Directory: $DIRECTORY"
echo "$ICON_GEAR Profile: $PROFILE_NAME"
echo "$ICON_DISK Disk check: $([[ "$SKIP_DISK_CHECK" == "true" ]] && echo "skipped (ZFS pool)" || echo "passed")"
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
echo "━━━━━ $ICON_SUMMARY RSYNC SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_RUN Directory: $DIRECTORY"
echo "$ICON_GEAR Profile: $PROFILE_NAME"
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
[[ "$BYTES_TRANSFERRED" -gt 0 ]] && \
echo "$ICON_BANDWIDTH Transferred: $(awk "BEGIN {printf \"%.2fGB\", $BYTES_TRANSFERRED / 1073741824}")"
if [[ "$RSYNC_SUCCESS" == true ]]; then
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
notify "Rsync complete — $DIRECTORY ($PROFILE_NAME) in $(format_duration $DURATION)" "Rsync" "normal"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — no changes made"
elif [[ "$RSYNC_SUCCESS" == true ]]; then
log "$ICON_DONE Status: $ICON_SUCCESS DONE"
else
echo "$ICON_ERROR Status: $ICON_ERROR FAILED after $RETRY_COUNT attempts"
notify "Rsync failed$DIRECTORY ($PROFILE_NAME) after $RETRY_COUNT attempts" "Rsync" "warning"
echo "$ICON_ERROR Status: FAILED after $RETRY_COUNT attempts"
notify "Rsync FAILED$DIRECTORY ($PROFILE_NAME) after $RETRY_COUNT attempts on $(hostname)" \
"Rsync" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"
[[ "$RSYNC_SUCCESS" == false ]] && exit 1
[[ "$RSYNC_SUCCESS" == false ]] && [[ "$DRY_RUN" == false ]] && exit 1
exit 0