diff --git a/Master.conf b/Master.conf index 888c091..bd310d4 100644 --- a/Master.conf +++ b/Master.conf @@ -21,7 +21,6 @@ HOST1="unRAID-Gmer4Lfe" HOST2="unRAID-Jayred365" - #----------------------------------------------------------------------------------------------- #----------------- User Variables -------------------------------------------------------------- #----------------------------------------------------------------------------------------------- @@ -36,13 +35,13 @@ SLEEP=300 # Max number of concurrently running rsync processes MAX_RSYNC_PROCS=1 # Container start/stop/restart before and after rsync -CRITICAL_CONTAINER_NAMES=("") +CRITICAL_CONTAINER_NAMES=() # Containers that need delayed before starting -DELAYED_CONTAINERS=("") +DELAYED_CONTAINERS=() # Delay in seconds between starting containers, useful for things like Authelia CONTAINER_DELAY=5 # Not include logs during transfer -EXCLUDE_DIRS=("") +EXCLUDE_DIRS=() #-------------------- unRAID essentail scripts ------------------ # unRAID reboot script user warning time (seconds) @@ -114,7 +113,7 @@ PROFILE_CONTAINER_DELAY["important-data"]=10 PROFILE_CONTAINER_DELAY["emby"]=5 # Not include logs or other file types during transfer per profile declare -A PROFILE_EXCLUDE_DIRS -PROFILE_EXCLUDE_DIRS["arrs_stack"]="logs *.tmp" +PROFILE_EXCLUDE_DIRS["arrs_stack"]="logs" PROFILE_EXCLUDE_DIRS["critical-data"]="logs *.tmp" PROFILE_EXCLUDE_DIRS["gmer4lfe"]="logs *.tmp" PROFILE_EXCLUDE_DIRS["important-data"]="logs *.tmp" diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index 5594735..dcb49e8 100644 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -4,54 +4,53 @@ set -e #----------------------------------------------------------------------------------------------- # --------------------------- Core Rsync Script for Unraid ------------------------------------- #----------------------------------------------------------------------------------------------- -#----------------- User Variables, Please adjust in Master.conf as needed ---------------------- -#----------------------------------------------------------------------------------------------- -# -# User variables can be adjusted in Master.conf -# -# Scripts now just contain runtime logic -# -# This new setup allows for profiles and arguments to use the core script -# Use arguments in unRAID User Plugin for directory -# Example: /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-test/Arrs/ -# -# /mnt/user/appdata-test/Arrs/ is the argument -# If the last name /Arrs/ matches a profile in Master.conf, then profile defaults are used -# Otherwise, global configuration in Master.conf is used -#----------------------------------------------------------------------------------------------- -#---------------End Of User Variables, Please adjust in Master.conf as needed ------------------ -#----------------------------------------------------------------------------------------------- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../common.sh" -# Load central config +# Load config FIRST load_master_conf -# Input Arguments +#---------------------------------------------------------- +# Input Arguments (MUST come before parse_args) +#---------------------------------------------------------- DIRECTORY="$1" if [[ -z "$DIRECTORY" ]]; then echo "Usage: $0 [--dry-run|-n] [VAR=value ...]" exit 1 fi -shift # remove DIRECTORY +shift -# Parse CLI args (flags + VAR=value) +#---------------------------------------------------------- +# Parse CLI args AFTER shifting directory +#---------------------------------------------------------- parse_args "$@" +#---------------------------------------------------------- +# At this point, common.sh has already: +# - LOCAL_SERVER_NAME +# - REMOTE_SERVER_NAME +# - REMOTE_SERVER +# - SSH_KEY +#---------------------------------------------------------- + # Determine profile PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]') echo "Detected profile: $PROFILE_NAME" -# Apply profile defaults or fallback to Master.conf +# Apply profile defaults (safe fallbacks) MAX_RSYNC_PROCS=${PROFILE_MAX_PROCS[$PROFILE_NAME]:-$MAX_RSYNC_PROCS} BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT} -CRITICAL_CONTAINER_NAMES=(${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]}) -DELAYED_CONTAINERS=(${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]}) + +# Convert string → array safely +IFS=' ' read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]}" +IFS=' ' read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]}" + CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY} RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT} SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP} -EXCLUDE_DIRS=(${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-${EXCLUDE_DIRS[@]}}) + +IFS=' ' read -r -a EXCLUDE_DIRS <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-${EXCLUDE_DIRS[*]}}" # Validate numeric variables validate_int MAX_RSYNC_PROCS "$MAX_RSYNC_PROCS" @@ -60,19 +59,30 @@ validate_int CONTAINER_DELAY "$CONTAINER_DELAY" validate_int RETRY_COUNT "$RETRY_COUNT" validate_int SLEEP "$SLEEP" -echo "Settings: MAX_RSYNC_PROCS=$MAX_RSYNC_PROCS, BW_LIMIT=$BW_LIMIT, CONTAINERS=${CRITICAL_CONTAINER_NAMES[*]}, EXCLUDES=${EXCLUDE_DIRS[*]}" +echo "Settings:" +echo " MAX_RSYNC_PROCS=$MAX_RSYNC_PROCS" +echo " BW_LIMIT=$BW_LIMIT" +echo " CONTAINERS=${CRITICAL_CONTAINER_NAMES[*]}" +echo " EXCLUDES=${EXCLUDE_DIRS[*]}" # Dependency check for cmd in rsync ssh ping timeout docker tailscale; do - command -v "$cmd" >/dev/null 2>&1 || { echo "Error: Required command '$cmd' is not installed."; exit 1; } + command -v "$cmd" >/dev/null 2>&1 || { + echo "Error: Required command '$cmd' is not installed." + exit 1 + } done trap 'echo "Script interrupted. Exiting."; exit 130' INT TERM -# Helper functions +#---------------------------------------------------------- +# Helper Functions +#---------------------------------------------------------- + wait_for_rsync() { for attempt in $(seq 1 "$RETRY_COUNT"); do RSYNC_COUNT=$(pgrep -fc "rsync.*root@${REMOTE_SERVER}" || true) + if [ "$RSYNC_COUNT" -ge "$MAX_RSYNC_PROCS" ]; then echo "[$LOCAL_SERVER_NAME] Waiting... ($RSYNC_COUNT active rsync processes)" sleep "$SLEEP" @@ -80,6 +90,7 @@ wait_for_rsync() { return 0 fi done + echo "Too many rsync processes, exiting." exit 1 } @@ -93,12 +104,17 @@ declare -a RUNNING_CONTAINERS=() stop_containers() { echo "[$LOCAL_SERVER_NAME] Checking containers on ${REMOTE_SERVER_NAME}..." RUNNING_CONTAINERS=() + for container in "${CRITICAL_CONTAINER_NAMES[@]}"; do + [ -z "$container" ] && continue + STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "docker inspect -f '{{.State.Running}}' $container 2>/dev/null" || echo "false") + if [ "$STATUS" == "true" ]; then echo "Stopping $container" RUNNING_CONTAINERS+=("$container") + if [ "$DRY_RUN" = true ]; then echo "[DRY-RUN] Would stop container $container" else @@ -110,11 +126,13 @@ stop_containers() { start_containers() { echo "[$LOCAL_SERVER_NAME] Restarting containers on ${REMOTE_SERVER_NAME}..." + for container in "${RUNNING_CONTAINERS[@]}"; do if [[ " ${DELAYED_CONTAINERS[*]} " == *" $container "* ]]; then echo "Delaying start of $container by ${CONTAINER_DELAY}s..." [ "$DRY_RUN" = false ] && sleep "$CONTAINER_DELAY" fi + if [ "$DRY_RUN" = true ]; then echo "[DRY-RUN] Would start container $container" else @@ -123,7 +141,10 @@ start_containers() { done } -# Main sync logic +#---------------------------------------------------------- +# Main +#---------------------------------------------------------- + main() { echo "[$LOCAL_SERVER_NAME] Syncing $DIRECTORY → $REMOTE_SERVER:$DIRECTORY" @@ -133,7 +154,10 @@ main() { sleep "$SLEEP" done - check_server_online || { echo "Remote $REMOTE_SERVER_NAME unreachable. Exiting."; exit 1; } + check_server_online || { + echo "Remote $REMOTE_SERVER_NAME unreachable. Exiting." + exit 1 + } wait_for_rsync stop_containers @@ -148,22 +172,25 @@ main() { --no-whole-file ) - # Apply multiple exclude patterns for pattern in "${EXCLUDE_DIRS[@]}"; do - RSYNC_OPTS+=(--exclude="$pattern") + [ -n "$pattern" ] && RSYNC_OPTS+=(--exclude="$pattern") done [ "$DRY_RUN" = true ] && RSYNC_OPTS+=("--dry-run") for attempt in $(seq 1 "$RETRY_COUNT"); do echo "Starting rsync attempt $attempt/$RETRY_COUNT..." + if rsync "${RSYNC_OPTS[@]}" \ -e "ssh -i \"$SSH_KEY\" -T -o Compression=no -o IPQoS=throughput" \ "$DIRECTORY" "root@${REMOTE_SERVER}:$DIRECTORY"; then + echo "Rsync completed successfully." break + else echo "Rsync attempt $attempt failed." + if [ "$attempt" -lt "$RETRY_COUNT" ]; then echo "Retrying in $SLEEP seconds..." sleep "$SLEEP" @@ -179,5 +206,4 @@ main() { echo "[$LOCAL_SERVER_NAME] Sync complete." } -# Execute main \ No newline at end of file diff --git a/common.sh b/common.sh index 3cda593..681e4f1 100644 --- a/common.sh +++ b/common.sh @@ -3,21 +3,14 @@ #----------------------------------------------------------------------------------------------- # --------------------- Common Shared Functions for Unraid Scripts ----------------------------- #----------------------------------------------------------------------------------------------- -#----------------- User Variables, Please adjust in Master.conf as needed ---------------------- -#----------------------------------------------------------------------------------------------- -# -# Nothing to adjust here, all user variables should be in Master.conf -# -# This is just a script that has all common functions and logic that are shared -# between the different scripts. -# -# This way we can avoid code duplication and have a single source -# of truth for all common functions and logic. -#----------------------------------------------------------------------------------------------- -#---------------End Of User Variables, Please adjust in Master.conf as needed ------------------ +# This file contains ONLY reusable logic. +# No code should execute on load — only functions. +# All user configuration lives in Master.conf #----------------------------------------------------------------------------------------------- +#----------------------------------------------------------------------------------------------- # Load Master.conf +#----------------------------------------------------------------------------------------------- load_master_conf() { local script_dir script_dir="$(cd "$(dirname "${BASH_SOURCE[1]}")" && pwd)" @@ -32,7 +25,9 @@ load_master_conf() { fi } -# Parse CLI args +#----------------------------------------------------------------------------------------------- +# Parse CLI args (flags + VAR=value overrides) +#----------------------------------------------------------------------------------------------- parse_args() { DRY_RUN=${DRY_RUN:-false} @@ -49,74 +44,92 @@ parse_args() { fi else case "$ARG" in - --dry-run|-n) DRY_RUN=true ;; + --dry-run|-n) + DRY_RUN=true + ;; --help|-h) echo "Usage: script [--dry-run|-n] [VAR=value ...]" - exit 0 ;; + exit 0 + ;; *) - echo "Warning: Unknown argument $ARG" ;; + echo "Warning: Unknown argument $ARG" + ;; esac fi done } -# Validate integer +#----------------------------------------------------------------------------------------------- +# Validate integer (non-negative) +#----------------------------------------------------------------------------------------------- validate_int() { local name="$1" local value="$2" + if ! [[ "${value:-}" =~ ^[0-9]+$ ]]; then echo "Error: $name must be a non-negative integer." exit 1 fi } +#----------------------------------------------------------------------------------------------- # Require variable to be set +#----------------------------------------------------------------------------------------------- require_var() { local var="$1" + if [[ -z "${!var:-}" ]]; then echo "Error: Required variable $var is not set." exit 1 fi } -# Detect local hostname -LOCAL_HOSTNAME="$(hostname)" -if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then - LOCAL_SERVER_NAME="$HOST1" - REMOTE_SERVER_NAME="$HOST2" -elif [[ "$LOCAL_HOSTNAME" == "$HOST2" ]]; then - LOCAL_SERVER_NAME="$HOST2" - REMOTE_SERVER_NAME="$HOST1" -else - echo "Error: Local hostname ($LOCAL_HOSTNAME) not recognized." - exit 1 -fi -echo "Local server: $LOCAL_SERVER_NAME" -echo "Remote server: $REMOTE_SERVER_NAME" - #----------------------------------------------------------------------------------------------- -#--------------------------- Remote Server Ip Resolution ---------------------------------------- +# Detect hosts + networking + SSH key #----------------------------------------------------------------------------------------------- -# Resolve Tailscale IP dynamically -REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) -if [[ -z "$REMOTE_SERVER" ]]; then - echo "Error: Could not resolve Tailscale IP for $REMOTE_SERVER_NAME" - exit 1 -fi +detect_hosts() { + # Ensure required config exists + require_var HOST1 + require_var HOST2 -#----------------------------------------------------------------------------------------------- -#----------------------------- SSH Key Mapping -------------------------------------------------- -#----------------------------------------------------------------------------------------------- + # Detect local hostname + LOCAL_HOSTNAME="$(hostname)" -declare -A SSH_KEYS -SSH_KEYS["$HOST1|$HOST2"]="/root/.ssh/Gmer4Lfe-rsync-key" -SSH_KEYS["$HOST2|$HOST1"]="/root/.ssh/Jayred365-rsync-key" + if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then + LOCAL_SERVER_NAME="$HOST1" + REMOTE_SERVER_NAME="$HOST2" + elif [[ "$LOCAL_HOSTNAME" == "$HOST2" ]]; then + LOCAL_SERVER_NAME="$HOST2" + REMOTE_SERVER_NAME="$HOST1" + else + echo "Error: Local hostname ($LOCAL_HOSTNAME) not recognized." + exit 1 + fi -KEY_ID="$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME" -if [[ -n "${SSH_KEYS[$KEY_ID]}" ]]; then - SSH_KEY="${SSH_KEYS[$KEY_ID]}" -else - echo "Error: No SSH key defined for $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" - exit 1 -fi -echo "Using SSH key: $SSH_KEY" \ No newline at end of file + echo "Local server: $LOCAL_SERVER_NAME" + echo "Remote server: $REMOTE_SERVER_NAME" + + # Resolve Tailscale IP + REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) + + if [[ -z "$REMOTE_SERVER" ]]; then + echo "Error: Could not resolve Tailscale IP for $REMOTE_SERVER_NAME" + exit 1 + fi + + # SSH Key Mapping (kept here for stability) + declare -A SSH_KEYS + SSH_KEYS["$HOST1|$HOST2"]="/root/.ssh/Gmer4Lfe-rsync-key" + SSH_KEYS["$HOST2|$HOST1"]="/root/.ssh/Jayred365-rsync-key" + + KEY_ID="$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME" + + if [[ -n "${SSH_KEYS[$KEY_ID]}" ]]; then + SSH_KEY="${SSH_KEYS[$KEY_ID]}" + else + echo "Error: No SSH key defined for $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" + exit 1 + fi + + echo "Using SSH key: $SSH_KEY" +} \ No newline at end of file