Structure Changes

Master.conf

@@ -3,13 +3,16 @@
 ############################## Master Variables for unRAID scripts ################################
 ##################################################################################################
 
+#-----------------------------------------------------------------------------------------------
+#----------------------------- Host Detection ---------------------------------------------------
+#-----------------------------------------------------------------------------------------------
+
 # Define the hostnames of both servers
 HOST1="unRAID-Gmer4Lfe"
 HOST2="unRAID-Jayred365"
 
 # Detect local hostname
 LOCAL_HOSTNAME="$(hostname)"
-
 if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then
     LOCAL_SERVER_NAME="$HOST1"
     REMOTE_SERVER_NAME="$HOST2"
@@ -20,50 +23,101 @@ else
     echo "Error: Local hostname ($LOCAL_HOSTNAME) not recognized."
     exit 1
 fi
-
 echo "Local server: $LOCAL_SERVER_NAME"
 echo "Remote server: $REMOTE_SERVER_NAME"
 
-# Resolve Tailscale IP dynamically, This will try to get the Tailscale IPv4 address for that host
+#-----------------------------------------------------------------------------------------------
+#----------------------------- Remote Server Resolution -----------------------------------------
+#-----------------------------------------------------------------------------------------------
+# Resolve Tailscale IP dynamically
 REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)
 if [[ -z "$REMOTE_SERVER" ]]; then
     echo "Error: Could not resolve Tailscale IP for $REMOTE_SERVER_NAME"
     exit 1
 fi
 
-# SSH key mapping per hostpair
+#-----------------------------------------------------------------------------------------------
+#----------------------------- SSH Key Mapping --------------------------------------------------
+#-----------------------------------------------------------------------------------------------
+
 declare -A SSH_KEYS
 SSH_KEYS["$HOST1|$HOST2"]="/root/.ssh/Gmer4Lfe-rsync-key"
 SSH_KEYS["$HOST2|$HOST1"]="/root/.ssh/Jayred365-rsync-key"
 
 KEY_ID="$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME"
-
 if [[ -n "${SSH_KEYS[$KEY_ID]}" ]]; then
     SSH_KEY="${SSH_KEYS[$KEY_ID]}"
 else
     echo "Error: No SSH key defined for $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME"
     exit 1
 fi
-
 echo "Using SSH key: $SSH_KEY"
 
-# Network speed limit 100Mbps=12500 KB/s, 200Mbps=25600 KB/s
+#-----------------------------------------------------------------------------------------------
+#----------------------------- Global Defaults --------------------------------------------------
+#-----------------------------------------------------------------------------------------------
+# Used when a profile does not match
+
+# Network speed limit (KB/s)
 BW_LIMIT=12500
-
+# Retry logic
-# Amount of times to retry
 RETRY_COUNT=3
-
+# Sleep between retries (seconds)
-# Time in seconds waiting to retry
 SLEEP=300
-
 # Max number of concurrently running rsync processes
 MAX_RSYNC_PROCS=1
-
+# Container start/stop/restart before and after rsync
-# Containers to stop/start on backup server
 CRITICAL_CONTAINER_NAMES=()
-
+# Containers that need delayed before starting
-# Containers from CRITICAL_CONTAINER_NAMES that should have delayed startup
 DELAYED_CONTAINERS=()
-
+# Delay in seconds between starting containers, useful for things like Authelia
-# Time in seconds to delay the listed containers
 CONTAINER_DELAY=5
+
+#-----------------------------------------------------------------------------------------------
+#----------------------------- Profile System ---------------------------------------------------
+#-----------------------------------------------------------------------------------------------
+# Profiles are inferred from the directory basename (lowercased)
+
+#-----------------------------------------------------------------------------------------------
+#----------------------------- Profile Performance Tuning ---------------------------------------
+#-----------------------------------------------------------------------------------------------
+
+# Max rsync processes per profile
+declare -A PROFILE_MAX_PROCS
+PROFILE_MAX_PROCS["arrs_stack"]=2
+PROFILE_MAX_PROCS["movies"]=1
+PROFILE_MAX_PROCS["shows"]=1
+# Bandwidth limits per profile (KB/s, 0 = unlimited)
+declare -A PROFILE_BW_LIMIT
+PROFILE_BW_LIMIT["arrs_stack"]=5000
+PROFILE_BW_LIMIT["movies"]=0
+PROFILE_BW_LIMIT["shows"]=8000
+# Retry logic per profile
+declare -A PROFILE_RETRY_COUNT
+PROFILE_RETRY_COUNT["arrs_stack"]=3
+PROFILE_RETRY_COUNT["movies"]=3
+PROFILE_RETRY_COUNT["shows"]=3
+# Sleep between retries per profile (seconds)
+declare -A PROFILE_SLEEP
+PROFILE_SLEEP["arrs_stack"]=300
+PROFILE_SLEEP["movies"]=300
+PROFILE_SLEEP["shows"]=300
+# Containers to stop/start per profile (space-separated strings)
+declare -A PROFILE_CRITICAL_CONTAINER_NAMES
+PROFILE_CRITICAL_CONTAINER_NAMES["arrs_stack"]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
+PROFILE_CRITICAL_CONTAINER_NAMES["movies"]=""
+PROFILE_CRITICAL_CONTAINER_NAMES["shows"]=""
+# Delayed containers per profile (space-separated strings)
+declare -A PROFILE_DELAYED_CONTAINERS
+PROFILE_DELAYED_CONTAINERS["arrs_stack"]=""
+PROFILE_DELAYED_CONTAINERS["movies"]=""
+PROFILE_DELAYED_CONTAINERS["shows"]=""
+# Delay in seconds before starting delayed containers per profile
+declare -A PROFILE_CONTAINER_DELAY
+PROFILE_CONTAINER_DELAY["arrs_stack"]=5
+PROFILE_CONTAINER_DELAY["movies"]=0
+PROFILE_CONTAINER_DELAY["shows"]=0
+
+#-----------------------------------------------------------------------------------------------
+#----------------------------- End of Master Config ---------------------------------------------
+#-----------------------------------------------------------------------------------------------

Rsync/Rsync-Master_Variables.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-##################################################################################################
-############################## Master Variables for Rsync scripts ################################
-##################################################################################################
-
-# Backup server Tailscale hostname, Use the machine name. 
-BACKUP_SERVER_NAME="unRAID-Jayred365"
-
-# Resolve Tailscale IP dynamically, This will try to get the Tailscale IPv4 address for that host
-BACKUP_SERVER=$(tailscale ip -4 "$BACKUP_SERVER_NAME" 2>/dev/null || echo "127.0.0.1")
-
-# SSH key for unRAID-Gmer4Lfe
-SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key"
-
-# Network speed limit 100Mbps=12500 KB/s, 200Mbps=25600 KB/s
-BW_LIMIT=12500
-
-# Amount of times to retry
-RETRY_COUNT=3
-
-# Time in seconds waiting to retry
-SLEEP=300
-
-# Max number of concurrently running rsync processes
-MAX_RSYNC_PROCS=1
-
-# Containers to stop/start on backup server
-CRITICAL_CONTAINER_NAMES=()
-
-# Containers from CRITICAL_CONTAINER_NAMES that should have delayed startup
-DELAYED_CONTAINERS=()
-
-# Time in seconds to delay the listed containers
-CONTAINER_DELAY=5

Rsync/rsync.sh

@@ -1,8 +1,28 @@
 #!/bin/bash
 set -e
+#-----------------------------------------------------------------------------------------------
+#----------------- User Variables, Please adjust in Master.cong as needed ----------------------
+#-----------------------------------------------------------------------------------------------
+#
+# User vaiables can be adjusted in Master.conf
+#   scripts now just contain runtime codes
+#
+# This allows for profiles and multiple things to use the core script
+#  use arguments in unRAID User Plugin for directory
+#    Example = /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh
+#                         /mnt/user/appdata-Failover/Arrs_Stack/
+#
+# /mnt/user/appdata-Failover/Arrs_Stack/ is the argument
+#   If the last name /Arrs-Stack/ matches a profile in Master.conf then profiles are used
+#     If no matches then Global Configureation is used in Master.conf
+#
+#-----------------------------------------------------------------------------------------------
+#---------------End Of User Variables, Please adjust in Master.cong as needed ------------------
+#-----------------------------------------------------------------------------------------------
 # Detect script directory
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-# Load Master.conf from Git repo
+
+# Load Master.conf
 MASTER_CONF="$SCRIPT_DIR/../Master.conf"
 if [ -f "$MASTER_CONF" ]; then
     source "$MASTER_CONF"
@@ -11,24 +31,27 @@ else
     echo "Error: Master.conf not found at $MASTER_CONF"
     exit 1
 fi
-# Input Arguments & Directory
+
+# Input Arguments
 DIRECTORY="$1"   # Directory to sync
 if [[ -z "$DIRECTORY" ]]; then
     echo "Usage: $0 <directory-to-sync> [--dry-run] [VAR=value ...]"
     exit 1
 fi
+
+# Optional dry-run
 DRY_RUN=false
-# Handle optional dry-run
+shift 1
-if [[ "$2" == "--dry-run" ]]; then
+if [[ "$1" == "--dry-run" ]]; then
     DRY_RUN=true
     shift 1
 fi
-# Handle optional variable overrides: VAR=value format
+
-for ARG in "${@:2}"; do
+# Optional variable overrides: VAR=value
+for ARG in "$@"; do
     if [[ "$ARG" == *=* ]]; then
         VAR_NAME="${ARG%%=*}"
         VAR_VALUE="${ARG#*=}"
-        # Only override if variable already exists
         if declare -p "$VAR_NAME" &>/dev/null; then
             eval "$VAR_NAME=\"$VAR_VALUE\""
             echo "Overriding $VAR_NAME -> $VAR_VALUE"
@@ -38,19 +61,20 @@ for ARG in "${@:2}"; do
     fi
 done
 
-#-----------------------------------------------------------------------------------------------
+# Determine profile
-#----------------------------- User Variables, Please adjust as needed -------------------------
+PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]')
-#-----------------------------------------------------------------------------------------------
+echo "Detected profile: $PROFILE_NAME"
-#---------------------------- These variables Over Ride The Master.conf ------------------------
+
-#-----------------------------------------------------------------------------------------------
+# Apply profile defaults or fallback to master.conf
-CRITICAL_CONTAINER_NAMES=("Sonarr" "Lidarr" "Readarr" "Radarr" "Prowlarr" "Bazarr" "Pinchflat")
+MAX_RSYNC_PROCS=${PROFILE_MAX_PROCS[$PROFILE_NAME]:-$MAX_RSYNC_PROCS}
-DELAYED_CONTAINERS=("")
+BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
-EXCLUDE_DIRS=("")
+CRITICAL_CONTAINER_NAMES=(${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]})
-#-----------------------------------------------------------------------------------------------
+DELAYED_CONTAINERS=(${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]})
-#---------------------------- These variables Over Ride The Master.conf ------------------------
+CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY}
-#-----------------------------------------------------------------------------------------------
+RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
-#------------------------End Of User Variables, Please adjust Above as needed ------------------
+SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
-#-----------------------------------------------------------------------------------------------
+
+echo "Settings: MAX_RSYNC_PROCS=$MAX_RSYNC_PROCS, BW_LIMIT=$BW_LIMIT, CONTAINERS=${CRITICAL_CONTAINER_NAMES[*]}"
 
 # Dependency check
 for cmd in rsync ssh ping timeout docker tailscale; do
@@ -62,11 +86,11 @@ done
 
 trap 'echo "Script interrupted. Exiting."; exit 130' INT TERM
 
-#------------------------- Helper Functions -------------------------
+# Helper functions
 wait_for_rsync() {
     for attempt in $(seq 1 "$RETRY_COUNT"); do
         RSYNC_COUNT=$(pgrep -fc "rsync.*root@${REMOTE_SERVER}" || true)
-        if [ "$RSYNC_COUNT" -gt "$MAX_RSYNC_PROCS" ]; then
+        if [ "$RSYNC_COUNT" -ge "$MAX_RSYNC_PROCS" ]; then
             echo "[$LOCAL_SERVER_NAME] Waiting... ($RSYNC_COUNT active rsync processes)"
             sleep "$SLEEP"
         else
@@ -78,7 +102,7 @@ wait_for_rsync() {
 }
 
 check_server_online() {
-    timeout 5 ping -c 1 "$REMOTE_SERVER" &> /dev/null
+    timeout 5 ping -c 1 "$REMOTE_SERVER" &>/dev/null
 }
 
 declare -a RUNNING_CONTAINERS=()
@@ -108,7 +132,8 @@ start_containers() {
     done
 }
 
-#------------------------- Main -------------------------
+
+# Main sync logic
 main() {
     echo "[$LOCAL_SERVER_NAME] Syncing $DIRECTORY → $REMOTE_SERVER:$DIRECTORY"
 
@@ -135,10 +160,6 @@ main() {
         --no-whole-file
     )
 
-    for ex in "${EXCLUDE_DIRS[@]}"; do
-        RSYNC_OPTS+=(--exclude="$ex")
-    done
-
     [ "$DRY_RUN" = true ] && RSYNC_OPTS+=("--dry-run")
 
     for attempt in $(seq 1 "$RETRY_COUNT"); do
@@ -165,5 +186,5 @@ main() {
     echo "[$LOCAL_SERVER_NAME] Sync complete."
 }
 
-# Run
+# Execute
 main

Rsync/rsync_appdata-failover_arrs_data.sh

@@ -1,153 +0,0 @@
-#!/bin/bash
-set -e
-# Detect script directory
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-# Load Master.conf from Git repo
-MASTER_CONF="$SCRIPT_DIR/../Master.conf"
-if [ -f "$MASTER_CONF" ]; then
-    source "$MASTER_CONF"
-    echo "Loaded Master.conf from $MASTER_CONF"
-else
-    echo "Error: Master.conf not found at $MASTER_CONF"
-    exit 1
-fi
-# Directory
-DIRECTORY="$1"   # The directory to sync, passed as argument
-
-
-if [[ -z "$DIRECTORY" ]]; then
-    echo "Usage: $0 <directory-to-sync>"
-    exit 1
-fi
-
-echo "[$LOCAL_SERVER_NAME] Syncing $DIRECTORY → $REMOTE_SERVER:$DIRECTORY"
-
-#-----------------------------------------------------------------------------------------------
-#----------------------------- User Variables, Please adjust as needed -------------------------
-#-----------------------------------------------------------------------------------------------
-DRY_RUN=false
-CRITICAL_CONTAINER_NAMES=("Sonarr" "Lidarr" "Readarr" "Radarr" "Prowlarr" "Bazarr" "Pinchflat")
-DELAYED_CONTAINERS=("")
-EXCLUDE_DIRS=("")
-MAX_RSYNC_PROCS=2
-#-----------------------------------------------------------------------------------------------
-#------------------------End Of User Variables, Please adjust Above as needed ------------------
-#-----------------------------------------------------------------------------------------------
-
-# Dependency check
-for cmd in rsync ssh ping timeout docker tailscale; do
-    command -v "$cmd" >/dev/null 2>&1 || {
-        echo "Error: Required command '$cmd' is not installed."
-        exit 1
-    }
-done
-
-trap 'echo "Script interrupted. Exiting."; exit 130' INT TERM
-
-
-# Helper functions
-wait_for_rsync() {
-    for attempt in $(seq 1 "$RETRY_COUNT"); do
-        RSYNC_COUNT=$(pgrep -fc "rsync.*root@${REMOTE_SERVER}" || true)
-        if [ "$RSYNC_COUNT" -gt "$MAX_RSYNC_PROCS" ]; then
-            echo "[$LOCAL_SERVER_NAME] Waiting... ($RSYNC_COUNT active rsync processes)"
-            sleep "$SLEEP"
-        else
-            return 0
-        fi
-    done
-    echo "Too many rsync processes, exiting."
-    exit 1
-}
-
-check_server_online() {
-    timeout 5 ping -c 1 "$REMOTE_SERVER" &> /dev/null
-}
-
-declare -a RUNNING_CONTAINERS=()
-
-stop_containers() {
-    echo "[$LOCAL_SERVER_NAME] Checking containers on ${REMOTE_SERVER_NAME}..."
-    RUNNING_CONTAINERS=()
-    for container in "${CRITICAL_CONTAINER_NAMES[@]}"; do
-        STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
-            "docker inspect -f '{{.State.Running}}' $container 2>/dev/null" || echo "false")
-        if [ "$STATUS" == "true" ]; then
-            echo "Stopping $container"
-            RUNNING_CONTAINERS+=("$container")
-            ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $container"
-        fi
-    done
-}
-
-start_containers() {
-    echo "[$LOCAL_SERVER_NAME] Restarting containers on ${REMOTE_SERVER_NAME}..."
-    for container in "${RUNNING_CONTAINERS[@]}"; do
-        if [[ " ${DELAYED_CONTAINERS[*]} " == *" $container "* ]]; then
-            echo "Delaying start of $container by ${CONTAINER_DELAY}s..."
-            sleep "$CONTAINER_DELAY"
-        fi
-        ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $container"
-    done
-}
-
-
-# Main sync logic
-main() {
-    echo "[$LOCAL_SERVER_NAME] Syncing $DIRECTORY → $REMOTE_SERVER:$DIRECTORY"
-
-    for i in $(seq 1 "$RETRY_COUNT"); do
-        if check_server_online; then
-            break
-        fi
-        echo "Remote $REMOTE_SERVER_NAME down, retrying in $SLEEP seconds..."
-        sleep "$SLEEP"
-    done
-
-    check_server_online || { echo "Remote $REMOTE_SERVER_NAME unreachable. Exiting."; exit 1; }
-
-    wait_for_rsync
-    stop_containers
-
-    RSYNC_OPTS=(
-        -av
-        --info=progress2
-        --human-readable
-        --bwlimit="$BW_LIMIT"
-        --delete
-        --inplace
-        --no-whole-file
-    )
-
-    for ex in "${EXCLUDE_DIRS[@]}"; do
-        RSYNC_OPTS+=(--exclude="$ex")
-    done
-
-    [ "$DRY_RUN" = true ] && RSYNC_OPTS+=("--dry-run")
-
-    for attempt in $(seq 1 "$RETRY_COUNT"); do
-        echo "Starting rsync attempt $attempt/$RETRY_COUNT..."
-        if rsync "${RSYNC_OPTS[@]}" \
-            -e "ssh -i \"$SSH_KEY\" -T -o Compression=no -o IPQoS=throughput" \
-            "$DIRECTORY" "root@${REMOTE_SERVER}:$DIRECTORY"; then
-            echo "Rsync completed successfully."
-            break
-        else
-            echo "Rsync attempt $attempt failed."
-            if [ "$attempt" -lt "$RETRY_COUNT" ]; then
-                echo "Retrying in $SLEEP seconds..."
-                sleep "$SLEEP"
-            else
-                echo "All $RETRY_COUNT rsync attempts failed."
-                start_containers
-                exit 1
-            fi
-        fi
-    done
-
-    start_containers
-    echo "[$LOCAL_SERVER_NAME] Sync complete."
-}
-
-# Run
-main