From 546686ac8c9a9cffae63f119e69799be7dc25b83 Mon Sep 17 00:00:00 2001 From: FailedProxy Date: Sat, 21 Mar 2026 20:10:18 +0000 Subject: [PATCH] Structure Changes --- Master.conf | 90 ++++++++++--- Rsync/Rsync-Master_Variables.sh | 34 ----- Rsync/rsync.sh | 81 +++++++----- Rsync/rsync_appdata-failover_arrs_data.sh | 153 ---------------------- 4 files changed, 123 insertions(+), 235 deletions(-) delete mode 100644 Rsync/Rsync-Master_Variables.sh delete mode 100644 Rsync/rsync_appdata-failover_arrs_data.sh diff --git a/Master.conf b/Master.conf index 45a9214..967566d 100644 --- a/Master.conf +++ b/Master.conf @@ -3,13 +3,16 @@ ############################## Master Variables for unRAID scripts ################################ ################################################################################################## +#----------------------------------------------------------------------------------------------- +#----------------------------- Host Detection --------------------------------------------------- +#----------------------------------------------------------------------------------------------- + # Define the hostnames of both servers HOST1="unRAID-Gmer4Lfe" HOST2="unRAID-Jayred365" # Detect local hostname LOCAL_HOSTNAME="$(hostname)" - if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then LOCAL_SERVER_NAME="$HOST1" REMOTE_SERVER_NAME="$HOST2" @@ -20,50 +23,101 @@ else echo "Error: Local hostname ($LOCAL_HOSTNAME) not recognized." exit 1 fi - echo "Local server: $LOCAL_SERVER_NAME" echo "Remote server: $REMOTE_SERVER_NAME" -# Resolve Tailscale IP dynamically, This will try to get the Tailscale IPv4 address for that host +#----------------------------------------------------------------------------------------------- +#----------------------------- Remote Server Resolution ----------------------------------------- +#----------------------------------------------------------------------------------------------- +# Resolve Tailscale IP dynamically REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) if [[ -z "$REMOTE_SERVER" ]]; then echo "Error: Could not resolve Tailscale IP for $REMOTE_SERVER_NAME" exit 1 fi -# SSH key mapping per hostpair +#----------------------------------------------------------------------------------------------- +#----------------------------- SSH Key Mapping -------------------------------------------------- +#----------------------------------------------------------------------------------------------- + declare -A SSH_KEYS SSH_KEYS["$HOST1|$HOST2"]="/root/.ssh/Gmer4Lfe-rsync-key" SSH_KEYS["$HOST2|$HOST1"]="/root/.ssh/Jayred365-rsync-key" KEY_ID="$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME" - if [[ -n "${SSH_KEYS[$KEY_ID]}" ]]; then SSH_KEY="${SSH_KEYS[$KEY_ID]}" else echo "Error: No SSH key defined for $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" exit 1 fi - echo "Using SSH key: $SSH_KEY" -# Network speed limit 100Mbps=12500 KB/s, 200Mbps=25600 KB/s +#----------------------------------------------------------------------------------------------- +#----------------------------- Global Defaults -------------------------------------------------- +#----------------------------------------------------------------------------------------------- +# Used when a profile does not match + +# Network speed limit (KB/s) BW_LIMIT=12500 - -# Amount of times to retry +# Retry logic RETRY_COUNT=3 - -# Time in seconds waiting to retry +# Sleep between retries (seconds) SLEEP=300 - # Max number of concurrently running rsync processes MAX_RSYNC_PROCS=1 - -# Containers to stop/start on backup server +# Container start/stop/restart before and after rsync CRITICAL_CONTAINER_NAMES=() - -# Containers from CRITICAL_CONTAINER_NAMES that should have delayed startup +# Containers that need delayed before starting DELAYED_CONTAINERS=() - -# Time in seconds to delay the listed containers +# Delay in seconds between starting containers, useful for things like Authelia CONTAINER_DELAY=5 + +#----------------------------------------------------------------------------------------------- +#----------------------------- Profile System --------------------------------------------------- +#----------------------------------------------------------------------------------------------- +# Profiles are inferred from the directory basename (lowercased) + +#----------------------------------------------------------------------------------------------- +#----------------------------- Profile Performance Tuning --------------------------------------- +#----------------------------------------------------------------------------------------------- + +# Max rsync processes per profile +declare -A PROFILE_MAX_PROCS +PROFILE_MAX_PROCS["arrs_stack"]=2 +PROFILE_MAX_PROCS["movies"]=1 +PROFILE_MAX_PROCS["shows"]=1 +# Bandwidth limits per profile (KB/s, 0 = unlimited) +declare -A PROFILE_BW_LIMIT +PROFILE_BW_LIMIT["arrs_stack"]=5000 +PROFILE_BW_LIMIT["movies"]=0 +PROFILE_BW_LIMIT["shows"]=8000 +# Retry logic per profile +declare -A PROFILE_RETRY_COUNT +PROFILE_RETRY_COUNT["arrs_stack"]=3 +PROFILE_RETRY_COUNT["movies"]=3 +PROFILE_RETRY_COUNT["shows"]=3 +# Sleep between retries per profile (seconds) +declare -A PROFILE_SLEEP +PROFILE_SLEEP["arrs_stack"]=300 +PROFILE_SLEEP["movies"]=300 +PROFILE_SLEEP["shows"]=300 +# Containers to stop/start per profile (space-separated strings) +declare -A PROFILE_CRITICAL_CONTAINER_NAMES +PROFILE_CRITICAL_CONTAINER_NAMES["arrs_stack"]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" +PROFILE_CRITICAL_CONTAINER_NAMES["movies"]="" +PROFILE_CRITICAL_CONTAINER_NAMES["shows"]="" +# Delayed containers per profile (space-separated strings) +declare -A PROFILE_DELAYED_CONTAINERS +PROFILE_DELAYED_CONTAINERS["arrs_stack"]="" +PROFILE_DELAYED_CONTAINERS["movies"]="" +PROFILE_DELAYED_CONTAINERS["shows"]="" +# Delay in seconds before starting delayed containers per profile +declare -A PROFILE_CONTAINER_DELAY +PROFILE_CONTAINER_DELAY["arrs_stack"]=5 +PROFILE_CONTAINER_DELAY["movies"]=0 +PROFILE_CONTAINER_DELAY["shows"]=0 + +#----------------------------------------------------------------------------------------------- +#----------------------------- End of Master Config --------------------------------------------- +#----------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/Rsync/Rsync-Master_Variables.sh b/Rsync/Rsync-Master_Variables.sh deleted file mode 100644 index 1aaf2c3..0000000 --- a/Rsync/Rsync-Master_Variables.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -################################################################################################## -############################## Master Variables for Rsync scripts ################################ -################################################################################################## - -# Backup server Tailscale hostname, Use the machine name. -BACKUP_SERVER_NAME="unRAID-Jayred365" - -# Resolve Tailscale IP dynamically, This will try to get the Tailscale IPv4 address for that host -BACKUP_SERVER=$(tailscale ip -4 "$BACKUP_SERVER_NAME" 2>/dev/null || echo "127.0.0.1") - -# SSH key for unRAID-Gmer4Lfe -SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" - -# Network speed limit 100Mbps=12500 KB/s, 200Mbps=25600 KB/s -BW_LIMIT=12500 - -# Amount of times to retry -RETRY_COUNT=3 - -# Time in seconds waiting to retry -SLEEP=300 - -# Max number of concurrently running rsync processes -MAX_RSYNC_PROCS=1 - -# Containers to stop/start on backup server -CRITICAL_CONTAINER_NAMES=() - -# Containers from CRITICAL_CONTAINER_NAMES that should have delayed startup -DELAYED_CONTAINERS=() - -# Time in seconds to delay the listed containers -CONTAINER_DELAY=5 diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index 95f6cce..4735f55 100644 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -1,8 +1,28 @@ #!/bin/bash set -e +#----------------------------------------------------------------------------------------------- +#----------------- User Variables, Please adjust in Master.cong as needed ---------------------- +#----------------------------------------------------------------------------------------------- +# +# User vaiables can be adjusted in Master.conf +# scripts now just contain runtime codes +# +# This allows for profiles and multiple things to use the core script +# use arguments in unRAID User Plugin for directory +# Example = /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh +# /mnt/user/appdata-Failover/Arrs_Stack/ +# +# /mnt/user/appdata-Failover/Arrs_Stack/ is the argument +# If the last name /Arrs-Stack/ matches a profile in Master.conf then profiles are used +# If no matches then Global Configureation is used in Master.conf +# +#----------------------------------------------------------------------------------------------- +#---------------End Of User Variables, Please adjust in Master.cong as needed ------------------ +#----------------------------------------------------------------------------------------------- # Detect script directory SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# Load Master.conf from Git repo + +# Load Master.conf MASTER_CONF="$SCRIPT_DIR/../Master.conf" if [ -f "$MASTER_CONF" ]; then source "$MASTER_CONF" @@ -11,24 +31,27 @@ else echo "Error: Master.conf not found at $MASTER_CONF" exit 1 fi -# Input Arguments & Directory -DIRECTORY="$1" # Directory to sync + +# Input Arguments +DIRECTORY="$1" # Directory to sync if [[ -z "$DIRECTORY" ]]; then echo "Usage: $0 [--dry-run] [VAR=value ...]" exit 1 fi + +# Optional dry-run DRY_RUN=false -# Handle optional dry-run -if [[ "$2" == "--dry-run" ]]; then +shift 1 +if [[ "$1" == "--dry-run" ]]; then DRY_RUN=true shift 1 fi -# Handle optional variable overrides: VAR=value format -for ARG in "${@:2}"; do + +# Optional variable overrides: VAR=value +for ARG in "$@"; do if [[ "$ARG" == *=* ]]; then VAR_NAME="${ARG%%=*}" VAR_VALUE="${ARG#*=}" - # Only override if variable already exists if declare -p "$VAR_NAME" &>/dev/null; then eval "$VAR_NAME=\"$VAR_VALUE\"" echo "Overriding $VAR_NAME -> $VAR_VALUE" @@ -38,19 +61,20 @@ for ARG in "${@:2}"; do fi done -#----------------------------------------------------------------------------------------------- -#----------------------------- User Variables, Please adjust as needed ------------------------- -#----------------------------------------------------------------------------------------------- -#---------------------------- These variables Over Ride The Master.conf ------------------------ -#----------------------------------------------------------------------------------------------- -CRITICAL_CONTAINER_NAMES=("Sonarr" "Lidarr" "Readarr" "Radarr" "Prowlarr" "Bazarr" "Pinchflat") -DELAYED_CONTAINERS=("") -EXCLUDE_DIRS=("") -#----------------------------------------------------------------------------------------------- -#---------------------------- These variables Over Ride The Master.conf ------------------------ -#----------------------------------------------------------------------------------------------- -#------------------------End Of User Variables, Please adjust Above as needed ------------------ -#----------------------------------------------------------------------------------------------- +# Determine profile +PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]') +echo "Detected profile: $PROFILE_NAME" + +# Apply profile defaults or fallback to master.conf +MAX_RSYNC_PROCS=${PROFILE_MAX_PROCS[$PROFILE_NAME]:-$MAX_RSYNC_PROCS} +BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT} +CRITICAL_CONTAINER_NAMES=(${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]}) +DELAYED_CONTAINERS=(${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]}) +CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY} +RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT} +SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP} + +echo "Settings: MAX_RSYNC_PROCS=$MAX_RSYNC_PROCS, BW_LIMIT=$BW_LIMIT, CONTAINERS=${CRITICAL_CONTAINER_NAMES[*]}" # Dependency check for cmd in rsync ssh ping timeout docker tailscale; do @@ -62,11 +86,11 @@ done trap 'echo "Script interrupted. Exiting."; exit 130' INT TERM -#------------------------- Helper Functions ------------------------- +# Helper functions wait_for_rsync() { for attempt in $(seq 1 "$RETRY_COUNT"); do RSYNC_COUNT=$(pgrep -fc "rsync.*root@${REMOTE_SERVER}" || true) - if [ "$RSYNC_COUNT" -gt "$MAX_RSYNC_PROCS" ]; then + if [ "$RSYNC_COUNT" -ge "$MAX_RSYNC_PROCS" ]; then echo "[$LOCAL_SERVER_NAME] Waiting... ($RSYNC_COUNT active rsync processes)" sleep "$SLEEP" else @@ -78,7 +102,7 @@ wait_for_rsync() { } check_server_online() { - timeout 5 ping -c 1 "$REMOTE_SERVER" &> /dev/null + timeout 5 ping -c 1 "$REMOTE_SERVER" &>/dev/null } declare -a RUNNING_CONTAINERS=() @@ -108,7 +132,8 @@ start_containers() { done } -#------------------------- Main ------------------------- + +# Main sync logic main() { echo "[$LOCAL_SERVER_NAME] Syncing $DIRECTORY → $REMOTE_SERVER:$DIRECTORY" @@ -135,10 +160,6 @@ main() { --no-whole-file ) - for ex in "${EXCLUDE_DIRS[@]}"; do - RSYNC_OPTS+=(--exclude="$ex") - done - [ "$DRY_RUN" = true ] && RSYNC_OPTS+=("--dry-run") for attempt in $(seq 1 "$RETRY_COUNT"); do @@ -165,5 +186,5 @@ main() { echo "[$LOCAL_SERVER_NAME] Sync complete." } -# Run +# Execute main \ No newline at end of file diff --git a/Rsync/rsync_appdata-failover_arrs_data.sh b/Rsync/rsync_appdata-failover_arrs_data.sh deleted file mode 100644 index 5018eb0..0000000 --- a/Rsync/rsync_appdata-failover_arrs_data.sh +++ /dev/null @@ -1,153 +0,0 @@ -#!/bin/bash -set -e -# Detect script directory -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# Load Master.conf from Git repo -MASTER_CONF="$SCRIPT_DIR/../Master.conf" -if [ -f "$MASTER_CONF" ]; then - source "$MASTER_CONF" - echo "Loaded Master.conf from $MASTER_CONF" -else - echo "Error: Master.conf not found at $MASTER_CONF" - exit 1 -fi -# Directory -DIRECTORY="$1" # The directory to sync, passed as argument - - -if [[ -z "$DIRECTORY" ]]; then - echo "Usage: $0 " - exit 1 -fi - -echo "[$LOCAL_SERVER_NAME] Syncing $DIRECTORY → $REMOTE_SERVER:$DIRECTORY" - -#----------------------------------------------------------------------------------------------- -#----------------------------- User Variables, Please adjust as needed ------------------------- -#----------------------------------------------------------------------------------------------- -DRY_RUN=false -CRITICAL_CONTAINER_NAMES=("Sonarr" "Lidarr" "Readarr" "Radarr" "Prowlarr" "Bazarr" "Pinchflat") -DELAYED_CONTAINERS=("") -EXCLUDE_DIRS=("") -MAX_RSYNC_PROCS=2 -#----------------------------------------------------------------------------------------------- -#------------------------End Of User Variables, Please adjust Above as needed ------------------ -#----------------------------------------------------------------------------------------------- - -# Dependency check -for cmd in rsync ssh ping timeout docker tailscale; do - command -v "$cmd" >/dev/null 2>&1 || { - echo "Error: Required command '$cmd' is not installed." - exit 1 - } -done - -trap 'echo "Script interrupted. Exiting."; exit 130' INT TERM - - -# Helper functions -wait_for_rsync() { - for attempt in $(seq 1 "$RETRY_COUNT"); do - RSYNC_COUNT=$(pgrep -fc "rsync.*root@${REMOTE_SERVER}" || true) - if [ "$RSYNC_COUNT" -gt "$MAX_RSYNC_PROCS" ]; then - echo "[$LOCAL_SERVER_NAME] Waiting... ($RSYNC_COUNT active rsync processes)" - sleep "$SLEEP" - else - return 0 - fi - done - echo "Too many rsync processes, exiting." - exit 1 -} - -check_server_online() { - timeout 5 ping -c 1 "$REMOTE_SERVER" &> /dev/null -} - -declare -a RUNNING_CONTAINERS=() - -stop_containers() { - echo "[$LOCAL_SERVER_NAME] Checking containers on ${REMOTE_SERVER_NAME}..." - RUNNING_CONTAINERS=() - for container in "${CRITICAL_CONTAINER_NAMES[@]}"; do - STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ - "docker inspect -f '{{.State.Running}}' $container 2>/dev/null" || echo "false") - if [ "$STATUS" == "true" ]; then - echo "Stopping $container" - RUNNING_CONTAINERS+=("$container") - ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $container" - fi - done -} - -start_containers() { - echo "[$LOCAL_SERVER_NAME] Restarting containers on ${REMOTE_SERVER_NAME}..." - for container in "${RUNNING_CONTAINERS[@]}"; do - if [[ " ${DELAYED_CONTAINERS[*]} " == *" $container "* ]]; then - echo "Delaying start of $container by ${CONTAINER_DELAY}s..." - sleep "$CONTAINER_DELAY" - fi - ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $container" - done -} - - -# Main sync logic -main() { - echo "[$LOCAL_SERVER_NAME] Syncing $DIRECTORY → $REMOTE_SERVER:$DIRECTORY" - - for i in $(seq 1 "$RETRY_COUNT"); do - if check_server_online; then - break - fi - echo "Remote $REMOTE_SERVER_NAME down, retrying in $SLEEP seconds..." - sleep "$SLEEP" - done - - check_server_online || { echo "Remote $REMOTE_SERVER_NAME unreachable. Exiting."; exit 1; } - - wait_for_rsync - stop_containers - - RSYNC_OPTS=( - -av - --info=progress2 - --human-readable - --bwlimit="$BW_LIMIT" - --delete - --inplace - --no-whole-file - ) - - for ex in "${EXCLUDE_DIRS[@]}"; do - RSYNC_OPTS+=(--exclude="$ex") - done - - [ "$DRY_RUN" = true ] && RSYNC_OPTS+=("--dry-run") - - for attempt in $(seq 1 "$RETRY_COUNT"); do - echo "Starting rsync attempt $attempt/$RETRY_COUNT..." - if rsync "${RSYNC_OPTS[@]}" \ - -e "ssh -i \"$SSH_KEY\" -T -o Compression=no -o IPQoS=throughput" \ - "$DIRECTORY" "root@${REMOTE_SERVER}:$DIRECTORY"; then - echo "Rsync completed successfully." - break - else - echo "Rsync attempt $attempt failed." - if [ "$attempt" -lt "$RETRY_COUNT" ]; then - echo "Retrying in $SLEEP seconds..." - sleep "$SLEEP" - else - echo "All $RETRY_COUNT rsync attempts failed." - start_containers - exit 1 - fi - fi - done - - start_containers - echo "[$LOCAL_SERVER_NAME] Sync complete." -} - -# Run -main \ No newline at end of file