diff --git a/Master.conf b/Master.conf index 31c3ea4..18cc0a9 100644 --- a/Master.conf +++ b/Master.conf @@ -51,81 +51,64 @@ PHP_CONF="/etc/php-fpm.d/www.conf" # php_fpm_max_children.sh max children value PHP_MAX_CHILDREN=250 +# Profiles +declare -A PROFILE_MAX_PROCS=([arrs_stack]=2 [critical-data]=2 [gmer4lfe]=1 [important-data]=2 [emby]=1) +declare -A PROFILE_BW_LIMIT=([arrs_stack]=5000 [critical-data]=9500 [gmer4lfe]=8000 [important-data]=9500 [emby]=8000) +declare -A PROFILE_RETRY_COUNT=([arrs_stack]=3 [critical-data]=3 [gmer4lfe]=3 [important-data]=3 [emby]=3) +declare -A PROFILE_SLEEP=([arrs_stack]=300 [critical-data]=300 [gmer4lfe]=300 [important-data]=300 [emby]=300) +declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( + [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" + [critical-data]="Mariadb-Authelia Redis-Authelia Lldap-Gmer4Lfe NginxProxyManager Authelia" + [gmer4lfe]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe" + [important-data]="Postgres-NextCloud NextCloud" + [emby]="" +) +declare -A PROFILE_DELAYED_CONTAINERS=([arrs_stack]="" [critical-data]="Authelia" [gmer4lfe]="" [important-data]="NextCloud" [emby]="") +declare -A PROFILE_CONTAINER_DELAY=([arrs_stack]=5 [critical-data]=10 [gmer4lfe]=5 [important-data]=10 [emby]=5) +declare -A PROFILE_EXCLUDE_DIRS=([arrs_stack]="logs *.tmp" [critical-data]="logs *.tmp" [gmer4lfe]="logs *.tmp" [important-data]="logs *.tmp" [emby]="logs *.tmp") -#----------------------- Profile System ------------------------ +# ----------------------------- Common Helpers ----------------------------- +parse_args() { + DRY_RUN=${DRY_RUN:-false} + for ARG in "$@"; do + if [[ "$ARG" == *=* ]]; then + local VAR_NAME="${ARG%%=*}" + local VAR_VALUE="${ARG#*=}" + if declare -p "$VAR_NAME" &>/dev/null; then + printf -v "$VAR_NAME" '%s' "$VAR_VALUE" + echo "Overriding $VAR_NAME -> $VAR_VALUE" + else + echo "Warning: Unknown variable $VAR_NAME, ignoring." + fi + else + case "$ARG" in + --dry-run|-n) DRY_RUN=true ;; + --help|-h) echo "Usage: script [--dry-run|-n] [VAR=value ...]"; exit 0 ;; + *) echo "Warning: Unknown argument $ARG" ;; + esac + fi + done +} -# Profiles are mainly used for rsync operations, but can be used for any variable that needs -# to be different based on the source or destination. -# -# Profiles are inferred from the directory basename (lowercased) -# Max rsync processes per profile -declare -A PROFILE_MAX_PROCS -PROFILE_MAX_PROCS["arrs_stack"]=2 -PROFILE_MAX_PROCS["critical-data"]=2 -PROFILE_MAX_PROCS["gmer4lfe"]=1 -PROFILE_MAX_PROCS["important-data"]=2 -PROFILE_MAX_PROCS["emby"]=1 -# Bandwidth limits per profile (KB/s, 0 = unlimited) -declare -A PROFILE_BW_LIMIT -PROFILE_BW_LIMIT["arrs_stack"]=5000 -PROFILE_BW_LIMIT["critical-data"]=9500 -PROFILE_BW_LIMIT["gmer4lfe"]=8000 -PROFILE_BW_LIMIT["important-data"]=9500 -PROFILE_BW_LIMIT["emby"]=8000 -# Retry logic per profile -declare -A PROFILE_RETRY_COUNT -PROFILE_RETRY_COUNT["arrs_stack"]=3 -PROFILE_RETRY_COUNT["critical-data"]=3 -PROFILE_RETRY_COUNT["gmer4lfe"]=3 -PROFILE_RETRY_COUNT["important-data"]=3 -PROFILE_RETRY_COUNT["emby"]=3 -# Sleep between retries per profile (seconds) -declare -A PROFILE_SLEEP -PROFILE_SLEEP["arrs_stack"]=300 -PROFILE_SLEEP["critical-data"]=300 -PROFILE_SLEEP["gmer4lfe"]=300 -PROFILE_SLEEP["important-data"]=300 -PROFILE_SLEEP["emby"]=300 -# Containers to stop/start per profile (space-separated strings) -declare -A PROFILE_CRITICAL_CONTAINER_NAMES -PROFILE_CRITICAL_CONTAINER_NAMES["arrs_stack"]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" -PROFILE_CRITICAL_CONTAINER_NAMES["critical-data"]="Mariadb-Authelia Redis-Authelia Lldap-Gmer4Lfe NginxProxyManager Authelia" -PROFILE_CRITICAL_CONTAINER_NAMES["gmer4lfe"]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe" -PROFILE_CRITICAL_CONTAINER_NAMES["important-data"]="Postgres-NextCloud NextCloud" -PROFILE_CRITICAL_CONTAINER_NAMES["emby"]="" -# Delayed containers per profile (space-separated strings) -declare -A PROFILE_DELAYED_CONTAINERS -PROFILE_DELAYED_CONTAINERS["arrs_stack"]="" -PROFILE_DELAYED_CONTAINERS["critical-data"]="Authelia" -PROFILE_DELAYED_CONTAINERS["gmer4lfe"]="" -PROFILE_DELAYED_CONTAINERS["important-data"]="NextCloud" -PROFILE_DELAYED_CONTAINERS["emby"]="" -# Delay in seconds before starting delayed containers per profile (seconds) -declare -A PROFILE_CONTAINER_DELAY -PROFILE_CONTAINER_DELAY["arrs_stack"]=5 -PROFILE_CONTAINER_DELAY["critical-data"]=10 -PROFILE_CONTAINER_DELAY["gmer4lfe"]=5 -PROFILE_CONTAINER_DELAY["important-data"]=10 -PROFILE_CONTAINER_DELAY["emby"]=5 -# Not include logs or other file types during transfer per profile -declare -A PROFILE_EXCLUDE_DIRS -PROFILE_EXCLUDE_DIRS["arrs_stack"]="logs *.tmp" -PROFILE_EXCLUDE_DIRS["critical-data"]="logs *.tmp" -PROFILE_EXCLUDE_DIRS["gmer4lfe"]="logs *.tmp" -PROFILE_EXCLUDE_DIRS["important-data"]="logs *.tmp" -PROFILE_EXCLUDE_DIRS["emby"]="logs *.tmp" -#----------------------------------------------------------------------------------------------- -#----------------------End Of User Variables, Please adjust Above as needed -------------------- -#----------------------------------------------------------------------------------------------- -# -# Next is commom runtime logic, tried to include logic in a common script but was having trouble -# So now its at the botton of the file, but please do not adjust unless you understand the code -# and what it does, as changes here will affect all scripts that source this master.conf file -# -#----------------------------------------------------------------------------------------------- -#--------------------Run Logic, Do not adjust unless you understand the code ------------------- -#----------------------------------------------------------------------------------------------- -# Detect local hostname +require_var() { + local var="$1" + if [[ -z "${!var:-}" ]]; then + echo "Error: Required variable $var is not set." + exit 1 + fi +} + +validate_int() { + local name="$1" + local value="$2" + if ! [[ "${value:-}" =~ ^[0-9]+$ ]]; then + echo "Error: $name must be a non-negative integer." + exit 1 + fi +} + +# ----------------------------- Host Detection & SSH ----------------------------- +# Detect local/remote host LOCAL_HOSTNAME="$(hostname)" if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then LOCAL_SERVER_NAME="$HOST1" @@ -139,29 +122,23 @@ else fi echo "Local server: $LOCAL_SERVER_NAME" echo "Remote server: $REMOTE_SERVER_NAME" -echo "DEBUG: LOCAL_HOSTNAME=$LOCAL_HOSTNAME, LOCAL_SERVER_NAME=$LOCAL_SERVER_NAME, REMOTE_SERVER_NAME=$REMOTE_SERVER_NAME" -echo "DEBUG: KEY_ID=$KEY_ID, SSH_KEY=${SSH_KEYS[$KEY_ID]}" - -# Remote Server Ip Resolution -# Resolve Tailscale IP dynamically -REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) -if [[ -z "$REMOTE_SERVER" ]]; then - echo "Error: Could not resolve Tailscale IP for $REMOTE_SERVER_NAME" - exit 1 -fi - # SSH Key Mapping declare -A SSH_KEYS SSH_KEYS["$HOST1|$HOST2"]="$HOST1_SSH_KEY" SSH_KEYS["$HOST2|$HOST1"]="$HOST2_SSH_KEY" - -# Determine which key to use KEY_ID="$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME" -if [[ -n "${SSH_KEYS[$KEY_ID]}" ]]; then - SSH_KEY="${SSH_KEYS[$KEY_ID]}" -else +SSH_KEY="${SSH_KEYS[$KEY_ID]}" +if [[ -z "$SSH_KEY" ]]; then echo "Error: No SSH key defined for $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" exit 1 fi -echo "Using SSH key: $SSH_KEY" \ No newline at end of file +echo "Using SSH key: $SSH_KEY" + +# Remote IP via Tailscale +REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) +if [[ -z "$REMOTE_SERVER" ]]; then + echo "Error: Could not resolve Tailscale IP for $REMOTE_SERVER_NAME" + exit 1 +fi +echo "Remote IP: $REMOTE_SERVER" \ No newline at end of file diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index 9f8c01d..03e4ea2 100644 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -1,29 +1,12 @@ #!/bin/bash set -e -#----------------------------------------------------------------------------------------------- -#----------------- User Variables, Please adjust in Master.conf as needed ---------------------- -#----------------------------------------------------------------------------------------------- -# This is the main Core Rsync script -# -# User variables can be adjusted in Master.conf -# Scripts now just contain runtime logic -# -# This allows for profiles and multiple things to use the core script -# Use arguments in unRAID User Plugin for directory -# Example: /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-test/Arrs/ -# -# /mnt/user/appdata-test/Arrs/ is the argument -# If the last name /Arrs/ matches a profile in Master.conf, then profile defaults are used -# Otherwise, global configuration in Master.conf is used -#----------------------------------------------------------------------------------------------- -#---------------End Of User Variables ---------------------------------------------------------- -#----------------------------------------------------------------------------------------------- -# Detect script directory +#--------------------------------------------- +# Detect script directory and load Master.conf +#--------------------------------------------- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -# Load Master.conf MASTER_CONF="$SCRIPT_DIR/../Master.conf" + if [ -f "$MASTER_CONF" ]; then source "$MASTER_CONF" echo "Loaded Master.conf from $MASTER_CONF" @@ -32,40 +15,23 @@ else exit 1 fi -# Input Arguments -DIRECTORY="$1" # Directory to sync +#--------------------------------------------- +# Parse arguments (using Master.conf helper) +#--------------------------------------------- +parse_args "$@" + +DIRECTORY="$1" if [[ -z "$DIRECTORY" ]]; then echo "Usage: $0 [--dry-run] [VAR=value ...]" exit 1 fi -# Optional dry-run -DRY_RUN=false -shift 1 -if [[ "$1" == "--dry-run" ]]; then - DRY_RUN=true - shift 1 -fi - -# Optional variable overrides: VAR=value -for ARG in "$@"; do - if [[ "$ARG" == *=* ]]; then - VAR_NAME="${ARG%%=*}" - VAR_VALUE="${ARG#*=}" - if declare -p "$VAR_NAME" &>/dev/null; then - eval "$VAR_NAME=\"$VAR_VALUE\"" - echo "Overriding $VAR_NAME -> $VAR_VALUE" - else - echo "Warning: Unknown variable $VAR_NAME, ignoring." - fi - fi -done - -# Determine profile +#--------------------------------------------- +# Determine profile & apply defaults +#--------------------------------------------- PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]') echo "Detected profile: $PROFILE_NAME" -# Apply profile defaults or fallback to master.conf MAX_RSYNC_PROCS=${PROFILE_MAX_PROCS[$PROFILE_NAME]:-$MAX_RSYNC_PROCS} BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT} CRITICAL_CONTAINER_NAMES=(${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]}) @@ -77,7 +43,9 @@ EXCLUDE_DIRS=(${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-${EXCLUDE_DIRS[@]}}) echo "Settings: MAX_RSYNC_PROCS=$MAX_RSYNC_PROCS, BW_LIMIT=$BW_LIMIT, CONTAINERS=${CRITICAL_CONTAINER_NAMES[*]}, EXCLUDES=${EXCLUDE_DIRS[*]}" +#--------------------------------------------- # Dependency check +#--------------------------------------------- for cmd in rsync ssh ping timeout docker tailscale; do command -v "$cmd" >/dev/null 2>&1 || { echo "Error: Required command '$cmd' is not installed." @@ -87,7 +55,9 @@ done trap 'echo "Script interrupted. Exiting."; exit 130' INT TERM +#--------------------------------------------- # Helper functions +#--------------------------------------------- wait_for_rsync() { for attempt in $(seq 1 "$RETRY_COUNT"); do RSYNC_COUNT=$(pgrep -fc "rsync.*root@${REMOTE_SERVER}" || true) @@ -114,7 +84,7 @@ stop_containers() { for container in "${CRITICAL_CONTAINER_NAMES[@]}"; do STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "docker inspect -f '{{.State.Running}}' $container 2>/dev/null" || echo "false") - if [ "$STATUS" == "true" ]; then + if [[ "$STATUS" == "true" ]]; then echo "Stopping $container" RUNNING_CONTAINERS+=("$container") ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $container" @@ -133,7 +103,9 @@ start_containers() { done } +#--------------------------------------------- # Main sync logic +#--------------------------------------------- main() { echo "[$LOCAL_SERVER_NAME] Syncing $DIRECTORY → $REMOTE_SERVER:$DIRECTORY" @@ -160,7 +132,6 @@ main() { --no-whole-file ) - # Apply profile-specific excludes for ex in "${EXCLUDE_DIRS[@]}"; do RSYNC_OPTS+=(--exclude="$ex") done @@ -191,5 +162,7 @@ main() { echo "[$LOCAL_SERVER_NAME] Sync complete." } +#--------------------------------------------- # Execute +#--------------------------------------------- main \ No newline at end of file diff --git a/common.sh b/common.sh index 4733399..51c690b 100644 --- a/common.sh +++ b/common.sh @@ -1,77 +1,110 @@ #!/bin/bash +# ----------------------------- Master Variables for unRAID scripts ----------------------------- -#----------------------------------------------------------------------------------------------- -# Common Shared Functions for Unraid Scripts -#----------------------------------------------------------------------------------------------- +# Hosts +HOST1="unRAID-Gmer4Lfe" +HOST2="unRAID-Jayred365" +HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" +HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" -# Load Master.conf relative to calling script -load_master_conf() { - local script_dir - script_dir="$(cd "$(dirname "${BASH_SOURCE[1]}")" && pwd)" +# Rsync Defaults +BW_LIMIT=12500 +RETRY_COUNT=3 +SLEEP=300 +MAX_RSYNC_PROCS=1 +CRITICAL_CONTAINER_NAMES=("") +DELAYED_CONTAINERS=("") +CONTAINER_DELAY=5 +EXCLUDE_DIRS=("") - local master_conf="$script_dir/../Master.conf" +# Profiles +declare -A PROFILE_MAX_PROCS=([arrs_stack]=2 [critical-data]=2 [gmer4lfe]=1 [important-data]=2 [emby]=1) +declare -A PROFILE_BW_LIMIT=([arrs_stack]=5000 [critical-data]=9500 [gmer4lfe]=8000 [important-data]=9500 [emby]=8000) +declare -A PROFILE_RETRY_COUNT=([arrs_stack]=3 [critical-data]=3 [gmer4lfe]=3 [important-data]=3 [emby]=3) +declare -A PROFILE_SLEEP=([arrs_stack]=300 [critical-data]=300 [gmer4lfe]=300 [important-data]=300 [emby]=300) +declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( + [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" + [critical-data]="Mariadb-Authelia Redis-Authelia Lldap-Gmer4Lfe NginxProxyManager Authelia" + [gmer4lfe]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe" + [important-data]="Postgres-NextCloud NextCloud" + [emby]="" +) +declare -A PROFILE_DELAYED_CONTAINERS=([arrs_stack]="" [critical-data]="Authelia" [gmer4lfe]="" [important-data]="NextCloud" [emby]="") +declare -A PROFILE_CONTAINER_DELAY=([arrs_stack]=5 [critical-data]=10 [gmer4lfe]=5 [important-data]=10 [emby]=5) +declare -A PROFILE_EXCLUDE_DIRS=([arrs_stack]="logs *.tmp" [critical-data]="logs *.tmp" [gmer4lfe]="logs *.tmp" [important-data]="logs *.tmp" [emby]="logs *.tmp") - if [ -f "$master_conf" ]; then - source "$master_conf" - echo "Loaded Master.conf from $master_conf" - else - echo "Error: Master.conf not found at $master_conf" - exit 1 - fi -} - -# Parse CLI arguments (flags + VAR=value overrides) +# ----------------------------- Common Helpers ----------------------------- parse_args() { DRY_RUN=${DRY_RUN:-false} - for ARG in "$@"; do if [[ "$ARG" == *=* ]]; then local VAR_NAME="${ARG%%=*}" local VAR_VALUE="${ARG#*=}" - if declare -p "$VAR_NAME" &>/dev/null; then - if [[ "$VAR_NAME" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then - printf -v "$VAR_NAME" '%s' "$VAR_VALUE" - echo "Overriding $VAR_NAME -> $VAR_VALUE" - else - echo "Warning: Invalid variable name $VAR_NAME" - fi + printf -v "$VAR_NAME" '%s' "$VAR_VALUE" + echo "Overriding $VAR_NAME -> $VAR_VALUE" else echo "Warning: Unknown variable $VAR_NAME, ignoring." fi else case "$ARG" in - --dry-run|-n) - DRY_RUN=true - ;; - --help|-h) - echo "Usage: script [--dry-run|-n] [VAR=value ...]" - exit 0 - ;; - *) - echo "Warning: Unknown argument $ARG" - ;; + --dry-run|-n) DRY_RUN=true ;; + --help|-h) echo "Usage: script [--dry-run|-n] [VAR=value ...]"; exit 0 ;; + *) echo "Warning: Unknown argument $ARG" ;; esac fi done } -# Validate integer (non-negative) -validate_int() { - local name="$1" - local value="$2" - - if ! [[ "${value:-}" =~ ^[0-9]+$ ]]; then - echo "Error: $name must be a non-negative integer." - exit 1 - fi -} - -# Optional: require variable to be set require_var() { local var="$1" if [[ -z "${!var:-}" ]]; then echo "Error: Required variable $var is not set." exit 1 fi -} \ No newline at end of file +} + +validate_int() { + local name="$1" + local value="$2" + if ! [[ "${value:-}" =~ ^[0-9]+$ ]]; then + echo "Error: $name must be a non-negative integer." + exit 1 + fi +} + +# ----------------------------- Host Detection & SSH ----------------------------- +# Detect local/remote host +LOCAL_HOSTNAME="$(hostname)" +if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then + LOCAL_SERVER_NAME="$HOST1" + REMOTE_SERVER_NAME="$HOST2" +elif [[ "$LOCAL_HOSTNAME" == "$HOST2" ]]; then + LOCAL_SERVER_NAME="$HOST2" + REMOTE_SERVER_NAME="$HOST1" +else + echo "Error: Local hostname ($LOCAL_HOSTNAME) not recognized." + exit 1 +fi +echo "Local server: $LOCAL_SERVER_NAME" +echo "Remote server: $REMOTE_SERVER_NAME" + +# SSH Key Mapping +declare -A SSH_KEYS +SSH_KEYS["$HOST1|$HOST2"]="$HOST1_SSH_KEY" +SSH_KEYS["$HOST2|$HOST1"]="$HOST2_SSH_KEY" +KEY_ID="$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME" +SSH_KEY="${SSH_KEYS[$KEY_ID]}" +if [[ -z "$SSH_KEY" ]]; then + echo "Error: No SSH key defined for $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" + exit 1 +fi +echo "Using SSH key: $SSH_KEY" + +# Remote IP via Tailscale +REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) +if [[ -z "$REMOTE_SERVER" ]]; then + echo "Error: Could not resolve Tailscale IP for $REMOTE_SERVER_NAME" + exit 1 +fi +echo "Remote IP: $REMOTE_SERVER" \ No newline at end of file