master with all helpers

This commit is contained in:
2026-03-24 22:43:43 +00:00
parent 56548f409e
commit 1d806711c9
3 changed files with 171 additions and 188 deletions
+67 -90
View File
@@ -51,81 +51,64 @@ PHP_CONF="/etc/php-fpm.d/www.conf"
# php_fpm_max_children.sh max children value
PHP_MAX_CHILDREN=250
# Profiles
declare -A PROFILE_MAX_PROCS=([arrs_stack]=2 [critical-data]=2 [gmer4lfe]=1 [important-data]=2 [emby]=1)
declare -A PROFILE_BW_LIMIT=([arrs_stack]=5000 [critical-data]=9500 [gmer4lfe]=8000 [important-data]=9500 [emby]=8000)
declare -A PROFILE_RETRY_COUNT=([arrs_stack]=3 [critical-data]=3 [gmer4lfe]=3 [important-data]=3 [emby]=3)
declare -A PROFILE_SLEEP=([arrs_stack]=300 [critical-data]=300 [gmer4lfe]=300 [important-data]=300 [emby]=300)
declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
[arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
[critical-data]="Mariadb-Authelia Redis-Authelia Lldap-Gmer4Lfe NginxProxyManager Authelia"
[gmer4lfe]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
[important-data]="Postgres-NextCloud NextCloud"
[emby]=""
)
declare -A PROFILE_DELAYED_CONTAINERS=([arrs_stack]="" [critical-data]="Authelia" [gmer4lfe]="" [important-data]="NextCloud" [emby]="")
declare -A PROFILE_CONTAINER_DELAY=([arrs_stack]=5 [critical-data]=10 [gmer4lfe]=5 [important-data]=10 [emby]=5)
declare -A PROFILE_EXCLUDE_DIRS=([arrs_stack]="logs *.tmp" [critical-data]="logs *.tmp" [gmer4lfe]="logs *.tmp" [important-data]="logs *.tmp" [emby]="logs *.tmp")
#----------------------- Profile System ------------------------
# ----------------------------- Common Helpers -----------------------------
parse_args() {
DRY_RUN=${DRY_RUN:-false}
for ARG in "$@"; do
if [[ "$ARG" == *=* ]]; then
local VAR_NAME="${ARG%%=*}"
local VAR_VALUE="${ARG#*=}"
if declare -p "$VAR_NAME" &>/dev/null; then
printf -v "$VAR_NAME" '%s' "$VAR_VALUE"
echo "Overriding $VAR_NAME -> $VAR_VALUE"
else
echo "Warning: Unknown variable $VAR_NAME, ignoring."
fi
else
case "$ARG" in
--dry-run|-n) DRY_RUN=true ;;
--help|-h) echo "Usage: script [--dry-run|-n] [VAR=value ...]"; exit 0 ;;
*) echo "Warning: Unknown argument $ARG" ;;
esac
fi
done
}
# Profiles are mainly used for rsync operations, but can be used for any variable that needs
# to be different based on the source or destination.
#
# Profiles are inferred from the directory basename (lowercased)
# Max rsync processes per profile
declare -A PROFILE_MAX_PROCS
PROFILE_MAX_PROCS["arrs_stack"]=2
PROFILE_MAX_PROCS["critical-data"]=2
PROFILE_MAX_PROCS["gmer4lfe"]=1
PROFILE_MAX_PROCS["important-data"]=2
PROFILE_MAX_PROCS["emby"]=1
# Bandwidth limits per profile (KB/s, 0 = unlimited)
declare -A PROFILE_BW_LIMIT
PROFILE_BW_LIMIT["arrs_stack"]=5000
PROFILE_BW_LIMIT["critical-data"]=9500
PROFILE_BW_LIMIT["gmer4lfe"]=8000
PROFILE_BW_LIMIT["important-data"]=9500
PROFILE_BW_LIMIT["emby"]=8000
# Retry logic per profile
declare -A PROFILE_RETRY_COUNT
PROFILE_RETRY_COUNT["arrs_stack"]=3
PROFILE_RETRY_COUNT["critical-data"]=3
PROFILE_RETRY_COUNT["gmer4lfe"]=3
PROFILE_RETRY_COUNT["important-data"]=3
PROFILE_RETRY_COUNT["emby"]=3
# Sleep between retries per profile (seconds)
declare -A PROFILE_SLEEP
PROFILE_SLEEP["arrs_stack"]=300
PROFILE_SLEEP["critical-data"]=300
PROFILE_SLEEP["gmer4lfe"]=300
PROFILE_SLEEP["important-data"]=300
PROFILE_SLEEP["emby"]=300
# Containers to stop/start per profile (space-separated strings)
declare -A PROFILE_CRITICAL_CONTAINER_NAMES
PROFILE_CRITICAL_CONTAINER_NAMES["arrs_stack"]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
PROFILE_CRITICAL_CONTAINER_NAMES["critical-data"]="Mariadb-Authelia Redis-Authelia Lldap-Gmer4Lfe NginxProxyManager Authelia"
PROFILE_CRITICAL_CONTAINER_NAMES["gmer4lfe"]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
PROFILE_CRITICAL_CONTAINER_NAMES["important-data"]="Postgres-NextCloud NextCloud"
PROFILE_CRITICAL_CONTAINER_NAMES["emby"]=""
# Delayed containers per profile (space-separated strings)
declare -A PROFILE_DELAYED_CONTAINERS
PROFILE_DELAYED_CONTAINERS["arrs_stack"]=""
PROFILE_DELAYED_CONTAINERS["critical-data"]="Authelia"
PROFILE_DELAYED_CONTAINERS["gmer4lfe"]=""
PROFILE_DELAYED_CONTAINERS["important-data"]="NextCloud"
PROFILE_DELAYED_CONTAINERS["emby"]=""
# Delay in seconds before starting delayed containers per profile (seconds)
declare -A PROFILE_CONTAINER_DELAY
PROFILE_CONTAINER_DELAY["arrs_stack"]=5
PROFILE_CONTAINER_DELAY["critical-data"]=10
PROFILE_CONTAINER_DELAY["gmer4lfe"]=5
PROFILE_CONTAINER_DELAY["important-data"]=10
PROFILE_CONTAINER_DELAY["emby"]=5
# Not include logs or other file types during transfer per profile
declare -A PROFILE_EXCLUDE_DIRS
PROFILE_EXCLUDE_DIRS["arrs_stack"]="logs *.tmp"
PROFILE_EXCLUDE_DIRS["critical-data"]="logs *.tmp"
PROFILE_EXCLUDE_DIRS["gmer4lfe"]="logs *.tmp"
PROFILE_EXCLUDE_DIRS["important-data"]="logs *.tmp"
PROFILE_EXCLUDE_DIRS["emby"]="logs *.tmp"
#-----------------------------------------------------------------------------------------------
#----------------------End Of User Variables, Please adjust Above as needed --------------------
#-----------------------------------------------------------------------------------------------
#
# Next is commom runtime logic, tried to include logic in a common script but was having trouble
# So now its at the botton of the file, but please do not adjust unless you understand the code
# and what it does, as changes here will affect all scripts that source this master.conf file
#
#-----------------------------------------------------------------------------------------------
#--------------------Run Logic, Do not adjust unless you understand the code -------------------
#-----------------------------------------------------------------------------------------------
# Detect local hostname
require_var() {
local var="$1"
if [[ -z "${!var:-}" ]]; then
echo "Error: Required variable $var is not set."
exit 1
fi
}
validate_int() {
local name="$1"
local value="$2"
if ! [[ "${value:-}" =~ ^[0-9]+$ ]]; then
echo "Error: $name must be a non-negative integer."
exit 1
fi
}
# ----------------------------- Host Detection & SSH -----------------------------
# Detect local/remote host
LOCAL_HOSTNAME="$(hostname)"
if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then
LOCAL_SERVER_NAME="$HOST1"
@@ -139,29 +122,23 @@ else
fi
echo "Local server: $LOCAL_SERVER_NAME"
echo "Remote server: $REMOTE_SERVER_NAME"
echo "DEBUG: LOCAL_HOSTNAME=$LOCAL_HOSTNAME, LOCAL_SERVER_NAME=$LOCAL_SERVER_NAME, REMOTE_SERVER_NAME=$REMOTE_SERVER_NAME"
echo "DEBUG: KEY_ID=$KEY_ID, SSH_KEY=${SSH_KEYS[$KEY_ID]}"
# Remote Server Ip Resolution
# Resolve Tailscale IP dynamically
REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)
if [[ -z "$REMOTE_SERVER" ]]; then
echo "Error: Could not resolve Tailscale IP for $REMOTE_SERVER_NAME"
exit 1
fi
# SSH Key Mapping
declare -A SSH_KEYS
SSH_KEYS["$HOST1|$HOST2"]="$HOST1_SSH_KEY"
SSH_KEYS["$HOST2|$HOST1"]="$HOST2_SSH_KEY"
# Determine which key to use
KEY_ID="$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME"
if [[ -n "${SSH_KEYS[$KEY_ID]}" ]]; then
SSH_KEY="${SSH_KEYS[$KEY_ID]}"
else
SSH_KEY="${SSH_KEYS[$KEY_ID]}"
if [[ -z "$SSH_KEY" ]]; then
echo "Error: No SSH key defined for $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME"
exit 1
fi
echo "Using SSH key: $SSH_KEY"
echo "Using SSH key: $SSH_KEY"
# Remote IP via Tailscale
REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)
if [[ -z "$REMOTE_SERVER" ]]; then
echo "Error: Could not resolve Tailscale IP for $REMOTE_SERVER_NAME"
exit 1
fi
echo "Remote IP: $REMOTE_SERVER"
+22 -49
View File
@@ -1,29 +1,12 @@
#!/bin/bash
set -e
#-----------------------------------------------------------------------------------------------
#----------------- User Variables, Please adjust in Master.conf as needed ----------------------
#-----------------------------------------------------------------------------------------------
# This is the main Core Rsync script
#
# User variables can be adjusted in Master.conf
# Scripts now just contain runtime logic
#
# This allows for profiles and multiple things to use the core script
# Use arguments in unRAID User Plugin for directory
# Example: /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-test/Arrs/
#
# /mnt/user/appdata-test/Arrs/ is the argument
# If the last name /Arrs/ matches a profile in Master.conf, then profile defaults are used
# Otherwise, global configuration in Master.conf is used
#-----------------------------------------------------------------------------------------------
#---------------End Of User Variables ----------------------------------------------------------
#-----------------------------------------------------------------------------------------------
# Detect script directory
#---------------------------------------------
# Detect script directory and load Master.conf
#---------------------------------------------
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Load Master.conf
MASTER_CONF="$SCRIPT_DIR/../Master.conf"
if [ -f "$MASTER_CONF" ]; then
source "$MASTER_CONF"
echo "Loaded Master.conf from $MASTER_CONF"
@@ -32,40 +15,23 @@ else
exit 1
fi
# Input Arguments
DIRECTORY="$1" # Directory to sync
#---------------------------------------------
# Parse arguments (using Master.conf helper)
#---------------------------------------------
parse_args "$@"
DIRECTORY="$1"
if [[ -z "$DIRECTORY" ]]; then
echo "Usage: $0 <directory-to-sync> [--dry-run] [VAR=value ...]"
exit 1
fi
# Optional dry-run
DRY_RUN=false
shift 1
if [[ "$1" == "--dry-run" ]]; then
DRY_RUN=true
shift 1
fi
# Optional variable overrides: VAR=value
for ARG in "$@"; do
if [[ "$ARG" == *=* ]]; then
VAR_NAME="${ARG%%=*}"
VAR_VALUE="${ARG#*=}"
if declare -p "$VAR_NAME" &>/dev/null; then
eval "$VAR_NAME=\"$VAR_VALUE\""
echo "Overriding $VAR_NAME -> $VAR_VALUE"
else
echo "Warning: Unknown variable $VAR_NAME, ignoring."
fi
fi
done
# Determine profile
#---------------------------------------------
# Determine profile & apply defaults
#---------------------------------------------
PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]')
echo "Detected profile: $PROFILE_NAME"
# Apply profile defaults or fallback to master.conf
MAX_RSYNC_PROCS=${PROFILE_MAX_PROCS[$PROFILE_NAME]:-$MAX_RSYNC_PROCS}
BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
CRITICAL_CONTAINER_NAMES=(${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]})
@@ -77,7 +43,9 @@ EXCLUDE_DIRS=(${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-${EXCLUDE_DIRS[@]}})
echo "Settings: MAX_RSYNC_PROCS=$MAX_RSYNC_PROCS, BW_LIMIT=$BW_LIMIT, CONTAINERS=${CRITICAL_CONTAINER_NAMES[*]}, EXCLUDES=${EXCLUDE_DIRS[*]}"
#---------------------------------------------
# Dependency check
#---------------------------------------------
for cmd in rsync ssh ping timeout docker tailscale; do
command -v "$cmd" >/dev/null 2>&1 || {
echo "Error: Required command '$cmd' is not installed."
@@ -87,7 +55,9 @@ done
trap 'echo "Script interrupted. Exiting."; exit 130' INT TERM
#---------------------------------------------
# Helper functions
#---------------------------------------------
wait_for_rsync() {
for attempt in $(seq 1 "$RETRY_COUNT"); do
RSYNC_COUNT=$(pgrep -fc "rsync.*root@${REMOTE_SERVER}" || true)
@@ -114,7 +84,7 @@ stop_containers() {
for container in "${CRITICAL_CONTAINER_NAMES[@]}"; do
STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"docker inspect -f '{{.State.Running}}' $container 2>/dev/null" || echo "false")
if [ "$STATUS" == "true" ]; then
if [[ "$STATUS" == "true" ]]; then
echo "Stopping $container"
RUNNING_CONTAINERS+=("$container")
ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $container"
@@ -133,7 +103,9 @@ start_containers() {
done
}
#---------------------------------------------
# Main sync logic
#---------------------------------------------
main() {
echo "[$LOCAL_SERVER_NAME] Syncing $DIRECTORY$REMOTE_SERVER:$DIRECTORY"
@@ -160,7 +132,6 @@ main() {
--no-whole-file
)
# Apply profile-specific excludes
for ex in "${EXCLUDE_DIRS[@]}"; do
RSYNC_OPTS+=(--exclude="$ex")
done
@@ -191,5 +162,7 @@ main() {
echo "[$LOCAL_SERVER_NAME] Sync complete."
}
#---------------------------------------------
# Execute
#---------------------------------------------
main
+82 -49
View File
@@ -1,77 +1,110 @@
#!/bin/bash
# ----------------------------- Master Variables for unRAID scripts -----------------------------
#-----------------------------------------------------------------------------------------------
# Common Shared Functions for Unraid Scripts
#-----------------------------------------------------------------------------------------------
# Hosts
HOST1="unRAID-Gmer4Lfe"
HOST2="unRAID-Jayred365"
HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key"
HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key"
# Load Master.conf relative to calling script
load_master_conf() {
local script_dir
script_dir="$(cd "$(dirname "${BASH_SOURCE[1]}")" && pwd)"
# Rsync Defaults
BW_LIMIT=12500
RETRY_COUNT=3
SLEEP=300
MAX_RSYNC_PROCS=1
CRITICAL_CONTAINER_NAMES=("")
DELAYED_CONTAINERS=("")
CONTAINER_DELAY=5
EXCLUDE_DIRS=("")
local master_conf="$script_dir/../Master.conf"
# Profiles
declare -A PROFILE_MAX_PROCS=([arrs_stack]=2 [critical-data]=2 [gmer4lfe]=1 [important-data]=2 [emby]=1)
declare -A PROFILE_BW_LIMIT=([arrs_stack]=5000 [critical-data]=9500 [gmer4lfe]=8000 [important-data]=9500 [emby]=8000)
declare -A PROFILE_RETRY_COUNT=([arrs_stack]=3 [critical-data]=3 [gmer4lfe]=3 [important-data]=3 [emby]=3)
declare -A PROFILE_SLEEP=([arrs_stack]=300 [critical-data]=300 [gmer4lfe]=300 [important-data]=300 [emby]=300)
declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
[arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
[critical-data]="Mariadb-Authelia Redis-Authelia Lldap-Gmer4Lfe NginxProxyManager Authelia"
[gmer4lfe]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
[important-data]="Postgres-NextCloud NextCloud"
[emby]=""
)
declare -A PROFILE_DELAYED_CONTAINERS=([arrs_stack]="" [critical-data]="Authelia" [gmer4lfe]="" [important-data]="NextCloud" [emby]="")
declare -A PROFILE_CONTAINER_DELAY=([arrs_stack]=5 [critical-data]=10 [gmer4lfe]=5 [important-data]=10 [emby]=5)
declare -A PROFILE_EXCLUDE_DIRS=([arrs_stack]="logs *.tmp" [critical-data]="logs *.tmp" [gmer4lfe]="logs *.tmp" [important-data]="logs *.tmp" [emby]="logs *.tmp")
if [ -f "$master_conf" ]; then
source "$master_conf"
echo "Loaded Master.conf from $master_conf"
else
echo "Error: Master.conf not found at $master_conf"
exit 1
fi
}
# Parse CLI arguments (flags + VAR=value overrides)
# ----------------------------- Common Helpers -----------------------------
parse_args() {
DRY_RUN=${DRY_RUN:-false}
for ARG in "$@"; do
if [[ "$ARG" == *=* ]]; then
local VAR_NAME="${ARG%%=*}"
local VAR_VALUE="${ARG#*=}"
if declare -p "$VAR_NAME" &>/dev/null; then
if [[ "$VAR_NAME" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then
printf -v "$VAR_NAME" '%s' "$VAR_VALUE"
echo "Overriding $VAR_NAME -> $VAR_VALUE"
else
echo "Warning: Invalid variable name $VAR_NAME"
fi
printf -v "$VAR_NAME" '%s' "$VAR_VALUE"
echo "Overriding $VAR_NAME -> $VAR_VALUE"
else
echo "Warning: Unknown variable $VAR_NAME, ignoring."
fi
else
case "$ARG" in
--dry-run|-n)
DRY_RUN=true
;;
--help|-h)
echo "Usage: script [--dry-run|-n] [VAR=value ...]"
exit 0
;;
*)
echo "Warning: Unknown argument $ARG"
;;
--dry-run|-n) DRY_RUN=true ;;
--help|-h) echo "Usage: script [--dry-run|-n] [VAR=value ...]"; exit 0 ;;
*) echo "Warning: Unknown argument $ARG" ;;
esac
fi
done
}
# Validate integer (non-negative)
validate_int() {
local name="$1"
local value="$2"
if ! [[ "${value:-}" =~ ^[0-9]+$ ]]; then
echo "Error: $name must be a non-negative integer."
exit 1
fi
}
# Optional: require variable to be set
require_var() {
local var="$1"
if [[ -z "${!var:-}" ]]; then
echo "Error: Required variable $var is not set."
exit 1
fi
}
}
validate_int() {
local name="$1"
local value="$2"
if ! [[ "${value:-}" =~ ^[0-9]+$ ]]; then
echo "Error: $name must be a non-negative integer."
exit 1
fi
}
# ----------------------------- Host Detection & SSH -----------------------------
# Detect local/remote host
LOCAL_HOSTNAME="$(hostname)"
if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then
LOCAL_SERVER_NAME="$HOST1"
REMOTE_SERVER_NAME="$HOST2"
elif [[ "$LOCAL_HOSTNAME" == "$HOST2" ]]; then
LOCAL_SERVER_NAME="$HOST2"
REMOTE_SERVER_NAME="$HOST1"
else
echo "Error: Local hostname ($LOCAL_HOSTNAME) not recognized."
exit 1
fi
echo "Local server: $LOCAL_SERVER_NAME"
echo "Remote server: $REMOTE_SERVER_NAME"
# SSH Key Mapping
declare -A SSH_KEYS
SSH_KEYS["$HOST1|$HOST2"]="$HOST1_SSH_KEY"
SSH_KEYS["$HOST2|$HOST1"]="$HOST2_SSH_KEY"
KEY_ID="$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME"
SSH_KEY="${SSH_KEYS[$KEY_ID]}"
if [[ -z "$SSH_KEY" ]]; then
echo "Error: No SSH key defined for $LOCAL_SERVER_NAME$REMOTE_SERVER_NAME"
exit 1
fi
echo "Using SSH key: $SSH_KEY"
# Remote IP via Tailscale
REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)
if [[ -z "$REMOTE_SERVER" ]]; then
echo "Error: Could not resolve Tailscale IP for $REMOTE_SERVER_NAME"
exit 1
fi
echo "Remote IP: $REMOTE_SERVER"