FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Getting order correct

Browse Source
This commit is contained in:
FailedProxy
2026-03-24 20:04:38 +00:00
parent 6df103061e
commit b6e39cc8c4
3 changed files with 129 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Master.conf
+4 -5
View File
@@ -21,7 +21,6 @@
HOST1="unRAID-Gmer4Lfe"
HOST2="unRAID-Jayred365"
#-----------------------------------------------------------------------------------------------
#----------------- User Variables --------------------------------------------------------------
#-----------------------------------------------------------------------------------------------
@@ -36,13 +35,13 @@ SLEEP=300
# Max number of concurrently running rsync processes
MAX_RSYNC_PROCS=1
# Container start/stop/restart before and after rsync
CRITICAL_CONTAINER_NAMES=("")
CRITICAL_CONTAINER_NAMES=()
# Containers that need delayed before starting
DELAYED_CONTAINERS=("")
DELAYED_CONTAINERS=()
# Delay in seconds between starting containers, useful for things like Authelia
CONTAINER_DELAY=5
# Not include logs during transfer
EXCLUDE_DIRS=("")
EXCLUDE_DIRS=()
#-------------------- unRAID essentail scripts ------------------
# unRAID reboot script user warning time (seconds)
@@ -114,7 +113,7 @@ PROFILE_CONTAINER_DELAY["important-data"]=10
PROFILE_CONTAINER_DELAY["emby"]=5
# Not include logs or other file types during transfer per profile
declare -A PROFILE_EXCLUDE_DIRS
PROFILE_EXCLUDE_DIRS["arrs_stack"]="logs *.tmp"
PROFILE_EXCLUDE_DIRS["arrs_stack"]="logs"
PROFILE_EXCLUDE_DIRS["critical-data"]="logs *.tmp"
PROFILE_EXCLUDE_DIRS["gmer4lfe"]="logs *.tmp"
PROFILE_EXCLUDE_DIRS["important-data"]="logs *.tmp"
Rsync/rsync.sh
+59 -33
View File
@@ -4,54 +4,53 @@ set -e
#-----------------------------------------------------------------------------------------------
# --------------------------- Core Rsync Script for Unraid -------------------------------------
#-----------------------------------------------------------------------------------------------
#----------------- User Variables, Please adjust in Master.conf as needed ----------------------
#-----------------------------------------------------------------------------------------------
#
# User variables can be adjusted in Master.conf
#
# Scripts now just contain runtime logic
#
# This new setup allows for profiles and arguments to use the core script
# Use arguments in unRAID User Plugin for directory
# Example: /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-test/Arrs/
#
# /mnt/user/appdata-test/Arrs/ is the argument
# If the last name /Arrs/ matches a profile in Master.conf, then profile defaults are used
# Otherwise, global configuration in Master.conf is used
#-----------------------------------------------------------------------------------------------
#---------------End Of User Variables, Please adjust in Master.conf as needed ------------------
#-----------------------------------------------------------------------------------------------
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../common.sh"
# Load central config
# Load config FIRST
load_master_conf
# Input Arguments
#----------------------------------------------------------
# Input Arguments (MUST come before parse_args)
#----------------------------------------------------------
DIRECTORY="$1"
if [[ -z "$DIRECTORY" ]]; then
echo "Usage: $0 <directory-to-sync> [--dry-run|-n] [VAR=value ...]"
exit 1
fi
shift # remove DIRECTORY
shift
# Parse CLI args (flags + VAR=value)
#----------------------------------------------------------
# Parse CLI args AFTER shifting directory
#----------------------------------------------------------
parse_args "$@"
#----------------------------------------------------------
# At this point, common.sh has already:
# - LOCAL_SERVER_NAME
# - REMOTE_SERVER_NAME
# - REMOTE_SERVER
# - SSH_KEY
#----------------------------------------------------------
# Determine profile
PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]')
echo "Detected profile: $PROFILE_NAME"
# Apply profile defaults or fallback to Master.conf
# Apply profile defaults (safe fallbacks)
MAX_RSYNC_PROCS=${PROFILE_MAX_PROCS[$PROFILE_NAME]:-$MAX_RSYNC_PROCS}
BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
CRITICAL_CONTAINER_NAMES=(${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]})
DELAYED_CONTAINERS=(${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]})
# Convert string → array safely
IFS=' ' read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]}"
IFS=' ' read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]}"
CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY}
RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
EXCLUDE_DIRS=(${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-${EXCLUDE_DIRS[@]}})
IFS=' ' read -r -a EXCLUDE_DIRS <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-${EXCLUDE_DIRS[*]}}"
# Validate numeric variables
validate_int MAX_RSYNC_PROCS "$MAX_RSYNC_PROCS"
@@ -60,19 +59,30 @@ validate_int CONTAINER_DELAY "$CONTAINER_DELAY"
validate_int RETRY_COUNT "$RETRY_COUNT"
validate_int SLEEP "$SLEEP"
echo "Settings: MAX_RSYNC_PROCS=$MAX_RSYNC_PROCS, BW_LIMIT=$BW_LIMIT, CONTAINERS=${CRITICAL_CONTAINER_NAMES[*]}, EXCLUDES=${EXCLUDE_DIRS[*]}"
echo "Settings:"
echo " MAX_RSYNC_PROCS=$MAX_RSYNC_PROCS"
echo " BW_LIMIT=$BW_LIMIT"
echo " CONTAINERS=${CRITICAL_CONTAINER_NAMES[*]}"
echo " EXCLUDES=${EXCLUDE_DIRS[*]}"
# Dependency check
for cmd in rsync ssh ping timeout docker tailscale; do
command -v "$cmd" >/dev/null 2>&1 || { echo "Error: Required command '$cmd' is not installed."; exit 1; }
command -v "$cmd" >/dev/null 2>&1 || {
echo "Error: Required command '$cmd' is not installed."
exit 1
}
done
trap 'echo "Script interrupted. Exiting."; exit 130' INT TERM
# Helper functions
#----------------------------------------------------------
# Helper Functions
#----------------------------------------------------------
wait_for_rsync() {
for attempt in $(seq 1 "$RETRY_COUNT"); do
RSYNC_COUNT=$(pgrep -fc "rsync.*root@${REMOTE_SERVER}" || true)
if [ "$RSYNC_COUNT" -ge "$MAX_RSYNC_PROCS" ]; then
echo "[$LOCAL_SERVER_NAME] Waiting... ($RSYNC_COUNT active rsync processes)"
sleep "$SLEEP"
@@ -80,6 +90,7 @@ wait_for_rsync() {
return 0
fi
done
echo "Too many rsync processes, exiting."
exit 1
}
@@ -93,12 +104,17 @@ declare -a RUNNING_CONTAINERS=()
stop_containers() {
echo "[$LOCAL_SERVER_NAME] Checking containers on ${REMOTE_SERVER_NAME}..."
RUNNING_CONTAINERS=()
for container in "${CRITICAL_CONTAINER_NAMES[@]}"; do
[ -z "$container" ] && continue
STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"docker inspect -f '{{.State.Running}}' $container 2>/dev/null" || echo "false")
if [ "$STATUS" == "true" ]; then
echo "Stopping $container"
RUNNING_CONTAINERS+=("$container")
if [ "$DRY_RUN" = true ]; then
echo "[DRY-RUN] Would stop container $container"
else
@@ -110,11 +126,13 @@ stop_containers() {
start_containers() {
echo "[$LOCAL_SERVER_NAME] Restarting containers on ${REMOTE_SERVER_NAME}..."
for container in "${RUNNING_CONTAINERS[@]}"; do
if [[ " ${DELAYED_CONTAINERS[*]} " == *" $container "* ]]; then
echo "Delaying start of $container by ${CONTAINER_DELAY}s..."
[ "$DRY_RUN" = false ] && sleep "$CONTAINER_DELAY"
fi
if [ "$DRY_RUN" = true ]; then
echo "[DRY-RUN] Would start container $container"
else
@@ -123,7 +141,10 @@ start_containers() {
done
}
# Main sync logic
#----------------------------------------------------------
# Main
#----------------------------------------------------------
main() {
echo "[$LOCAL_SERVER_NAME] Syncing $DIRECTORY → $REMOTE_SERVER:$DIRECTORY"
@@ -133,7 +154,10 @@ main() {
sleep "$SLEEP"
done
check_server_online || { echo "Remote $REMOTE_SERVER_NAME unreachable. Exiting."; exit 1; }
check_server_online || {
echo "Remote $REMOTE_SERVER_NAME unreachable. Exiting."
exit 1
}
wait_for_rsync
stop_containers
@@ -148,22 +172,25 @@ main() {
--no-whole-file
)
# Apply multiple exclude patterns
for pattern in "${EXCLUDE_DIRS[@]}"; do
RSYNC_OPTS+=(--exclude="$pattern")
[ -n "$pattern" ] && RSYNC_OPTS+=(--exclude="$pattern")
done
[ "$DRY_RUN" = true ] && RSYNC_OPTS+=("--dry-run")
for attempt in $(seq 1 "$RETRY_COUNT"); do
echo "Starting rsync attempt $attempt/$RETRY_COUNT..."
if rsync "${RSYNC_OPTS[@]}" \
-e "ssh -i \"$SSH_KEY\" -T -o Compression=no -o IPQoS=throughput" \
"$DIRECTORY" "root@${REMOTE_SERVER}:$DIRECTORY"; then
echo "Rsync completed successfully."
break
else
echo "Rsync attempt $attempt failed."
if [ "$attempt" -lt "$RETRY_COUNT" ]; then
echo "Retrying in $SLEEP seconds..."
sleep "$SLEEP"
@@ -179,5 +206,4 @@ main() {
echo "[$LOCAL_SERVER_NAME] Sync complete."
}
# Execute
main
common.sh
+66 -53
View File
@@ -3,21 +3,14 @@
#-----------------------------------------------------------------------------------------------
# --------------------- Common Shared Functions for Unraid Scripts -----------------------------
#-----------------------------------------------------------------------------------------------
#----------------- User Variables, Please adjust in Master.conf as needed ----------------------
#-----------------------------------------------------------------------------------------------
#
# Nothing to adjust here, all user variables should be in Master.conf
#
# This is just a script that has all common functions and logic that are shared
# between the different scripts.
#
# This way we can avoid code duplication and have a single source
# of truth for all common functions and logic.
#-----------------------------------------------------------------------------------------------
#---------------End Of User Variables, Please adjust in Master.conf as needed ------------------
# This file contains ONLY reusable logic.
# No code should execute on load — only functions.
# All user configuration lives in Master.conf
#-----------------------------------------------------------------------------------------------
#-----------------------------------------------------------------------------------------------
# Load Master.conf
#-----------------------------------------------------------------------------------------------
load_master_conf() {
local script_dir
script_dir="$(cd "$(dirname "${BASH_SOURCE[1]}")" && pwd)"
@@ -32,7 +25,9 @@ load_master_conf() {
fi
}
# Parse CLI args
#-----------------------------------------------------------------------------------------------
# Parse CLI args (flags + VAR=value overrides)
#-----------------------------------------------------------------------------------------------
parse_args() {
DRY_RUN=${DRY_RUN:-false}
@@ -49,74 +44,92 @@ parse_args() {
fi
else
case "$ARG" in
--dry-run|-n) DRY_RUN=true ;;
--dry-run|-n)
DRY_RUN=true
;;
--help|-h)
echo "Usage: script [--dry-run|-n] [VAR=value ...]"
exit 0 ;;
exit 0
;;
*)
echo "Warning: Unknown argument $ARG" ;;
echo "Warning: Unknown argument $ARG"
;;
esac
fi
done
}
# Validate integer
#-----------------------------------------------------------------------------------------------
# Validate integer (non-negative)
#-----------------------------------------------------------------------------------------------
validate_int() {
local name="$1"
local value="$2"
if ! [[ "${value:-}" =~ ^[0-9]+$ ]]; then
echo "Error: $name must be a non-negative integer."
exit 1
fi
}
#-----------------------------------------------------------------------------------------------
# Require variable to be set
#-----------------------------------------------------------------------------------------------
require_var() {
local var="$1"
if [[ -z "${!var:-}" ]]; then
echo "Error: Required variable $var is not set."
exit 1
fi
}
# Detect local hostname
LOCAL_HOSTNAME="$(hostname)"
if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then
LOCAL_SERVER_NAME="$HOST1"
REMOTE_SERVER_NAME="$HOST2"
elif [[ "$LOCAL_HOSTNAME" == "$HOST2" ]]; then
LOCAL_SERVER_NAME="$HOST2"
REMOTE_SERVER_NAME="$HOST1"
else
echo "Error: Local hostname ($LOCAL_HOSTNAME) not recognized."
exit 1
fi
echo "Local server: $LOCAL_SERVER_NAME"
echo "Remote server: $REMOTE_SERVER_NAME"
#-----------------------------------------------------------------------------------------------
#--------------------------- Remote Server Ip Resolution ----------------------------------------
# Detect hosts + networking + SSH key
#-----------------------------------------------------------------------------------------------
# Resolve Tailscale IP dynamically
REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)
if [[ -z "$REMOTE_SERVER" ]]; then
echo "Error: Could not resolve Tailscale IP for $REMOTE_SERVER_NAME"
exit 1
fi
detect_hosts() {
# Ensure required config exists
require_var HOST1
require_var HOST2
#-----------------------------------------------------------------------------------------------
#----------------------------- SSH Key Mapping --------------------------------------------------
#-----------------------------------------------------------------------------------------------
# Detect local hostname
LOCAL_HOSTNAME="$(hostname)"
declare -A SSH_KEYS
SSH_KEYS["$HOST1|$HOST2"]="/root/.ssh/Gmer4Lfe-rsync-key"
SSH_KEYS["$HOST2|$HOST1"]="/root/.ssh/Jayred365-rsync-key"
if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then
LOCAL_SERVER_NAME="$HOST1"
REMOTE_SERVER_NAME="$HOST2"
elif [[ "$LOCAL_HOSTNAME" == "$HOST2" ]]; then
LOCAL_SERVER_NAME="$HOST2"
REMOTE_SERVER_NAME="$HOST1"
else
echo "Error: Local hostname ($LOCAL_HOSTNAME) not recognized."
exit 1
fi
KEY_ID="$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME"
if [[ -n "${SSH_KEYS[$KEY_ID]}" ]]; then
SSH_KEY="${SSH_KEYS[$KEY_ID]}"
else
echo "Error: No SSH key defined for $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME"
exit 1
fi
echo "Using SSH key: $SSH_KEY"
echo "Local server: $LOCAL_SERVER_NAME"
echo "Remote server: $REMOTE_SERVER_NAME"
# Resolve Tailscale IP
REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)
if [[ -z "$REMOTE_SERVER" ]]; then
echo "Error: Could not resolve Tailscale IP for $REMOTE_SERVER_NAME"
exit 1
fi
# SSH Key Mapping (kept here for stability)
declare -A SSH_KEYS
SSH_KEYS["$HOST1|$HOST2"]="/root/.ssh/Gmer4Lfe-rsync-key"
SSH_KEYS["$HOST2|$HOST1"]="/root/.ssh/Jayred365-rsync-key"
KEY_ID="$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME"
if [[ -n "${SSH_KEYS[$KEY_ID]}" ]]; then
SSH_KEY="${SSH_KEYS[$KEY_ID]}"
else
echo "Error: No SSH key defined for $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME"
exit 1
fi
echo "Using SSH key: $SSH_KEY"
}