diff --git a/Master.conf b/Master.conf new file mode 100644 index 0000000..45a9214 --- /dev/null +++ b/Master.conf @@ -0,0 +1,69 @@ +#!/bin/bash +################################################################################################## +############################## Master Variables for unRAID scripts ################################ +################################################################################################## + +# Define the hostnames of both servers +HOST1="unRAID-Gmer4Lfe" +HOST2="unRAID-Jayred365" + +# Detect local hostname +LOCAL_HOSTNAME="$(hostname)" + +if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then + LOCAL_SERVER_NAME="$HOST1" + REMOTE_SERVER_NAME="$HOST2" +elif [[ "$LOCAL_HOSTNAME" == "$HOST2" ]]; then + LOCAL_SERVER_NAME="$HOST2" + REMOTE_SERVER_NAME="$HOST1" +else + echo "Error: Local hostname ($LOCAL_HOSTNAME) not recognized." + exit 1 +fi + +echo "Local server: $LOCAL_SERVER_NAME" +echo "Remote server: $REMOTE_SERVER_NAME" + +# Resolve Tailscale IP dynamically, This will try to get the Tailscale IPv4 address for that host +REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) +if [[ -z "$REMOTE_SERVER" ]]; then + echo "Error: Could not resolve Tailscale IP for $REMOTE_SERVER_NAME" + exit 1 +fi + +# SSH key mapping per hostpair +declare -A SSH_KEYS +SSH_KEYS["$HOST1|$HOST2"]="/root/.ssh/Gmer4Lfe-rsync-key" +SSH_KEYS["$HOST2|$HOST1"]="/root/.ssh/Jayred365-rsync-key" + +KEY_ID="$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME" + +if [[ -n "${SSH_KEYS[$KEY_ID]}" ]]; then + SSH_KEY="${SSH_KEYS[$KEY_ID]}" +else + echo "Error: No SSH key defined for $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" + exit 1 +fi + +echo "Using SSH key: $SSH_KEY" + +# Network speed limit 100Mbps=12500 KB/s, 200Mbps=25600 KB/s +BW_LIMIT=12500 + +# Amount of times to retry +RETRY_COUNT=3 + +# Time in seconds waiting to retry +SLEEP=300 + +# Max number of concurrently running rsync processes +MAX_RSYNC_PROCS=1 + +# Containers to stop/start on backup server +CRITICAL_CONTAINER_NAMES=() + +# Containers from CRITICAL_CONTAINER_NAMES that should have delayed startup +DELAYED_CONTAINERS=() + +# Time in seconds to delay the listed containers +CONTAINER_DELAY=5 diff --git a/Rsync/rsync_appdata-failover_arrs_data.sh b/Rsync/rsync_appdata-failover_arrs_data.sh new file mode 100644 index 0000000..9fd833a --- /dev/null +++ b/Rsync/rsync_appdata-failover_arrs_data.sh @@ -0,0 +1,148 @@ +#!/bin/bash +set -e + +# Detect script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Load Master.conf from Git repo +MASTER_CONF="$SCRIPT_DIR/../Master.conf" + +if [ -f "$MASTER_CONF" ]; then + source "$MASTER_CONF" + echo "Loaded Master.conf from $MASTER_CONF" +else + echo "Error: Master.conf not found at $MASTER_CONF" + exit 1 +fi + +#----------------------------------------------------------------------------------------------- +#----------------------------- User Variables, Please adjust as needed ------------------------- +#----------------------------------------------------------------------------------------------- +LOCAL_DIR="/mnt/user/appdata-Failover/Arrs_Stack/" +REMOTE_DIR="/mnt/user/appdata-Failover/Arrs_Stack/" +DRY_RUN=false +CRITICAL_CONTAINER_NAMES=("Sonarr" "Lidarr" "Readarr" "Radarr" "Prowlarr" "Bazarr" "Pinchflat") +DELAYED_CONTAINERS=("") +EXCLUDE_DIRS=("") +MAX_RSYNC_PROCS=2 +#----------------------------------------------------------------------------------------------- +#------------------------End Of User Variables, Please adjust Above as needed ------------------ +#----------------------------------------------------------------------------------------------- + +# Dependency check +for cmd in rsync ssh ping timeout docker tailscale; do + command -v "$cmd" >/dev/null 2>&1 || { + echo "Error: Required command '$cmd' is not installed." + exit 1 + } +done + +trap 'echo "Script interrupted. Exiting."; exit 130' INT TERM + + +# Helper functions +wait_for_rsync() { + for attempt in $(seq 1 "$RETRY_COUNT"); do + RSYNC_COUNT=$(pgrep -fc "rsync.*root@${REMOTE_SERVER}" || true) + if [ "$RSYNC_COUNT" -gt "$MAX_RSYNC_PROCS" ]; then + echo "[$LOCAL_SERVER_NAME] Waiting... ($RSYNC_COUNT active rsync processes)" + sleep "$SLEEP" + else + return 0 + fi + done + echo "Too many rsync processes, exiting." + exit 1 +} + +check_server_online() { + timeout 5 ping -c 1 "$REMOTE_SERVER" &> /dev/null +} + +declare -a RUNNING_CONTAINERS=() + +stop_containers() { + echo "[$LOCAL_SERVER_NAME] Checking containers on ${REMOTE_SERVER_NAME}..." + RUNNING_CONTAINERS=() + for container in "${CRITICAL_CONTAINER_NAMES[@]}"; do + STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "docker inspect -f '{{.State.Running}}' $container 2>/dev/null" || echo "false") + if [ "$STATUS" == "true" ]; then + echo "Stopping $container" + RUNNING_CONTAINERS+=("$container") + ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $container" + fi + done +} + +start_containers() { + echo "[$LOCAL_SERVER_NAME] Restarting containers on ${REMOTE_SERVER_NAME}..." + for container in "${RUNNING_CONTAINERS[@]}"; do + if [[ " ${DELAYED_CONTAINERS[*]} " == *" $container "* ]]; then + echo "Delaying start of $container by ${CONTAINER_DELAY}s..." + sleep "$CONTAINER_DELAY" + fi + ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $container" + done +} + + +# Main sync logic +main() { + echo "[$LOCAL_SERVER_NAME] Syncing to ${REMOTE_SERVER_NAME}..." + + for i in $(seq 1 "$RETRY_COUNT"); do + if check_server_online; then + break + fi + echo "Remote $REMOTE_SERVER_NAME down, retrying in $SLEEP seconds..." + sleep "$SLEEP" + done + + check_server_online || { echo "Remote $REMOTE_SERVER_NAME unreachable. Exiting."; exit 1; } + echo "[$LOCAL_SERVER_NAME] Syncing $LOCAL_DIR → $REMOTE_SERVER:$REMOTE_DIR" + wait_for_rsync + stop_containers + + RSYNC_OPTS=( + -av + --info=progress2 + --human-readable + --bwlimit="$BW_LIMIT" + --delete + --inplace + --no-whole-file + ) + + for ex in "${EXCLUDE_DIRS[@]}"; do + RSYNC_OPTS+=(--exclude="$ex") + done + + [ "$DRY_RUN" = true ] && RSYNC_OPTS+=("--dry-run") + + for attempt in $(seq 1 "$RETRY_COUNT"); do + echo "Starting rsync attempt $attempt/$RETRY_COUNT..." + if rsync "${RSYNC_OPTS[@]}" \ + -e "ssh -i \"$SSH_KEY\" -T -o Compression=no -o IPQoS=throughput" \ + "$LOCAL_DIR" "root@${REMOTE_SERVER}:$REMOTE_DIR"; then + echo "Rsync completed successfully." + break + else + echo "Rsync attempt $attempt failed." + if [ "$attempt" -lt "$RETRY_COUNT" ]; then + echo "Retrying in $SLEEP seconds..." + sleep "$SLEEP" + else + echo "All $RETRY_COUNT rsync attempts failed." + start_containers + exit 1 + fi + fi + done + + start_containers + echo "[$LOCAL_SERVER_NAME] Sync complete." +} + +# Run +main \ No newline at end of file