diff --git a/Master.conf b/Master.conf index 0393be5..9e0f685 100644 --- a/Master.conf +++ b/Master.conf @@ -16,13 +16,15 @@ ENABLE_LOGGING=true # false = only echo user-facing messages # ------------------ Git, Pull & Execute Script ----------------- - REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git" - TARGET_DIR="/mnt/user/appdata/unraid_scripts" - GITEA_SSH_KEY="/root/.ssh/id_gitea_rsync" - SSH_PORT=221 +REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git" +TARGET_DIR="/mnt/user/appdata/unraid_scripts" +GITEA_SSH_KEY="/root/.ssh/id_gitea_rsync" +SSH_PORT=221 # -------------------Rsync Script Defaults----------------------- -# Network speed limit (KB/s) +# These are the fallback values used when no matching profile is found. +# Any share whose directory basename does not match a profile key below +# will use these globals for all rsync behaviour. BW_LIMIT=12500 # Retry logic RETRY_COUNT=3 @@ -34,14 +36,15 @@ DELAYED_CONTAINERS=() # Delay in seconds between starting containers, useful for things like Authelia CONTAINER_DELAY=5 -# Not include logs during transfer +# Directories to exclude during transfer EXCLUDE_DIRS=() # Default global rsync options DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file) # ------------------- Daily Sync Shares ------------------------- # Shares synced once daily by Orchestrators/daily_sync.sh -# Add or remove paths here to manage what gets synced +# Add or remove paths here to manage what gets synced. +# These shares have no profile entry and fall through to DEFAULT_RSYNC_OPTS above. DAILY_SYNC_SHARES=( /mnt/user/Anime_Movies-Old /mnt/user/Anime_Shows-Old @@ -55,7 +58,6 @@ DAILY_SYNC_SHARES=( /mnt/user/Nextcloud /mnt/user/stand-up_comedy /mnt/user/Tv_Shows - /mnt/user/appdata-Failover/Gmer4Lfe/ ) # ------------------- unRAID essentail scripts ------------------ @@ -71,9 +73,21 @@ DAILY_SYNC_SHARES=( PHP_MAX_CHILDREN=250 # --------------------- Profile System ------------------------- -# Profiles are inferred from the directory basename (lowercased) -# Profile-specific rsync options (override global) must list all options as they do not inherit from global -# Shares with no profile entry fall through to DEFAULT_RSYNC_OPTS above +# Profiles are matched by directory basename (lowercased). +# Example: /mnt/user/appdata-Failover/Arrs_Stack → profile key = arrs_stack +# +# How fallthrough works: +# - If a key exists in a profile array, that value is used +# - If a key is missing, the global default above is used instead +# - Shares in DAILY_SYNC_SHARES have no profile and always use globals +# +# To add a new profile: +# 1. Add a key to each array below with your chosen profile name +# 2. Call rsync.sh with a directory whose basename matches that key +# 3. Any array you omit will fall back to its global default +# +# Note: PROFILE_RSYNC_OPTS does NOT inherit from DEFAULT_RSYNC_OPTS — +# if you define it for a profile you must list all desired options explicitly # SPACE-SEPARATED STRINGS declare -A PROFILE_RSYNC_OPTS=( diff --git a/Orchestrators/daily_sync.sh b/Orchestrators/daily_sync.sh index d3f7dd3..3a630c5 100644 --- a/Orchestrators/daily_sync.sh +++ b/Orchestrators/daily_sync.sh @@ -14,6 +14,19 @@ source "$SCRIPT_DIR/../common.sh" RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh" +# ----------------------------------------------------------------------------------------------- +# --- $ICON_GEAR Setup --- +# ----------------------------------------------------------------------------------------------- +echo "" +echo "--- $ICON_GEAR Setup ---" + +detect_hosts +resolve_remote_ip + +# Single connectivity check upfront — fail fast before attempting all shares +# against an unreachable host rather than failing per-share through the loop +check_connectivity + # ----------------------------------------------------------------------------------------------- # Tracking # ----------------------------------------------------------------------------------------------- @@ -21,22 +34,24 @@ PASS=() FAIL=() SHARE_TIMES=() TOTAL_START=$(date +%s) +SHARE_COUNT=${#DAILY_SYNC_SHARES[@]} # ----------------------------------------------------------------------------------------------- -# Run +# --- $ICON_SYNC Transfer --- # ----------------------------------------------------------------------------------------------- echo "" -echo "$ICON_SYNC Daily Sync Starting — $(date '+%Y-%m-%d %H:%M:%S')" -echo "Shares: ${#DAILY_SYNC_SHARES[@]}" +echo "--- $ICON_SYNC Daily Sync Starting — $(date '+%Y-%m-%d %H:%M:%S') ---" +echo "Shares: $SHARE_COUNT" echo "" +SHARE_INDEX=0 + for SHARE in "${DAILY_SYNC_SHARES[@]}"; do + SHARE_INDEX=$((SHARE_INDEX + 1)) SHARE_NAME=$(basename "$SHARE") SHARE_START=$(date +%s) - echo "----------------------------------------" - info "Syncing $SHARE_NAME..." - echo "----------------------------------------" + echo "--- $ICON_SYNC Share $SHARE_INDEX of $SHARE_COUNT: $SHARE_NAME ---" if bash "$RSYNC_SCRIPT" "$SHARE"; then SHARE_END=$(date +%s) @@ -56,7 +71,7 @@ TOTAL_END=$(date +%s) TOTAL_DURATION=$((TOTAL_END - TOTAL_START)) # ----------------------------------------------------------------------------------------------- -# Summary +# --- $ICON_SUCCESS Summary --- # ----------------------------------------------------------------------------------------------- echo "===== DAILY SYNC SUMMARY =====" echo "Started: $(date -d @$TOTAL_START '+%Y-%m-%d %H:%M:%S')" @@ -72,8 +87,8 @@ for entry in "${SHARE_TIMES[@]}"; do fi done echo "" -echo " Passed: ${#PASS[@]}/${#DAILY_SYNC_SHARES[@]}" -echo " Failed: ${#FAIL[@]}/${#DAILY_SYNC_SHARES[@]}" +echo " Passed: ${#PASS[@]}/$SHARE_COUNT" +echo " Failed: ${#FAIL[@]}/$SHARE_COUNT" echo " Duration: $(format_duration $TOTAL_DURATION)" echo "==============================" diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index f6ca990..9eaa5c2 100644 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -11,7 +11,8 @@ source "$SCRIPT_DIR/../Master.conf" source "$SCRIPT_DIR/../common.sh" # ----------------------------------------------------------------------------------------------- -# Separate the positional directory argument from flag/key=value args +# Separate the positional directory argument from flag/key=value args. +# Flags and key=value pairs are passed to parse_args — directory is handled here. # ----------------------------------------------------------------------------------------------- DIRECTORY="" RAW_ARGS=() @@ -27,23 +28,28 @@ parse_args "${RAW_ARGS[@]}" [[ -z "$DIRECTORY" ]] && error "No directory specified. Usage: rsync.sh [--dry-run] [--log]" && exit 1 +# ----------------------------------------------------------------------------------------------- +# --- $ICON_GEAR Setup --- +# ----------------------------------------------------------------------------------------------- +echo "" +echo "--- $ICON_GEAR Setup ---" + detect_hosts resolve_remote_ip check_connectivity -# ----------------------------------------------------------------------------------------------- -# Profile resolution — inferred from the directory basename -# ----------------------------------------------------------------------------------------------- +# Profile is inferred from the directory basename (lowercased) +# e.g. /mnt/user/appdata-Failover/Arrs_Stack → arrs_stack PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]') -info "Profile: $PROFILE_NAME" +info "$ICON_GEAR Loading profile: $PROFILE_NAME" -# Scalar overrides from profile +# Scalar overrides — use profile value if defined, fall back to global default BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT} RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT} SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP} CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY} -# Array overrides from profile — convert space-separated strings to proper bash arrays +# Array overrides — profile strings must be converted to bash arrays before use read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]:-}" read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]:-}" read -r -a EXCLUDE_DIRS <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-}" @@ -51,20 +57,26 @@ read -r -a EXCLUDE_DIRS <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:- [[ "$SHOW_STATUS" == true ]] && show_status && exit 0 # ----------------------------------------------------------------------------------------------- -# Run +# --- $ICON_STOP Containers --- # ----------------------------------------------------------------------------------------------- echo "" -echo "$ICON_RUN Sync Starting" +echo "--- $ICON_STOP Containers ---" + +stop_containers + +# ----------------------------------------------------------------------------------------------- +# --- $ICON_SYNC Transfer --- +# ----------------------------------------------------------------------------------------------- +echo "" +echo "--- $ICON_SYNC Transfer ---" echo "Source: $DIRECTORY" echo "Remote: $REMOTE_SERVER:$DIRECTORY" echo "Profile: $PROFILE_NAME" echo "" -stop_containers - get_rsync_opts -# Append excludes +# Append profile excludes to rsync options for ex in "${EXCLUDE_DIRS[@]}"; do [[ -n "$ex" ]] && RSYNC_OPTS+=(--exclude="$ex") done @@ -75,8 +87,10 @@ START=$(date +%s) RSYNC_SUCCESS=false for i in $(seq 1 "$RETRY_COUNT"); do - # dirname "$DIRECTORY" is intentional — rsync recreates the final directory on the remote - # e.g. source /mnt/user/Movies → remote receives /mnt/user/Movies (not /mnt/user/Movies/Movies) + info "$ICON_RETRY Attempt $i of $RETRY_COUNT..." + + # dirname "$DIRECTORY" is intentional — rsync places the directory itself on the remote + # e.g. source /mnt/user/Movies syncs to remote /mnt/user/Movies (not /mnt/user/Movies/Movies) if rsync "${RSYNC_OPTS[@]}" \ -e "ssh -i $SSH_KEY -T -o Compression=no -o IPQoS=throughput" \ "$DIRECTORY" "root@${REMOTE_SERVER}:$(dirname "$DIRECTORY")/"; then @@ -89,10 +103,19 @@ for i in $(seq 1 "$RETRY_COUNT"); do fi done +# ----------------------------------------------------------------------------------------------- +# --- $ICON_RUN Containers --- +# ----------------------------------------------------------------------------------------------- +echo "" +echo "--- $ICON_RUN Containers ---" + start_containers END=$(date +%s) +# ----------------------------------------------------------------------------------------------- +# --- $ICON_SUCCESS Summary --- +# ----------------------------------------------------------------------------------------------- echo "" echo "===== SUMMARY =====" echo "Directory: $DIRECTORY" diff --git a/common.sh b/common.sh index 6960f9c..7dbf869 100644 --- a/common.sh +++ b/common.sh @@ -2,10 +2,16 @@ # ----------------------------------------------------------------------------------------------- # ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ---------------------------- # ----------------------------------------------------------------------------------------------- -# Version: 1.1 -# Changed: format_duration moved here from daily_sync.sh for shared use -# SSH_KEY collision documented — gitea key renamed GITEA_SSH_KEY in Master.conf +# Version: 1.2 +# ----------------------------------------------------------------------------------------------- +# Changelog: +# v1.0 — Initial stable framework +# v1.1 — format_duration moved here from daily_sync.sh for shared use +# SSH_KEY collision resolved — gitea key renamed GITEA_SSH_KEY in Master.conf # Version and changelog tracking added +# v1.2 — Consistent function header comment blocks across all functions +# check_connectivity friendlier error output with tailscale hint +# check_connectivity added as standalone function # ----------------------------------------------------------------------------------------------- # ICONS @@ -19,7 +25,11 @@ ICON_RETRY="🔁" ICON_SYNC="🔄" ICON_GEAR="⚙️" -# OUTPUT +# ----------------------------------------------------------------------------------------------- +# OUTPUT HELPERS +# Standardised output functions used across all scripts. +# log() is gated by ENABLE_LOGGING — set in Master.conf or via --log flag. +# ----------------------------------------------------------------------------------------------- info() { echo "$ICON_INFO [INFO] $*"; } warn() { echo "$ICON_WARN [WARN] $*"; } error() { echo "$ICON_ERROR [ERROR] $*"; } @@ -31,8 +41,9 @@ log() { # ----------------------------------------------------------------------------------------------- # DURATION FORMATTER -# Converts raw seconds to human readable format — e.g. 10m53s or 47s -# Used by rsync.sh and daily_sync.sh +# Converts raw seconds into a human readable string — e.g. 10m53s or 47s +# Used by rsync.sh summary and daily_sync.sh summary. +# Sourced from common.sh so both scripts share the same implementation. # ----------------------------------------------------------------------------------------------- format_duration() { local secs=$1 @@ -43,6 +54,11 @@ format_duration() { # ----------------------------------------------------------------------------------------------- # ARG PARSER +# Processes all flags and key=value pairs passed to any script. +# Positional arguments (directory paths) are separated before calling this — see rsync.sh. +# Supported flags: --dry-run, --log, --no-log, --status, --help +# Supported key=value: LOG=true/false, or any declared variable e.g. BW_LIMIT=5000 +# Unparsed positional args are returned in PARSED_ARGS array. # ----------------------------------------------------------------------------------------------- parse_args() { ENABLE_LOGGING=${ENABLE_LOGGING:-false} @@ -90,6 +106,9 @@ parse_args() { # ----------------------------------------------------------------------------------------------- # VALIDATION +# Checks that a required variable is set and non-empty. +# Usage: require_var VAR_NAME +# Exits with error if the variable is missing. # ----------------------------------------------------------------------------------------------- require_var() { [[ -z "${!1:-}" ]] && error "Missing required: $1" && exit 1 @@ -97,6 +116,9 @@ require_var() { # ----------------------------------------------------------------------------------------------- # HOST DETECTION +# Determines local and remote server names by comparing hostname against HOST1/HOST2. +# Sets LOCAL_SERVER_NAME, REMOTE_SERVER_NAME, and SSH_KEY for the current run direction. +# Both HOST1 and HOST2 must be defined in Master.conf. # ----------------------------------------------------------------------------------------------- detect_hosts() { LOCAL_HOSTNAME="$(hostname)" @@ -118,41 +140,48 @@ detect_hosts() { SSH_KEY="${SSH_KEYS[$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME]}" - [[ -z "$SSH_KEY" ]] && error "Missing SSH key mapping" && exit 1 + [[ -z "$SSH_KEY" ]] && error "Missing SSH key mapping for $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" && exit 1 - info "Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" + info "$ICON_GEAR Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" } # ----------------------------------------------------------------------------------------------- -# REMOTE IP +# REMOTE IP RESOLUTION +# Resolves the Tailscale IPv4 address of the remote server. +# Sets REMOTE_SERVER used by all subsequent SSH and rsync calls. +# Exits if resolution fails — likely means Tailscale is down or peer is offline. # ----------------------------------------------------------------------------------------------- resolve_remote_ip() { - log "Resolving remote IP..." + log "$ICON_SYNC Resolving remote IP for $REMOTE_SERVER_NAME..." REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) - [[ -z "$REMOTE_SERVER" ]] && error "Failed to resolve remote IP for $REMOTE_SERVER_NAME" && exit 1 + [[ -z "$REMOTE_SERVER" ]] && error "Failed to resolve Tailscale IP for $REMOTE_SERVER_NAME" && exit 1 - info "Remote IP: $REMOTE_SERVER" + info "$ICON_SYNC Remote IP: $REMOTE_SERVER" } # ----------------------------------------------------------------------------------------------- # CONNECTIVITY CHECK -# Verifies remote is reachable before attempting rsync. -# Prevents retry loop burning all attempts against an unreachable host. +# Pings the remote server to confirm it is reachable before starting any transfers. +# Prevents the rsync retry loop from burning all attempts against an unreachable host. +# If ping fails, prints a tailscale status hint to aid diagnosis before exiting. # ----------------------------------------------------------------------------------------------- check_connectivity() { log "Checking connectivity to $REMOTE_SERVER..." if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then - error "Remote $REMOTE_SERVER is unreachable — aborting" + error "Remote $REMOTE_SERVER ($REMOTE_SERVER_NAME) is unreachable" + info "Hint: tailscale status | grep $REMOTE_SERVER_NAME" exit 1 fi log "Remote is reachable" } # ----------------------------------------------------------------------------------------------- -# CONTAINERS -# CRITICAL_CONTAINER_NAMES and DELAYED_CONTAINERS must be bash arrays before calling these. -# rsync.sh handles the read -r -a conversion from profile strings. +# CONTAINER MANAGEMENT — STOP +# Stops all containers listed in CRITICAL_CONTAINER_NAMES on the remote server. +# Only stops containers that are currently running — skips those already stopped. +# Tracks stopped containers in RUNNING_CONTAINERS for restart after rsync completes. +# CRITICAL_CONTAINER_NAMES must be a bash array — rsync.sh handles conversion from profile strings. # ----------------------------------------------------------------------------------------------- RUNNING_CONTAINERS=() @@ -189,6 +218,13 @@ stop_containers() { done } +# ----------------------------------------------------------------------------------------------- +# CONTAINER MANAGEMENT — START +# Restarts only the containers that were running before rsync and were stopped by stop_containers. +# Containers listed in DELAYED_CONTAINERS receive a sleep of CONTAINER_DELAY seconds before +# starting — useful for dependencies like Authelia that need upstream services ready first. +# DELAYED_CONTAINERS must be a bash array — rsync.sh handles conversion from profile strings. +# ----------------------------------------------------------------------------------------------- start_containers() { if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then log "No containers to restart." @@ -224,6 +260,10 @@ start_containers() { # ----------------------------------------------------------------------------------------------- # RSYNC OPTIONS +# Loads rsync options for the current profile from PROFILE_RSYNC_OPTS in Master.conf. +# If no profile match is found, falls back to DEFAULT_RSYNC_OPTS. +# Note: profile opts do NOT inherit from defaults — all desired flags must be listed explicitly. +# Sets RSYNC_OPTS array used directly in the rsync call in rsync.sh. # ----------------------------------------------------------------------------------------------- get_rsync_opts() { if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then @@ -236,7 +276,10 @@ get_rsync_opts() { } # ----------------------------------------------------------------------------------------------- -# STATUS +# STATUS DISPLAY +# Prints a summary of the current runtime configuration. +# Triggered by --status or --summary flag passed to rsync.sh. +# Useful for verifying profile resolution and variable state before a live run. # ----------------------------------------------------------------------------------------------- show_status() { echo "===== STATUS ====="