Changed daily script to master.conf variables

Master.conf

@@ -2,7 +2,7 @@
 # ----------------------------------------------------------------------------------------------
 # ---------------------------- Master Variables for unRAID scripts -----------------------------
 # ----------------------------------------------------------------------------------------------
-
+ 
 # ----------------------- Host Configuration --------------------
 # List the Hostnames of both servers
     HOST1="unRAID-Gmer4Lfe"
@@ -11,27 +11,23 @@
 #   Must have its key added to HOST1_SSH_Key same applies for HOST2
     HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key"
     HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key"
-
+ 
 # -------------------------- Logging ----------------------------
     ENABLE_LOGGING=true    # false = only echo user-facing messages
-
+ 
 # ------------------ Git, Pull & Execute Script -----------------
-REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git"
+    REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git"
-TARGET_DIR="/mnt/user/appdata/unraid_scripts"
+    TARGET_DIR="/mnt/user/appdata/unraid_scripts"
-SSH_KEY="/root/.ssh/id_gitea_rsync"
+    SSH_KEY="/root/.ssh/id_gitea_rsync"
-SSH_PORT=221
+    SSH_PORT=221
-
+ 
 # -------------------Rsync Script Defaults-----------------------
-# turns on or off progress info for rsync
-    ENABLE_PROGRESS=true
 # Network speed limit (KB/s)
     BW_LIMIT=12500
 # Retry logic
     RETRY_COUNT=3
 # Sleep between retries (seconds)
     SLEEP=300
-# Max number of concurrently running rsync processes
-    MAX_RSYNC_PROCS=1
 # Container start/stop/restart before and after rsync
     CRITICAL_CONTAINER_NAMES=()
 # Containers that need delayed before starting
@@ -42,7 +38,26 @@ SSH_PORT=221
     EXCLUDE_DIRS=()
 # Default global rsync options
     DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file)
-
+ 
+# ------------------- Daily Sync Shares -------------------------
+# Shares synced once daily by Orchestrators/daily_sync.sh
+# Add or remove paths here to manage what gets synced
+DAILY_SYNC_SHARES=(
+    /mnt/user/Anime_Movies-Old
+    /mnt/user/Anime_Shows-Old
+    /mnt/user/Anime_Shows
+    /mnt/user/Books
+    /mnt/user/Intros
+    /mnt/user/Kids_Movies
+    /mnt/user/Kids_Tv_Shows
+    /mnt/user/Movies
+    /mnt/user/Music_Videos
+    /mnt/user/Nextcloud
+    /mnt/user/stand-up_comedy
+    /mnt/user/Tv_Shows
+    /mnt/user/appdata-Failover/Gmer4Lfe/
+)
+ 
 # ------------------- unRAID essentail scripts ------------------
 # unRAID reboot script user warning time (seconds)
     REBOOT_SLEEP=300
@@ -54,11 +69,12 @@ SSH_PORT=221
     PHP_CONF="/etc/php-fpm.d/www.conf"
 # php_fpm_max_children.sh max children value
     PHP_MAX_CHILDREN=250
-
+ 
 # --------------------- Profile System -------------------------
 # Profiles are inferred from the directory basename (lowercased)
 # Profile-specific rsync options (override global) must list all options as they do not inherit from global
-
+# Shares with no profile entry fall through to DEFAULT_RSYNC_OPTS above
+ 
 # SPACE-SEPARATED STRINGS
 declare -A PROFILE_RSYNC_OPTS=(
     [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace"
@@ -67,15 +83,7 @@ declare -A PROFILE_RSYNC_OPTS=(
     [important-data]="-av --human-readable --bwlimit=$BW_LIMIT"
     [emby]="-av --human-readable --bwlimit=$BW_LIMIT"
 )
-
+ 
-declare -A PROFILE_MAX_PROCS=(
-    [arrs_stack]=2
-    [critical-data]=2
-    [gmer4lfe]=1
-    [important-data]=2
-    [emby]=1
-)
-
 declare -A PROFILE_BW_LIMIT=(
     [arrs_stack]=5000
     [critical-data]=9500
@@ -83,7 +91,7 @@ declare -A PROFILE_BW_LIMIT=(
     [important-data]=9500
     [emby]=8000
 )
-
+ 
 declare -A PROFILE_RETRY_COUNT=(
     [arrs_stack]=3
     [critical-data]=3
@@ -91,7 +99,7 @@ declare -A PROFILE_RETRY_COUNT=(
     [important-data]=3
     [emby]=3
 )
-
+ 
 declare -A PROFILE_SLEEP=(
     [arrs_stack]=300
     [critical-data]=300
@@ -99,7 +107,7 @@ declare -A PROFILE_SLEEP=(
     [important-data]=300
     [emby]=300
 )
-
+ 
 # SPACE-SEPARATED STRINGS
 declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
     [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
@@ -108,7 +116,7 @@ declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
     [important-data]="Postgres-NextCloud NextCloud"
     [emby]=""
 )
-
+ 
 # SPACE-SEPARATED STRINGS — containers that need a delay before starting
 declare -A PROFILE_DELAYED_CONTAINERS=(
     [arrs_stack]=""
@@ -117,7 +125,7 @@ declare -A PROFILE_DELAYED_CONTAINERS=(
     [important-data]="NextCloud"
     [emby]=""
 )
-
+ 
 declare -A PROFILE_CONTAINER_DELAY=(
     [arrs_stack]=5
     [critical-data]=10
@@ -125,7 +133,7 @@ declare -A PROFILE_CONTAINER_DELAY=(
     [important-data]=10
     [emby]=5
 )
-
+ 
 # SPACE-SEPARATED STRINGS
 declare -A PROFILE_EXCLUDE_DIRS=(
     [arrs_stack]="logs *.tmp"
@@ -136,4 +144,5 @@ declare -A PROFILE_EXCLUDE_DIRS=(
 )
 #-----------------------------------------------------------------------------------------------
 #---------------------- End Of User Variables, Please adjust above as needed -------------------
-#-----------------------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------------------------
+ 

Orchestrators/daily_sync.sh

@@ -4,30 +4,16 @@
 # -----------------------------------------------------------------------------------------------
 # Runs all bulk media shares sequentially. Scheduled via unRAID User Scripts plugin at 1am.
 # Per-share rsync handled by rsync.sh — this script tracks pass/fail and total time only.
+# Add or remove shares in Master.conf under DAILY_SYNC_SHARES.
 # -----------------------------------------------------------------------------------------------
-
+ 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
+ 
 source "$SCRIPT_DIR/../Master.conf"
 source "$SCRIPT_DIR/../common.sh"
-
+ 
 RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
-
+ 
-SHARES=(
-    /mnt/user/Anime_Movies-Old
-    /mnt/user/Anime_Shows-Old
-    /mnt/user/Anime_Shows
-    /mnt/user/Books
-    /mnt/user/Intros
-    /mnt/user/Kids_Movies
-    /mnt/user/Kids_Tv_Shows
-    /mnt/user/Movies
-    /mnt/user/Music_Videos
-    /mnt/user/Nextcloud
-    /mnt/user/stand-up_comedy
-    /mnt/user/Tv_Shows
-)
-
 # -----------------------------------------------------------------------------------------------
 # Duration Formatter
 # -----------------------------------------------------------------------------------------------
@@ -37,7 +23,7 @@ format_duration() {
     local rem=$((secs % 60))
     [[ $mins -gt 0 ]] && echo "${mins}m${rem}s" || echo "${rem}s"
 }
-
+ 
 # -----------------------------------------------------------------------------------------------
 # Tracking
 # -----------------------------------------------------------------------------------------------
@@ -45,23 +31,23 @@ PASS=()
 FAIL=()
 SHARE_TIMES=()
 TOTAL_START=$(date +%s)
-
+ 
 # -----------------------------------------------------------------------------------------------
 # Run
 # -----------------------------------------------------------------------------------------------
 echo ""
 echo "$ICON_SYNC Daily Sync Starting — $(date '+%Y-%m-%d %H:%M:%S')"
-echo "Shares: ${#SHARES[@]}"
+echo "Shares: ${#DAILY_SYNC_SHARES[@]}"
 echo ""
-
+ 
-for SHARE in "${SHARES[@]}"; do
+for SHARE in "${DAILY_SYNC_SHARES[@]}"; do
     SHARE_NAME=$(basename "$SHARE")
     SHARE_START=$(date +%s)
-
+ 
     echo "----------------------------------------"
     info "Syncing $SHARE_NAME..."
     echo "----------------------------------------"
-
+ 
     if bash "$RSYNC_SCRIPT" "$SHARE"; then
         SHARE_END=$(date +%s)
         SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))")
@@ -72,13 +58,13 @@ for SHARE in "${SHARES[@]}"; do
         FAIL+=("$SHARE_NAME")
         error "$SHARE_NAME failed — continuing to next share"
     fi
-
+ 
     echo ""
 done
-
+ 
 TOTAL_END=$(date +%s)
 TOTAL_DURATION=$((TOTAL_END - TOTAL_START))
-
+ 
 # -----------------------------------------------------------------------------------------------
 # Summary
 # -----------------------------------------------------------------------------------------------
@@ -96,10 +82,10 @@ for entry in "${SHARE_TIMES[@]}"; do
     fi
 done
 echo ""
-echo "  Passed:   ${#PASS[@]}/${#SHARES[@]}"
+echo "  Passed:   ${#PASS[@]}/${#DAILY_SYNC_SHARES[@]}"
-echo "  Failed:   ${#FAIL[@]}/${#SHARES[@]}"
+echo "  Failed:   ${#FAIL[@]}/${#DAILY_SYNC_SHARES[@]}"
 echo "  Duration: $(format_duration $TOTAL_DURATION)"
 echo "=============================="
-
+ 
 [[ ${#FAIL[@]} -gt 0 ]] && exit 1
 exit 0

Rsync/rsync.sh

@@ -4,54 +4,51 @@
 # ----------------------------------------------------------------------------------------------
 # ---------------- User Variables, Please adjust in Master.conf as needed ----------------------
 # ----------------------------------------------------------------------------------------------
-
+ 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
+ 
 source "$SCRIPT_DIR/../Master.conf"
 source "$SCRIPT_DIR/../common.sh"
-
+ 
 # -----------------------------------------------------------------------------------------------
 # Separate the positional directory argument from flag/key=value args
 # -----------------------------------------------------------------------------------------------
 DIRECTORY=""
 RAW_ARGS=()
-
+ 
 for ARG in "$@"; do
     case "$ARG" in
         --*|*=*) RAW_ARGS+=("$ARG") ;;
         *) [[ -z "$DIRECTORY" ]] && DIRECTORY="$ARG" || RAW_ARGS+=("$ARG") ;;
     esac
 done
-
+ 
 parse_args "${RAW_ARGS[@]}"
-
+ 
 [[ -z "$DIRECTORY" ]] && error "No directory specified. Usage: rsync.sh <dir> [--dry-run] [--log]" && exit 1
-
+ 
 detect_hosts
 resolve_remote_ip
-
+ 
 # -----------------------------------------------------------------------------------------------
 # Profile resolution — inferred from the directory basename
 # -----------------------------------------------------------------------------------------------
 PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]')
 info "Profile: $PROFILE_NAME"
-
+ 
 # Scalar overrides from profile
-MAX_RSYNC_PROCS=${PROFILE_MAX_PROCS[$PROFILE_NAME]:-$MAX_RSYNC_PROCS}
 BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
 RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
 SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
 CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY}
-
+ 
 # Array overrides from profile — convert space-separated strings to proper bash arrays
 read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]:-}"
 read -r -a DELAYED_CONTAINERS       <<< "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]:-}"
 read -r -a EXCLUDE_DIRS             <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-}"
-echo "DEBUG containers array size: ${#CRITICAL_CONTAINER_NAMES[@]}"
+ 
-echo "DEBUG containers content: ${CRITICAL_CONTAINER_NAMES[*]}"
-
 [[ "$SHOW_STATUS" == true ]] && show_status && exit 0
-
+ 
 # -----------------------------------------------------------------------------------------------
 # Run
 # -----------------------------------------------------------------------------------------------
@@ -61,23 +58,21 @@ echo "Source:  $DIRECTORY"
 echo "Remote:  $REMOTE_SERVER:$DIRECTORY"
 echo "Profile: $PROFILE_NAME"
 echo ""
-
+ 
-wait_for_rsync
-
 stop_containers
-
+ 
 get_rsync_opts
-
+ 
 # Append excludes
 for ex in "${EXCLUDE_DIRS[@]}"; do
     [[ -n "$ex" ]] && RSYNC_OPTS+=(--exclude="$ex")
 done
-
+ 
 [[ "$DRY_RUN" == true ]] && RSYNC_OPTS+=("--dry-run") && warn "DRY RUN — no changes will be made"
-
+ 
 START=$(date +%s)
 RSYNC_SUCCESS=false
-
+ 
 for i in $(seq 1 "$RETRY_COUNT"); do
     if rsync "${RSYNC_OPTS[@]}" \
         -e "ssh -i $SSH_KEY -T -o Compression=no -o IPQoS=throughput" \
@@ -90,11 +85,11 @@ for i in $(seq 1 "$RETRY_COUNT"); do
         [[ "$i" -lt "$RETRY_COUNT" ]] && info "Retrying in ${SLEEP}s..." && sleep "$SLEEP"
     fi
 done
-
+ 
 start_containers
-
+ 
 END=$(date +%s)
-
+ 
 echo ""
 echo "===== SUMMARY ====="
 echo "Directory: $DIRECTORY"
@@ -106,6 +101,6 @@ else
     echo "Status:    $ICON_ERROR FAILED after $RETRY_COUNT attempts"
 fi
 echo "==================="
-
+ 
 [[ "$RSYNC_SUCCESS" == false ]] && exit 1
-exit 0
+exit 0

common.sh

@@ -191,30 +191,6 @@ start_containers() {
     done
 }
  
-# -----------------------------------------------------------------------------------------------
-# RSYNC SLOT LIMITER
-# -----------------------------------------------------------------------------------------------
-wait_for_rsync() {
-    : "${RETRY_COUNT:=5}"
-    : "${MAX_RSYNC_PROCS:=2}"
-    : "${SLEEP:=2}"
- 
-    for i in $(seq 1 "$RETRY_COUNT"); do
-        COUNT=$(ps -eo cmd | grep "[r]sync .*${REMOTE_SERVER}" | wc -l || true)
- 
-        if [[ "$COUNT" -ge "$MAX_RSYNC_PROCS" ]]; then
-            warn "$ICON_RETRY Waiting for rsync slot ($COUNT/$MAX_RSYNC_PROCS) — attempt $i/$RETRY_COUNT"
-            sleep "$SLEEP"
-        else
-            log "Rsync slot available ($COUNT/$MAX_RSYNC_PROCS)"
-            return 0
-        fi
-    done
- 
-    error "Too many rsync processes after $RETRY_COUNT attempts, aborting."
-    exit 1
-}
- 
 # -----------------------------------------------------------------------------------------------
 # RSYNC OPTIONS
 # -----------------------------------------------------------------------------------------------