From 1376bf3411c3cda7795f6aaaf4d0312bfa6b69ad Mon Sep 17 00:00:00 2001 From: FailedProxy Date: Sun, 5 Apr 2026 06:13:05 +0000 Subject: [PATCH] Changed daily script to master.conf variables --- Master.conf | 71 +++++++++++++++++++++---------------- Orchestrators/daily_sync.sh | 50 ++++++++++---------------- Rsync/rsync.sh | 51 ++++++++++++-------------- common.sh | 24 ------------- 4 files changed, 81 insertions(+), 115 deletions(-) diff --git a/Master.conf b/Master.conf index fc49620..10e8a84 100644 --- a/Master.conf +++ b/Master.conf @@ -2,7 +2,7 @@ # ---------------------------------------------------------------------------------------------- # ---------------------------- Master Variables for unRAID scripts ----------------------------- # ---------------------------------------------------------------------------------------------- - + # ----------------------- Host Configuration -------------------- # List the Hostnames of both servers HOST1="unRAID-Gmer4Lfe" @@ -11,27 +11,23 @@ # Must have its key added to HOST1_SSH_Key same applies for HOST2 HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" - + # -------------------------- Logging ---------------------------- ENABLE_LOGGING=true # false = only echo user-facing messages - + # ------------------ Git, Pull & Execute Script ----------------- -REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git" -TARGET_DIR="/mnt/user/appdata/unraid_scripts" -SSH_KEY="/root/.ssh/id_gitea_rsync" -SSH_PORT=221 - + REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git" + TARGET_DIR="/mnt/user/appdata/unraid_scripts" + SSH_KEY="/root/.ssh/id_gitea_rsync" + SSH_PORT=221 + # -------------------Rsync Script Defaults----------------------- -# turns on or off progress info for rsync - ENABLE_PROGRESS=true # Network speed limit (KB/s) BW_LIMIT=12500 # Retry logic RETRY_COUNT=3 # Sleep between retries (seconds) SLEEP=300 -# Max number of concurrently running rsync processes - MAX_RSYNC_PROCS=1 # Container start/stop/restart before and after rsync CRITICAL_CONTAINER_NAMES=() # Containers that need delayed before starting @@ -42,7 +38,26 @@ SSH_PORT=221 EXCLUDE_DIRS=() # Default global rsync options DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file) - + +# ------------------- Daily Sync Shares ------------------------- +# Shares synced once daily by Orchestrators/daily_sync.sh +# Add or remove paths here to manage what gets synced +DAILY_SYNC_SHARES=( + /mnt/user/Anime_Movies-Old + /mnt/user/Anime_Shows-Old + /mnt/user/Anime_Shows + /mnt/user/Books + /mnt/user/Intros + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movies + /mnt/user/Music_Videos + /mnt/user/Nextcloud + /mnt/user/stand-up_comedy + /mnt/user/Tv_Shows + /mnt/user/appdata-Failover/Gmer4Lfe/ +) + # ------------------- unRAID essentail scripts ------------------ # unRAID reboot script user warning time (seconds) REBOOT_SLEEP=300 @@ -54,11 +69,12 @@ SSH_PORT=221 PHP_CONF="/etc/php-fpm.d/www.conf" # php_fpm_max_children.sh max children value PHP_MAX_CHILDREN=250 - + # --------------------- Profile System ------------------------- # Profiles are inferred from the directory basename (lowercased) # Profile-specific rsync options (override global) must list all options as they do not inherit from global - +# Shares with no profile entry fall through to DEFAULT_RSYNC_OPTS above + # SPACE-SEPARATED STRINGS declare -A PROFILE_RSYNC_OPTS=( [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" @@ -67,15 +83,7 @@ declare -A PROFILE_RSYNC_OPTS=( [important-data]="-av --human-readable --bwlimit=$BW_LIMIT" [emby]="-av --human-readable --bwlimit=$BW_LIMIT" ) - -declare -A PROFILE_MAX_PROCS=( - [arrs_stack]=2 - [critical-data]=2 - [gmer4lfe]=1 - [important-data]=2 - [emby]=1 -) - + declare -A PROFILE_BW_LIMIT=( [arrs_stack]=5000 [critical-data]=9500 @@ -83,7 +91,7 @@ declare -A PROFILE_BW_LIMIT=( [important-data]=9500 [emby]=8000 ) - + declare -A PROFILE_RETRY_COUNT=( [arrs_stack]=3 [critical-data]=3 @@ -91,7 +99,7 @@ declare -A PROFILE_RETRY_COUNT=( [important-data]=3 [emby]=3 ) - + declare -A PROFILE_SLEEP=( [arrs_stack]=300 [critical-data]=300 @@ -99,7 +107,7 @@ declare -A PROFILE_SLEEP=( [important-data]=300 [emby]=300 ) - + # SPACE-SEPARATED STRINGS declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" @@ -108,7 +116,7 @@ declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( [important-data]="Postgres-NextCloud NextCloud" [emby]="" ) - + # SPACE-SEPARATED STRINGS — containers that need a delay before starting declare -A PROFILE_DELAYED_CONTAINERS=( [arrs_stack]="" @@ -117,7 +125,7 @@ declare -A PROFILE_DELAYED_CONTAINERS=( [important-data]="NextCloud" [emby]="" ) - + declare -A PROFILE_CONTAINER_DELAY=( [arrs_stack]=5 [critical-data]=10 @@ -125,7 +133,7 @@ declare -A PROFILE_CONTAINER_DELAY=( [important-data]=10 [emby]=5 ) - + # SPACE-SEPARATED STRINGS declare -A PROFILE_EXCLUDE_DIRS=( [arrs_stack]="logs *.tmp" @@ -136,4 +144,5 @@ declare -A PROFILE_EXCLUDE_DIRS=( ) #----------------------------------------------------------------------------------------------- #---------------------- End Of User Variables, Please adjust above as needed ------------------- -#----------------------------------------------------------------------------------------------- \ No newline at end of file +#----------------------------------------------------------------------------------------------- + \ No newline at end of file diff --git a/Orchestrators/daily_sync.sh b/Orchestrators/daily_sync.sh index 550621c..6097a74 100644 --- a/Orchestrators/daily_sync.sh +++ b/Orchestrators/daily_sync.sh @@ -4,30 +4,16 @@ # ----------------------------------------------------------------------------------------------- # Runs all bulk media shares sequentially. Scheduled via unRAID User Scripts plugin at 1am. # Per-share rsync handled by rsync.sh — this script tracks pass/fail and total time only. +# Add or remove shares in Master.conf under DAILY_SYNC_SHARES. # ----------------------------------------------------------------------------------------------- - + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - + source "$SCRIPT_DIR/../Master.conf" source "$SCRIPT_DIR/../common.sh" - + RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh" - -SHARES=( - /mnt/user/Anime_Movies-Old - /mnt/user/Anime_Shows-Old - /mnt/user/Anime_Shows - /mnt/user/Books - /mnt/user/Intros - /mnt/user/Kids_Movies - /mnt/user/Kids_Tv_Shows - /mnt/user/Movies - /mnt/user/Music_Videos - /mnt/user/Nextcloud - /mnt/user/stand-up_comedy - /mnt/user/Tv_Shows -) - + # ----------------------------------------------------------------------------------------------- # Duration Formatter # ----------------------------------------------------------------------------------------------- @@ -37,7 +23,7 @@ format_duration() { local rem=$((secs % 60)) [[ $mins -gt 0 ]] && echo "${mins}m${rem}s" || echo "${rem}s" } - + # ----------------------------------------------------------------------------------------------- # Tracking # ----------------------------------------------------------------------------------------------- @@ -45,23 +31,23 @@ PASS=() FAIL=() SHARE_TIMES=() TOTAL_START=$(date +%s) - + # ----------------------------------------------------------------------------------------------- # Run # ----------------------------------------------------------------------------------------------- echo "" echo "$ICON_SYNC Daily Sync Starting — $(date '+%Y-%m-%d %H:%M:%S')" -echo "Shares: ${#SHARES[@]}" +echo "Shares: ${#DAILY_SYNC_SHARES[@]}" echo "" - -for SHARE in "${SHARES[@]}"; do + +for SHARE in "${DAILY_SYNC_SHARES[@]}"; do SHARE_NAME=$(basename "$SHARE") SHARE_START=$(date +%s) - + echo "----------------------------------------" info "Syncing $SHARE_NAME..." echo "----------------------------------------" - + if bash "$RSYNC_SCRIPT" "$SHARE"; then SHARE_END=$(date +%s) SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))") @@ -72,13 +58,13 @@ for SHARE in "${SHARES[@]}"; do FAIL+=("$SHARE_NAME") error "$SHARE_NAME failed — continuing to next share" fi - + echo "" done - + TOTAL_END=$(date +%s) TOTAL_DURATION=$((TOTAL_END - TOTAL_START)) - + # ----------------------------------------------------------------------------------------------- # Summary # ----------------------------------------------------------------------------------------------- @@ -96,10 +82,10 @@ for entry in "${SHARE_TIMES[@]}"; do fi done echo "" -echo " Passed: ${#PASS[@]}/${#SHARES[@]}" -echo " Failed: ${#FAIL[@]}/${#SHARES[@]}" +echo " Passed: ${#PASS[@]}/${#DAILY_SYNC_SHARES[@]}" +echo " Failed: ${#FAIL[@]}/${#DAILY_SYNC_SHARES[@]}" echo " Duration: $(format_duration $TOTAL_DURATION)" echo "==============================" - + [[ ${#FAIL[@]} -gt 0 ]] && exit 1 exit 0 \ No newline at end of file diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index a935f9d..4f9ed55 100644 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -4,54 +4,51 @@ # ---------------------------------------------------------------------------------------------- # ---------------- User Variables, Please adjust in Master.conf as needed ---------------------- # ---------------------------------------------------------------------------------------------- - + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - + source "$SCRIPT_DIR/../Master.conf" source "$SCRIPT_DIR/../common.sh" - + # ----------------------------------------------------------------------------------------------- # Separate the positional directory argument from flag/key=value args # ----------------------------------------------------------------------------------------------- DIRECTORY="" RAW_ARGS=() - + for ARG in "$@"; do case "$ARG" in --*|*=*) RAW_ARGS+=("$ARG") ;; *) [[ -z "$DIRECTORY" ]] && DIRECTORY="$ARG" || RAW_ARGS+=("$ARG") ;; esac done - + parse_args "${RAW_ARGS[@]}" - + [[ -z "$DIRECTORY" ]] && error "No directory specified. Usage: rsync.sh [--dry-run] [--log]" && exit 1 - + detect_hosts resolve_remote_ip - + # ----------------------------------------------------------------------------------------------- # Profile resolution — inferred from the directory basename # ----------------------------------------------------------------------------------------------- PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]') info "Profile: $PROFILE_NAME" - + # Scalar overrides from profile -MAX_RSYNC_PROCS=${PROFILE_MAX_PROCS[$PROFILE_NAME]:-$MAX_RSYNC_PROCS} BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT} RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT} SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP} CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY} - + # Array overrides from profile — convert space-separated strings to proper bash arrays read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]:-}" read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]:-}" read -r -a EXCLUDE_DIRS <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-}" -echo "DEBUG containers array size: ${#CRITICAL_CONTAINER_NAMES[@]}" -echo "DEBUG containers content: ${CRITICAL_CONTAINER_NAMES[*]}" - + [[ "$SHOW_STATUS" == true ]] && show_status && exit 0 - + # ----------------------------------------------------------------------------------------------- # Run # ----------------------------------------------------------------------------------------------- @@ -61,23 +58,21 @@ echo "Source: $DIRECTORY" echo "Remote: $REMOTE_SERVER:$DIRECTORY" echo "Profile: $PROFILE_NAME" echo "" - -wait_for_rsync - + stop_containers - + get_rsync_opts - + # Append excludes for ex in "${EXCLUDE_DIRS[@]}"; do [[ -n "$ex" ]] && RSYNC_OPTS+=(--exclude="$ex") done - + [[ "$DRY_RUN" == true ]] && RSYNC_OPTS+=("--dry-run") && warn "DRY RUN — no changes will be made" - + START=$(date +%s) RSYNC_SUCCESS=false - + for i in $(seq 1 "$RETRY_COUNT"); do if rsync "${RSYNC_OPTS[@]}" \ -e "ssh -i $SSH_KEY -T -o Compression=no -o IPQoS=throughput" \ @@ -90,11 +85,11 @@ for i in $(seq 1 "$RETRY_COUNT"); do [[ "$i" -lt "$RETRY_COUNT" ]] && info "Retrying in ${SLEEP}s..." && sleep "$SLEEP" fi done - + start_containers - + END=$(date +%s) - + echo "" echo "===== SUMMARY =====" echo "Directory: $DIRECTORY" @@ -106,6 +101,6 @@ else echo "Status: $ICON_ERROR FAILED after $RETRY_COUNT attempts" fi echo "===================" - + [[ "$RSYNC_SUCCESS" == false ]] && exit 1 -exit 0 +exit 0 \ No newline at end of file diff --git a/common.sh b/common.sh index 01127fa..8e00168 100644 --- a/common.sh +++ b/common.sh @@ -191,30 +191,6 @@ start_containers() { done } -# ----------------------------------------------------------------------------------------------- -# RSYNC SLOT LIMITER -# ----------------------------------------------------------------------------------------------- -wait_for_rsync() { - : "${RETRY_COUNT:=5}" - : "${MAX_RSYNC_PROCS:=2}" - : "${SLEEP:=2}" - - for i in $(seq 1 "$RETRY_COUNT"); do - COUNT=$(ps -eo cmd | grep "[r]sync .*${REMOTE_SERVER}" | wc -l || true) - - if [[ "$COUNT" -ge "$MAX_RSYNC_PROCS" ]]; then - warn "$ICON_RETRY Waiting for rsync slot ($COUNT/$MAX_RSYNC_PROCS) — attempt $i/$RETRY_COUNT" - sleep "$SLEEP" - else - log "Rsync slot available ($COUNT/$MAX_RSYNC_PROCS)" - return 0 - fi - done - - error "Too many rsync processes after $RETRY_COUNT attempts, aborting." - exit 1 -} - # ----------------------------------------------------------------------------------------------- # RSYNC OPTIONS # -----------------------------------------------------------------------------------------------