Changed daily script to master.conf variables
This commit is contained in:
+40
-31
@@ -2,7 +2,7 @@
|
||||
# ----------------------------------------------------------------------------------------------
|
||||
# ---------------------------- Master Variables for unRAID scripts -----------------------------
|
||||
# ----------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
# ----------------------- Host Configuration --------------------
|
||||
# List the Hostnames of both servers
|
||||
HOST1="unRAID-Gmer4Lfe"
|
||||
@@ -11,27 +11,23 @@
|
||||
# Must have its key added to HOST1_SSH_Key same applies for HOST2
|
||||
HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key"
|
||||
HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key"
|
||||
|
||||
|
||||
# -------------------------- Logging ----------------------------
|
||||
ENABLE_LOGGING=true # false = only echo user-facing messages
|
||||
|
||||
|
||||
# ------------------ Git, Pull & Execute Script -----------------
|
||||
REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git"
|
||||
TARGET_DIR="/mnt/user/appdata/unraid_scripts"
|
||||
SSH_KEY="/root/.ssh/id_gitea_rsync"
|
||||
SSH_PORT=221
|
||||
|
||||
REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git"
|
||||
TARGET_DIR="/mnt/user/appdata/unraid_scripts"
|
||||
SSH_KEY="/root/.ssh/id_gitea_rsync"
|
||||
SSH_PORT=221
|
||||
|
||||
# -------------------Rsync Script Defaults-----------------------
|
||||
# turns on or off progress info for rsync
|
||||
ENABLE_PROGRESS=true
|
||||
# Network speed limit (KB/s)
|
||||
BW_LIMIT=12500
|
||||
# Retry logic
|
||||
RETRY_COUNT=3
|
||||
# Sleep between retries (seconds)
|
||||
SLEEP=300
|
||||
# Max number of concurrently running rsync processes
|
||||
MAX_RSYNC_PROCS=1
|
||||
# Container start/stop/restart before and after rsync
|
||||
CRITICAL_CONTAINER_NAMES=()
|
||||
# Containers that need delayed before starting
|
||||
@@ -42,7 +38,26 @@ SSH_PORT=221
|
||||
EXCLUDE_DIRS=()
|
||||
# Default global rsync options
|
||||
DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file)
|
||||
|
||||
|
||||
# ------------------- Daily Sync Shares -------------------------
|
||||
# Shares synced once daily by Orchestrators/daily_sync.sh
|
||||
# Add or remove paths here to manage what gets synced
|
||||
DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Books
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Nextcloud
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/appdata-Failover/Gmer4Lfe/
|
||||
)
|
||||
|
||||
# ------------------- unRAID essentail scripts ------------------
|
||||
# unRAID reboot script user warning time (seconds)
|
||||
REBOOT_SLEEP=300
|
||||
@@ -54,11 +69,12 @@ SSH_PORT=221
|
||||
PHP_CONF="/etc/php-fpm.d/www.conf"
|
||||
# php_fpm_max_children.sh max children value
|
||||
PHP_MAX_CHILDREN=250
|
||||
|
||||
|
||||
# --------------------- Profile System -------------------------
|
||||
# Profiles are inferred from the directory basename (lowercased)
|
||||
# Profile-specific rsync options (override global) must list all options as they do not inherit from global
|
||||
|
||||
# Shares with no profile entry fall through to DEFAULT_RSYNC_OPTS above
|
||||
|
||||
# SPACE-SEPARATED STRINGS
|
||||
declare -A PROFILE_RSYNC_OPTS=(
|
||||
[arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace"
|
||||
@@ -67,15 +83,7 @@ declare -A PROFILE_RSYNC_OPTS=(
|
||||
[important-data]="-av --human-readable --bwlimit=$BW_LIMIT"
|
||||
[emby]="-av --human-readable --bwlimit=$BW_LIMIT"
|
||||
)
|
||||
|
||||
declare -A PROFILE_MAX_PROCS=(
|
||||
[arrs_stack]=2
|
||||
[critical-data]=2
|
||||
[gmer4lfe]=1
|
||||
[important-data]=2
|
||||
[emby]=1
|
||||
)
|
||||
|
||||
|
||||
declare -A PROFILE_BW_LIMIT=(
|
||||
[arrs_stack]=5000
|
||||
[critical-data]=9500
|
||||
@@ -83,7 +91,7 @@ declare -A PROFILE_BW_LIMIT=(
|
||||
[important-data]=9500
|
||||
[emby]=8000
|
||||
)
|
||||
|
||||
|
||||
declare -A PROFILE_RETRY_COUNT=(
|
||||
[arrs_stack]=3
|
||||
[critical-data]=3
|
||||
@@ -91,7 +99,7 @@ declare -A PROFILE_RETRY_COUNT=(
|
||||
[important-data]=3
|
||||
[emby]=3
|
||||
)
|
||||
|
||||
|
||||
declare -A PROFILE_SLEEP=(
|
||||
[arrs_stack]=300
|
||||
[critical-data]=300
|
||||
@@ -99,7 +107,7 @@ declare -A PROFILE_SLEEP=(
|
||||
[important-data]=300
|
||||
[emby]=300
|
||||
)
|
||||
|
||||
|
||||
# SPACE-SEPARATED STRINGS
|
||||
declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
|
||||
[arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
|
||||
@@ -108,7 +116,7 @@ declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
|
||||
[important-data]="Postgres-NextCloud NextCloud"
|
||||
[emby]=""
|
||||
)
|
||||
|
||||
|
||||
# SPACE-SEPARATED STRINGS — containers that need a delay before starting
|
||||
declare -A PROFILE_DELAYED_CONTAINERS=(
|
||||
[arrs_stack]=""
|
||||
@@ -117,7 +125,7 @@ declare -A PROFILE_DELAYED_CONTAINERS=(
|
||||
[important-data]="NextCloud"
|
||||
[emby]=""
|
||||
)
|
||||
|
||||
|
||||
declare -A PROFILE_CONTAINER_DELAY=(
|
||||
[arrs_stack]=5
|
||||
[critical-data]=10
|
||||
@@ -125,7 +133,7 @@ declare -A PROFILE_CONTAINER_DELAY=(
|
||||
[important-data]=10
|
||||
[emby]=5
|
||||
)
|
||||
|
||||
|
||||
# SPACE-SEPARATED STRINGS
|
||||
declare -A PROFILE_EXCLUDE_DIRS=(
|
||||
[arrs_stack]="logs *.tmp"
|
||||
@@ -136,4 +144,5 @@ declare -A PROFILE_EXCLUDE_DIRS=(
|
||||
)
|
||||
#-----------------------------------------------------------------------------------------------
|
||||
#---------------------- End Of User Variables, Please adjust above as needed -------------------
|
||||
#-----------------------------------------------------------------------------------------------
|
||||
#-----------------------------------------------------------------------------------------------
|
||||
|
||||
+18
-32
@@ -4,30 +4,16 @@
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Runs all bulk media shares sequentially. Scheduled via unRAID User Scripts plugin at 1am.
|
||||
# Per-share rsync handled by rsync.sh — this script tracks pass/fail and total time only.
|
||||
# Add or remove shares in Master.conf under DAILY_SYNC_SHARES.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
|
||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
||||
|
||||
SHARES=(
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Books
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Nextcloud
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Shows
|
||||
)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Duration Formatter
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
@@ -37,7 +23,7 @@ format_duration() {
|
||||
local rem=$((secs % 60))
|
||||
[[ $mins -gt 0 ]] && echo "${mins}m${rem}s" || echo "${rem}s"
|
||||
}
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Tracking
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
@@ -45,23 +31,23 @@ PASS=()
|
||||
FAIL=()
|
||||
SHARE_TIMES=()
|
||||
TOTAL_START=$(date +%s)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Run
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "$ICON_SYNC Daily Sync Starting — $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "Shares: ${#SHARES[@]}"
|
||||
echo "Shares: ${#DAILY_SYNC_SHARES[@]}"
|
||||
echo ""
|
||||
|
||||
for SHARE in "${SHARES[@]}"; do
|
||||
|
||||
for SHARE in "${DAILY_SYNC_SHARES[@]}"; do
|
||||
SHARE_NAME=$(basename "$SHARE")
|
||||
SHARE_START=$(date +%s)
|
||||
|
||||
|
||||
echo "----------------------------------------"
|
||||
info "Syncing $SHARE_NAME..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
|
||||
if bash "$RSYNC_SCRIPT" "$SHARE"; then
|
||||
SHARE_END=$(date +%s)
|
||||
SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))")
|
||||
@@ -72,13 +58,13 @@ for SHARE in "${SHARES[@]}"; do
|
||||
FAIL+=("$SHARE_NAME")
|
||||
error "$SHARE_NAME failed — continuing to next share"
|
||||
fi
|
||||
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
|
||||
TOTAL_END=$(date +%s)
|
||||
TOTAL_DURATION=$((TOTAL_END - TOTAL_START))
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Summary
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
@@ -96,10 +82,10 @@ for entry in "${SHARE_TIMES[@]}"; do
|
||||
fi
|
||||
done
|
||||
echo ""
|
||||
echo " Passed: ${#PASS[@]}/${#SHARES[@]}"
|
||||
echo " Failed: ${#FAIL[@]}/${#SHARES[@]}"
|
||||
echo " Passed: ${#PASS[@]}/${#DAILY_SYNC_SHARES[@]}"
|
||||
echo " Failed: ${#FAIL[@]}/${#DAILY_SYNC_SHARES[@]}"
|
||||
echo " Duration: $(format_duration $TOTAL_DURATION)"
|
||||
echo "=============================="
|
||||
|
||||
|
||||
[[ ${#FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
+23
-28
@@ -4,54 +4,51 @@
|
||||
# ----------------------------------------------------------------------------------------------
|
||||
# ---------------- User Variables, Please adjust in Master.conf as needed ----------------------
|
||||
# ----------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Separate the positional directory argument from flag/key=value args
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
DIRECTORY=""
|
||||
RAW_ARGS=()
|
||||
|
||||
|
||||
for ARG in "$@"; do
|
||||
case "$ARG" in
|
||||
--*|*=*) RAW_ARGS+=("$ARG") ;;
|
||||
*) [[ -z "$DIRECTORY" ]] && DIRECTORY="$ARG" || RAW_ARGS+=("$ARG") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
|
||||
parse_args "${RAW_ARGS[@]}"
|
||||
|
||||
|
||||
[[ -z "$DIRECTORY" ]] && error "No directory specified. Usage: rsync.sh <dir> [--dry-run] [--log]" && exit 1
|
||||
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Profile resolution — inferred from the directory basename
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]')
|
||||
info "Profile: $PROFILE_NAME"
|
||||
|
||||
|
||||
# Scalar overrides from profile
|
||||
MAX_RSYNC_PROCS=${PROFILE_MAX_PROCS[$PROFILE_NAME]:-$MAX_RSYNC_PROCS}
|
||||
BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
|
||||
RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
|
||||
SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
|
||||
CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY}
|
||||
|
||||
|
||||
# Array overrides from profile — convert space-separated strings to proper bash arrays
|
||||
read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]:-}"
|
||||
read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]:-}"
|
||||
read -r -a EXCLUDE_DIRS <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-}"
|
||||
echo "DEBUG containers array size: ${#CRITICAL_CONTAINER_NAMES[@]}"
|
||||
echo "DEBUG containers content: ${CRITICAL_CONTAINER_NAMES[*]}"
|
||||
|
||||
|
||||
[[ "$SHOW_STATUS" == true ]] && show_status && exit 0
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Run
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
@@ -61,23 +58,21 @@ echo "Source: $DIRECTORY"
|
||||
echo "Remote: $REMOTE_SERVER:$DIRECTORY"
|
||||
echo "Profile: $PROFILE_NAME"
|
||||
echo ""
|
||||
|
||||
wait_for_rsync
|
||||
|
||||
|
||||
stop_containers
|
||||
|
||||
|
||||
get_rsync_opts
|
||||
|
||||
|
||||
# Append excludes
|
||||
for ex in "${EXCLUDE_DIRS[@]}"; do
|
||||
[[ -n "$ex" ]] && RSYNC_OPTS+=(--exclude="$ex")
|
||||
done
|
||||
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && RSYNC_OPTS+=("--dry-run") && warn "DRY RUN — no changes will be made"
|
||||
|
||||
|
||||
START=$(date +%s)
|
||||
RSYNC_SUCCESS=false
|
||||
|
||||
|
||||
for i in $(seq 1 "$RETRY_COUNT"); do
|
||||
if rsync "${RSYNC_OPTS[@]}" \
|
||||
-e "ssh -i $SSH_KEY -T -o Compression=no -o IPQoS=throughput" \
|
||||
@@ -90,11 +85,11 @@ for i in $(seq 1 "$RETRY_COUNT"); do
|
||||
[[ "$i" -lt "$RETRY_COUNT" ]] && info "Retrying in ${SLEEP}s..." && sleep "$SLEEP"
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
start_containers
|
||||
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
|
||||
echo ""
|
||||
echo "===== SUMMARY ====="
|
||||
echo "Directory: $DIRECTORY"
|
||||
@@ -106,6 +101,6 @@ else
|
||||
echo "Status: $ICON_ERROR FAILED after $RETRY_COUNT attempts"
|
||||
fi
|
||||
echo "==================="
|
||||
|
||||
|
||||
[[ "$RSYNC_SUCCESS" == false ]] && exit 1
|
||||
exit 0
|
||||
exit 0
|
||||
@@ -191,30 +191,6 @@ start_containers() {
|
||||
done
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# RSYNC SLOT LIMITER
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
wait_for_rsync() {
|
||||
: "${RETRY_COUNT:=5}"
|
||||
: "${MAX_RSYNC_PROCS:=2}"
|
||||
: "${SLEEP:=2}"
|
||||
|
||||
for i in $(seq 1 "$RETRY_COUNT"); do
|
||||
COUNT=$(ps -eo cmd | grep "[r]sync .*${REMOTE_SERVER}" | wc -l || true)
|
||||
|
||||
if [[ "$COUNT" -ge "$MAX_RSYNC_PROCS" ]]; then
|
||||
warn "$ICON_RETRY Waiting for rsync slot ($COUNT/$MAX_RSYNC_PROCS) — attempt $i/$RETRY_COUNT"
|
||||
sleep "$SLEEP"
|
||||
else
|
||||
log "Rsync slot available ($COUNT/$MAX_RSYNC_PROCS)"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
error "Too many rsync processes after $RETRY_COUNT attempts, aborting."
|
||||
exit 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# RSYNC OPTIONS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user