#!/bin/bash # ----------------------------------------------------------------------------------------------- # ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1.5) --------------------------- # ----------------------------------------------------------------------------------------------- # Changelog: # v1.0 — Initial stable framework # v1.1 — format_duration moved here from daily_sync.sh for shared use # SSH_KEY collision resolved — gitea key renamed GITEA_SSH_KEY in Master.conf # Version and changelog tracking added # v1.2 — Consistent function header comment blocks across all functions # check_connectivity added as standalone function # check_connectivity friendlier error output with tailscale hint # v1.3 — check_remote_rootfs added — aborts if remote rootfs exceeds ROOTFS_WARN threshold # check_remote_share added — aborts if target directory is missing or empty on remote # Both protect against rsync running when remote array is down or drives are missing # v1.4 — check_remote_disks added — verifies all physical disks backing a share are mounted # Discovers disk layout automatically at runtime, no configuration required # Aborts if any single disk backing the share is offline or unmounted # v1.5 — Full icon set expanded — each operation and state has its own distinct icon # All function output updated to use correct icon per context # Icons grouped and commented by category for clarity # ----------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------- # ICONS # Each icon has one job — do not reuse across different contexts. # ----------------------------------------------------------------------------------------------- # System / Host ICON_HOST="🖥️" # host detection ICON_NET="🌐" # network / IP resolution ICON_PING="📡" # connectivity check ICON_GEAR="⚙️" # setup section header / profile load # Health Checks ICON_DISK="💾" # disk checks ICON_HEALTH="🩺" # rootfs / share health checks ICON_SHIELD="🛡️" # pre-flight section header # Containers ICON_STOP="⛔" # stop command being issued ICON_STOPPED="🔴" # container confirmed stopped ICON_START="▶️" # start command being issued ICON_STARTED="💚" # container confirmed started ICON_RUNNING="🟢" # container already running when checked ICON_NOT_RUNNING="🔴" # container already stopped when checked # Transfer ICON_SYNC="🔄" # transfer section header ICON_RUN="🚀" # sync starting / rsync attempt ICON_RETRY="🔁" # retry attempt ICON_DONE="🏁" # transfer complete # Summary ICON_SUMMARY="📋" # summary section header ICON_TIME="⏱️" # duration line # Output ICON_INFO="ℹ️" ICON_WARN="⚠️" ICON_ERROR="❌" ICON_SUCCESS="✅" # ----------------------------------------------------------------------------------------------- # OUTPUT HELPERS # Standardised output functions used across all scripts. # log() is gated by ENABLE_LOGGING — set in Master.conf or via --log flag. # ----------------------------------------------------------------------------------------------- info() { echo "$ICON_INFO [INFO] $*"; } warn() { echo "$ICON_WARN [WARN] $*"; } error() { echo "$ICON_ERROR [ERROR] $*"; } success() { echo "$ICON_SUCCESS [OK] $*"; } log() { [[ "${ENABLE_LOGGING:-false}" == true ]] && echo "[LOG] $*" } # ----------------------------------------------------------------------------------------------- # DURATION FORMATTER # Converts raw seconds into a human readable string — e.g. 10m53s or 47s # Used by rsync.sh summary and daily_sync.sh summary. # Sourced from common.sh so both scripts share the same implementation. # ----------------------------------------------------------------------------------------------- format_duration() { local secs=$1 local mins=$((secs / 60)) local rem=$((secs % 60)) [[ $mins -gt 0 ]] && echo "${mins}m${rem}s" || echo "${rem}s" } # ----------------------------------------------------------------------------------------------- # ARG PARSER # Processes all flags and key=value pairs passed to any script. # Positional arguments (directory paths) are separated before calling this — see rsync.sh. # Supported flags: --dry-run, --log, --no-log, --status, --help # Supported key=value: LOG=true/false, or any declared variable e.g. BW_LIMIT=5000 # Unparsed positional args are returned in PARSED_ARGS array. # ----------------------------------------------------------------------------------------------- parse_args() { ENABLE_LOGGING=${ENABLE_LOGGING:-false} DRY_RUN=${DRY_RUN:-false} SHOW_STATUS=${SHOW_STATUS:-false} CLEAN_ARGS=() for ARG in "$@"; do if [[ "$ARG" == *=* ]]; then VAR="${ARG%%=*}" VAL="${ARG#*=}" case "$VAR" in LOG) [[ "$VAL" == "true" ]] && ENABLE_LOGGING=true [[ "$VAL" == "false" ]] && ENABLE_LOGGING=false ;; *) if declare -p "$VAR" &>/dev/null; then printf -v "$VAR" '%s' "$VAL" log "Set $VAR=$VAL" else warn "Unknown variable: $VAR" fi ;; esac else case "$ARG" in --dry-run|-n) DRY_RUN=true ;; --log) ENABLE_LOGGING=true ;; --no-log) ENABLE_LOGGING=false ;; --status|--summary) SHOW_STATUS=true ;; --help|-h) echo "Usage: script [--dry-run] [--log] [--status]" exit 0 ;; *) CLEAN_ARGS+=("$ARG") ;; esac fi done PARSED_ARGS=("${CLEAN_ARGS[@]}") } # ----------------------------------------------------------------------------------------------- # VALIDATION # Checks that a required variable is set and non-empty. # Usage: require_var VAR_NAME # Exits with error if the variable is missing. # ----------------------------------------------------------------------------------------------- require_var() { [[ -z "${!1:-}" ]] && error "Missing required: $1" && exit 1 } # ----------------------------------------------------------------------------------------------- # HOST DETECTION # Determines local and remote server names by comparing hostname against HOST1/HOST2. # Sets LOCAL_SERVER_NAME, REMOTE_SERVER_NAME, and SSH_KEY for the current run direction. # Both HOST1 and HOST2 must be defined in Master.conf. # ----------------------------------------------------------------------------------------------- detect_hosts() { LOCAL_HOSTNAME="$(hostname)" if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then LOCAL_SERVER_NAME="$HOST1" REMOTE_SERVER_NAME="$HOST2" elif [[ "$LOCAL_HOSTNAME" == "$HOST2" ]]; then LOCAL_SERVER_NAME="$HOST2" REMOTE_SERVER_NAME="$HOST1" else error "Unknown host: $LOCAL_HOSTNAME" exit 1 fi declare -A SSH_KEYS SSH_KEYS["$HOST1|$HOST2"]="$HOST1_SSH_KEY" SSH_KEYS["$HOST2|$HOST1"]="$HOST2_SSH_KEY" SSH_KEY="${SSH_KEYS[$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME]}" [[ -z "$SSH_KEY" ]] && error "Missing SSH key mapping for $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" && exit 1 info "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" } # ----------------------------------------------------------------------------------------------- # REMOTE IP RESOLUTION # Resolves the Tailscale IPv4 address of the remote server. # Sets REMOTE_SERVER used by all subsequent SSH and rsync calls. # Exits if resolution fails — likely means Tailscale is down or peer is offline. # ----------------------------------------------------------------------------------------------- resolve_remote_ip() { log "Resolving remote IP for $REMOTE_SERVER_NAME..." REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) [[ -z "$REMOTE_SERVER" ]] && error "Failed to resolve Tailscale IP for $REMOTE_SERVER_NAME" && exit 1 info "$ICON_NET Remote IP: $REMOTE_SERVER" } # ----------------------------------------------------------------------------------------------- # CONNECTIVITY CHECK # Pings the remote server to confirm it is reachable before starting any transfers. # Prevents the rsync retry loop from burning all attempts against an unreachable host. # If ping fails, prints a tailscale status hint to aid diagnosis before exiting. # ----------------------------------------------------------------------------------------------- check_connectivity() { log "Checking connectivity to $REMOTE_SERVER..." if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then error "$ICON_PING Remote $REMOTE_SERVER ($REMOTE_SERVER_NAME) is unreachable" info "Hint: tailscale status | grep $REMOTE_SERVER_NAME" exit 1 fi info "$ICON_PING $REMOTE_SERVER_NAME is reachable" } # ----------------------------------------------------------------------------------------------- # REMOTE ROOTFS SPACE CHECK # Checks the remote server's rootfs usage before any rsync runs. # If the array is down or drives are missing, rsync writes land on rootfs instead of the array — # this can fill the remote filesystem rapidly and crash the server. # Threshold is set by ROOTFS_WARN in Master.conf (recommended: 75). # Aborts cleanly with a clear error showing current usage vs threshold. # ----------------------------------------------------------------------------------------------- check_remote_rootfs() { log "Checking remote rootfs usage..." REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null) if [[ -z "$REMOTE_USAGE" ]]; then error "Could not retrieve rootfs usage from $REMOTE_SERVER_NAME" exit 1 fi if [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then echo "" error "$ICON_HEALTH Remote rootfs is ${REMOTE_USAGE}% full — threshold is ${ROOTFS_WARN:-75}%" warn "Array may be down or drives missing on $REMOTE_SERVER_NAME" info "Hint: Check array status on $REMOTE_SERVER_NAME before retrying" echo "" exit 1 fi info "$ICON_HEALTH Remote rootfs: ${REMOTE_USAGE}% used (threshold: ${ROOTFS_WARN:-75}%)" } # ----------------------------------------------------------------------------------------------- # REMOTE SHARE VALIDATION # Verifies that the target directory exists and is not empty on the remote server. # Catches the scenario where the array is mounted but drives are not backing the share — # the path exists as an empty mountpoint, which would cause --delete to wipe the remote. # Called with the specific directory being synced so each share is checked individually. # Usage: check_remote_share "/mnt/user/Movies" # ----------------------------------------------------------------------------------------------- check_remote_share() { local dir="$1" log "Checking remote share: $dir..." SHARE_EXISTS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null) if [[ "$SHARE_EXISTS" != "yes" ]]; then echo "" error "$ICON_HEALTH Remote share does not exist: $dir" warn "Array may not be started or share is not configured on $REMOTE_SERVER_NAME" info "Hint: Check shares and array status on $REMOTE_SERVER_NAME before retrying" echo "" exit 1 fi SHARE_EMPTY=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null) if [[ "$SHARE_EMPTY" == "yes" ]]; then echo "" warn "$ICON_HEALTH Remote share exists but is empty: $dir" warn "Drives may not be mounted on $REMOTE_SERVER_NAME — aborting to protect data" info "Hint: Verify array and drive assignments on $REMOTE_SERVER_NAME before retrying" echo "" exit 1 fi info "$ICON_HEALTH Remote share verified: $dir" } # ----------------------------------------------------------------------------------------------- # REMOTE DISK CHECK # Verifies that all physical disks backing a share are online and mounted on the remote server. # Discovers disk layout automatically at runtime by finding all /mnt/diskN/sharename paths — # no configuration required, works for any share regardless of how many disks it spans. # Aborts if any single disk backing the share is offline — partial disk failure means # incomplete data which could result in files being deleted by --delete during sync. # Usage: check_remote_disks "/mnt/user/Movies" # ----------------------------------------------------------------------------------------------- check_remote_disks() { local dir="$1" local share_name share_name=$(basename "$dir") info "$ICON_DISK Checking disks backing $share_name on $REMOTE_SERVER_NAME..." # Find all /mnt/diskN paths that contain this share on the remote DISK_PATHS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "ls -d /mnt/disk*/$share_name 2>/dev/null" 2>/dev/null) if [[ -z "$DISK_PATHS" ]]; then echo "" error "$ICON_DISK No disks found backing share $share_name on $REMOTE_SERVER_NAME" warn "Share may not exist on any disk or array may not be started" info "Hint: Check array and share configuration on $REMOTE_SERVER_NAME" echo "" exit 1 fi local all_ok=true while IFS= read -r disk_share_path; do local disk_mount disk_mount=$(dirname "$disk_share_path") local disk_name disk_name=$(basename "$disk_mount") MOUNTED=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "mountpoint -q '$disk_mount' && echo yes || echo no" 2>/dev/null) if [[ "$MOUNTED" == "yes" ]]; then info "$ICON_DISK $disk_name $ICON_RUNNING — $share_name present" else error "$ICON_DISK $disk_name $ICON_STOPPED — $share_name missing or incomplete" all_ok=false fi done <<< "$DISK_PATHS" if [[ "$all_ok" == false ]]; then echo "" error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME" warn "Aborting to prevent partial or destructive sync" info "Hint: Check disk assignments and array status on $REMOTE_SERVER_NAME before retrying" echo "" exit 1 fi success "All disks backing $share_name are online" } # ----------------------------------------------------------------------------------------------- # CONTAINER MANAGEMENT — STOP # Stops all containers listed in CRITICAL_CONTAINER_NAMES on the remote server. # Only stops containers that are currently running — skips those already stopped. # Tracks stopped containers in RUNNING_CONTAINERS for restart after rsync completes. # CRITICAL_CONTAINER_NAMES must be a bash array — rsync.sh handles conversion from profile strings. # ----------------------------------------------------------------------------------------------- RUNNING_CONTAINERS=() stop_containers() { if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \ [[ "${CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then log "No containers configured for this profile, skipping stop." return fi info "Stopping containers..." RUNNING_CONTAINERS=() for c in "${CRITICAL_CONTAINER_NAMES[@]}"; do [[ -z "$c" ]] && continue info "Checking $c..." STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "docker inspect -f '{{.State.Running}}' $c 2>/dev/null" 2>/dev/null || echo "false") if [[ "$STATUS" == "true" ]]; then echo "$ICON_STOP Stopping $c..." RUNNING_CONTAINERS+=("$c") if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $c" >/dev/null; then echo "$ICON_STOPPED $c stopped" else error "Failed to stop $c" fi else echo "$ICON_NOT_RUNNING $c is not running, skipping" fi done } # ----------------------------------------------------------------------------------------------- # CONTAINER MANAGEMENT — START # Restarts only the containers that were running before rsync and were stopped by stop_containers. # Containers listed in DELAYED_CONTAINERS receive a sleep of CONTAINER_DELAY seconds before # starting — useful for dependencies like Authelia that need upstream services ready first. # DELAYED_CONTAINERS must be a bash array — rsync.sh handles conversion from profile strings. # ----------------------------------------------------------------------------------------------- start_containers() { if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then log "No containers to restart." return fi info "Starting containers..." for c in "${RUNNING_CONTAINERS[@]}"; do [[ -z "$c" ]] && continue local needs_delay=false for d in "${DELAYED_CONTAINERS[@]}"; do if [[ "$c" == "$d" ]]; then needs_delay=true break fi done if [[ "$needs_delay" == true ]]; then info "Waiting ${CONTAINER_DELAY}s before starting $c..." sleep "$CONTAINER_DELAY" fi echo "$ICON_START Starting $c..." if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $c" >/dev/null; then echo "$ICON_STARTED $c started" else error "Failed to start $c" fi done } # ----------------------------------------------------------------------------------------------- # RSYNC OPTIONS # Loads rsync options for the current profile from PROFILE_RSYNC_OPTS in Master.conf. # If no profile match is found, falls back to DEFAULT_RSYNC_OPTS. # Note: profile opts do NOT inherit from defaults — all desired flags must be listed explicitly. # Sets RSYNC_OPTS array used directly in the rsync call in rsync.sh. # ----------------------------------------------------------------------------------------------- get_rsync_opts() { if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then read -r -a RSYNC_OPTS <<< "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]}" log "Using profile rsync opts for $PROFILE_NAME: ${RSYNC_OPTS[*]}" else RSYNC_OPTS=("${DEFAULT_RSYNC_OPTS[@]}") log "Using default rsync opts: ${RSYNC_OPTS[*]}" fi } # ----------------------------------------------------------------------------------------------- # STATUS DISPLAY # Prints a summary of the current runtime configuration. # Triggered by --status or --summary flag passed to rsync.sh. # Useful for verifying profile resolution and variable state before a live run. # ----------------------------------------------------------------------------------------------- show_status() { echo "===== $ICON_SUMMARY STATUS =====" echo "Local: $LOCAL_SERVER_NAME" echo "Remote: $REMOTE_SERVER_NAME" echo "IP: $REMOTE_SERVER" echo "Profile: $PROFILE_NAME" echo "DryRun: $DRY_RUN" echo "Logging: $ENABLE_LOGGING" echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]}" echo "Delayed: ${DELAYED_CONTAINERS[*]}" echo "Excludes: ${EXCLUDE_DIRS[*]}" echo "========================" }