465 lines
20 KiB
Bash
465 lines
20 KiB
Bash
#!/bin/bash
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ----------------------------
|
||
# -----------------------------------------------------------------------------------------------
|
||
# Version: 1.5
|
||
# -----------------------------------------------------------------------------------------------
|
||
# Changelog:
|
||
# v1.0 — Initial stable framework
|
||
# v1.1 — format_duration moved here from daily_sync.sh for shared use
|
||
# SSH_KEY collision resolved — gitea key renamed GITEA_SSH_KEY in Master.conf
|
||
# Version and changelog tracking added
|
||
# v1.2 — Consistent function header comment blocks across all functions
|
||
# check_connectivity added as standalone function
|
||
# check_connectivity friendlier error output with tailscale hint
|
||
# v1.3 — check_remote_rootfs added — aborts if remote rootfs exceeds ROOTFS_WARN threshold
|
||
# check_remote_share added — aborts if target directory is missing or empty on remote
|
||
# Both protect against rsync running when remote array is down or drives are missing
|
||
# v1.4 — check_remote_disks added — verifies all physical disks backing a share are mounted
|
||
# Discovers disk layout automatically at runtime, no configuration required
|
||
# Aborts if any single disk backing the share is offline or unmounted
|
||
# v1.5 — Full icon set expanded — each operation and state has its own distinct icon
|
||
# All function output updated to use correct icon per context
|
||
# Icons grouped and commented by category for clarity
|
||
# -----------------------------------------------------------------------------------------------
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ICONS
|
||
# Each icon has one job — do not reuse across different contexts.
|
||
# -----------------------------------------------------------------------------------------------
|
||
|
||
# System / Host
|
||
ICON_HOST="🖥️" # host detection
|
||
ICON_NET="🌐" # network / IP resolution
|
||
ICON_PING="📡" # connectivity check
|
||
ICON_GEAR="⚙️" # setup section header / profile load
|
||
|
||
# Health Checks
|
||
ICON_DISK="💾" # disk checks
|
||
ICON_HEALTH="🩺" # rootfs / share health checks
|
||
ICON_SHIELD="🛡️" # pre-flight section header
|
||
|
||
# Containers
|
||
ICON_STOP="⛔" # stop command being issued
|
||
ICON_STOPPED="🔴" # container confirmed stopped
|
||
ICON_START="▶️" # start command being issued
|
||
ICON_STARTED="💚" # container confirmed started
|
||
ICON_RUNNING="🟢" # container already running when checked
|
||
ICON_NOT_RUNNING="🔴" # container already stopped when checked
|
||
|
||
# Transfer
|
||
ICON_SYNC="🔄" # transfer section header
|
||
ICON_RUN="🚀" # sync starting / rsync attempt
|
||
ICON_RETRY="🔁" # retry attempt
|
||
ICON_DONE="🏁" # transfer complete
|
||
|
||
# Summary
|
||
ICON_SUMMARY="📋" # summary section header
|
||
ICON_TIME="⏱️" # duration line
|
||
|
||
# Output
|
||
ICON_INFO="ℹ️"
|
||
ICON_WARN="⚠️"
|
||
ICON_ERROR="❌"
|
||
ICON_SUCCESS="✅"
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# OUTPUT HELPERS
|
||
# Standardised output functions used across all scripts.
|
||
# log() is gated by ENABLE_LOGGING — set in Master.conf or via --log flag.
|
||
# -----------------------------------------------------------------------------------------------
|
||
info() { echo "$ICON_INFO [INFO] $*"; }
|
||
warn() { echo "$ICON_WARN [WARN] $*"; }
|
||
error() { echo "$ICON_ERROR [ERROR] $*"; }
|
||
success() { echo "$ICON_SUCCESS [OK] $*"; }
|
||
|
||
log() {
|
||
[[ "${ENABLE_LOGGING:-false}" == true ]] && echo "[LOG] $*"
|
||
}
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# DURATION FORMATTER
|
||
# Converts raw seconds into a human readable string — e.g. 10m53s or 47s
|
||
# Used by rsync.sh summary and daily_sync.sh summary.
|
||
# Sourced from common.sh so both scripts share the same implementation.
|
||
# -----------------------------------------------------------------------------------------------
|
||
format_duration() {
|
||
local secs=$1
|
||
local mins=$((secs / 60))
|
||
local rem=$((secs % 60))
|
||
[[ $mins -gt 0 ]] && echo "${mins}m${rem}s" || echo "${rem}s"
|
||
}
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ARG PARSER
|
||
# Processes all flags and key=value pairs passed to any script.
|
||
# Positional arguments (directory paths) are separated before calling this — see rsync.sh.
|
||
# Supported flags: --dry-run, --log, --no-log, --status, --help
|
||
# Supported key=value: LOG=true/false, or any declared variable e.g. BW_LIMIT=5000
|
||
# Unparsed positional args are returned in PARSED_ARGS array.
|
||
# -----------------------------------------------------------------------------------------------
|
||
parse_args() {
|
||
ENABLE_LOGGING=${ENABLE_LOGGING:-false}
|
||
DRY_RUN=${DRY_RUN:-false}
|
||
SHOW_STATUS=${SHOW_STATUS:-false}
|
||
|
||
CLEAN_ARGS=()
|
||
|
||
for ARG in "$@"; do
|
||
if [[ "$ARG" == *=* ]]; then
|
||
VAR="${ARG%%=*}"
|
||
VAL="${ARG#*=}"
|
||
|
||
case "$VAR" in
|
||
LOG)
|
||
[[ "$VAL" == "true" ]] && ENABLE_LOGGING=true
|
||
[[ "$VAL" == "false" ]] && ENABLE_LOGGING=false
|
||
;;
|
||
*)
|
||
if declare -p "$VAR" &>/dev/null; then
|
||
printf -v "$VAR" '%s' "$VAL"
|
||
log "Set $VAR=$VAL"
|
||
else
|
||
warn "Unknown variable: $VAR"
|
||
fi
|
||
;;
|
||
esac
|
||
else
|
||
case "$ARG" in
|
||
--dry-run|-n) DRY_RUN=true ;;
|
||
--log) ENABLE_LOGGING=true ;;
|
||
--no-log) ENABLE_LOGGING=false ;;
|
||
--status|--summary) SHOW_STATUS=true ;;
|
||
--help|-h)
|
||
echo "Usage: script <dir> [--dry-run] [--log] [--status]"
|
||
exit 0
|
||
;;
|
||
*) CLEAN_ARGS+=("$ARG") ;;
|
||
esac
|
||
fi
|
||
done
|
||
|
||
PARSED_ARGS=("${CLEAN_ARGS[@]}")
|
||
}
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# VALIDATION
|
||
# Checks that a required variable is set and non-empty.
|
||
# Usage: require_var VAR_NAME
|
||
# Exits with error if the variable is missing.
|
||
# -----------------------------------------------------------------------------------------------
|
||
require_var() {
|
||
[[ -z "${!1:-}" ]] && error "Missing required: $1" && exit 1
|
||
}
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# HOST DETECTION
|
||
# Determines local and remote server names by comparing hostname against HOST1/HOST2.
|
||
# Sets LOCAL_SERVER_NAME, REMOTE_SERVER_NAME, and SSH_KEY for the current run direction.
|
||
# Both HOST1 and HOST2 must be defined in Master.conf.
|
||
# -----------------------------------------------------------------------------------------------
|
||
detect_hosts() {
|
||
LOCAL_HOSTNAME="$(hostname)"
|
||
|
||
if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then
|
||
LOCAL_SERVER_NAME="$HOST1"
|
||
REMOTE_SERVER_NAME="$HOST2"
|
||
elif [[ "$LOCAL_HOSTNAME" == "$HOST2" ]]; then
|
||
LOCAL_SERVER_NAME="$HOST2"
|
||
REMOTE_SERVER_NAME="$HOST1"
|
||
else
|
||
error "Unknown host: $LOCAL_HOSTNAME"
|
||
exit 1
|
||
fi
|
||
|
||
declare -A SSH_KEYS
|
||
SSH_KEYS["$HOST1|$HOST2"]="$HOST1_SSH_KEY"
|
||
SSH_KEYS["$HOST2|$HOST1"]="$HOST2_SSH_KEY"
|
||
|
||
SSH_KEY="${SSH_KEYS[$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME]}"
|
||
|
||
[[ -z "$SSH_KEY" ]] && error "Missing SSH key mapping for $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" && exit 1
|
||
|
||
info "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME"
|
||
}
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# REMOTE IP RESOLUTION
|
||
# Resolves the Tailscale IPv4 address of the remote server.
|
||
# Sets REMOTE_SERVER used by all subsequent SSH and rsync calls.
|
||
# Exits if resolution fails — likely means Tailscale is down or peer is offline.
|
||
# -----------------------------------------------------------------------------------------------
|
||
resolve_remote_ip() {
|
||
log "Resolving remote IP for $REMOTE_SERVER_NAME..."
|
||
REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)
|
||
|
||
[[ -z "$REMOTE_SERVER" ]] && error "Failed to resolve Tailscale IP for $REMOTE_SERVER_NAME" && exit 1
|
||
|
||
info "$ICON_NET Remote IP: $REMOTE_SERVER"
|
||
}
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# CONNECTIVITY CHECK
|
||
# Pings the remote server to confirm it is reachable before starting any transfers.
|
||
# Prevents the rsync retry loop from burning all attempts against an unreachable host.
|
||
# If ping fails, prints a tailscale status hint to aid diagnosis before exiting.
|
||
# -----------------------------------------------------------------------------------------------
|
||
check_connectivity() {
|
||
log "Checking connectivity to $REMOTE_SERVER..."
|
||
if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then
|
||
error "$ICON_PING Remote $REMOTE_SERVER ($REMOTE_SERVER_NAME) is unreachable"
|
||
info "Hint: tailscale status | grep $REMOTE_SERVER_NAME"
|
||
exit 1
|
||
fi
|
||
info "$ICON_PING $REMOTE_SERVER_NAME is reachable"
|
||
}
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# REMOTE ROOTFS SPACE CHECK
|
||
# Checks the remote server's rootfs usage before any rsync runs.
|
||
# If the array is down or drives are missing, rsync writes land on rootfs instead of the array —
|
||
# this can fill the remote filesystem rapidly and crash the server.
|
||
# Threshold is set by ROOTFS_WARN in Master.conf (recommended: 75).
|
||
# Aborts cleanly with a clear error showing current usage vs threshold.
|
||
# -----------------------------------------------------------------------------------------------
|
||
check_remote_rootfs() {
|
||
log "Checking remote rootfs usage..."
|
||
|
||
REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null)
|
||
|
||
if [[ -z "$REMOTE_USAGE" ]]; then
|
||
error "Could not retrieve rootfs usage from $REMOTE_SERVER_NAME"
|
||
exit 1
|
||
fi
|
||
|
||
if [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then
|
||
echo ""
|
||
error "$ICON_HEALTH Remote rootfs is ${REMOTE_USAGE}% full — threshold is ${ROOTFS_WARN:-75}%"
|
||
warn "Array may be down or drives missing on $REMOTE_SERVER_NAME"
|
||
info "Hint: Check array status on $REMOTE_SERVER_NAME before retrying"
|
||
echo ""
|
||
exit 1
|
||
fi
|
||
|
||
info "$ICON_HEALTH Remote rootfs: ${REMOTE_USAGE}% used (threshold: ${ROOTFS_WARN:-75}%)"
|
||
}
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# REMOTE SHARE VALIDATION
|
||
# Verifies that the target directory exists and is not empty on the remote server.
|
||
# Catches the scenario where the array is mounted but drives are not backing the share —
|
||
# the path exists as an empty mountpoint, which would cause --delete to wipe the remote.
|
||
# Called with the specific directory being synced so each share is checked individually.
|
||
# Usage: check_remote_share "/mnt/user/Movies"
|
||
# -----------------------------------------------------------------------------------------------
|
||
check_remote_share() {
|
||
local dir="$1"
|
||
|
||
log "Checking remote share: $dir..."
|
||
|
||
SHARE_EXISTS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null)
|
||
|
||
if [[ "$SHARE_EXISTS" != "yes" ]]; then
|
||
echo ""
|
||
error "$ICON_HEALTH Remote share does not exist: $dir"
|
||
warn "Array may not be started or share is not configured on $REMOTE_SERVER_NAME"
|
||
info "Hint: Check shares and array status on $REMOTE_SERVER_NAME before retrying"
|
||
echo ""
|
||
exit 1
|
||
fi
|
||
|
||
SHARE_EMPTY=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null)
|
||
|
||
if [[ "$SHARE_EMPTY" == "yes" ]]; then
|
||
echo ""
|
||
warn "$ICON_HEALTH Remote share exists but is empty: $dir"
|
||
warn "Drives may not be mounted on $REMOTE_SERVER_NAME — aborting to protect data"
|
||
info "Hint: Verify array and drive assignments on $REMOTE_SERVER_NAME before retrying"
|
||
echo ""
|
||
exit 1
|
||
fi
|
||
|
||
info "$ICON_HEALTH Remote share verified: $dir"
|
||
}
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# REMOTE DISK CHECK
|
||
# Verifies that all physical disks backing a share are online and mounted on the remote server.
|
||
# Discovers disk layout automatically at runtime by finding all /mnt/diskN/sharename paths —
|
||
# no configuration required, works for any share regardless of how many disks it spans.
|
||
# Aborts if any single disk backing the share is offline — partial disk failure means
|
||
# incomplete data which could result in files being deleted by --delete during sync.
|
||
# Usage: check_remote_disks "/mnt/user/Movies"
|
||
# -----------------------------------------------------------------------------------------------
|
||
check_remote_disks() {
|
||
local dir="$1"
|
||
local share_name
|
||
share_name=$(basename "$dir")
|
||
|
||
info "$ICON_DISK Checking disks backing $share_name on $REMOTE_SERVER_NAME..."
|
||
|
||
# Find all /mnt/diskN paths that contain this share on the remote
|
||
DISK_PATHS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"ls -d /mnt/disk*/$share_name 2>/dev/null" 2>/dev/null)
|
||
|
||
if [[ -z "$DISK_PATHS" ]]; then
|
||
echo ""
|
||
error "$ICON_DISK No disks found backing share $share_name on $REMOTE_SERVER_NAME"
|
||
warn "Share may not exist on any disk or array may not be started"
|
||
info "Hint: Check array and share configuration on $REMOTE_SERVER_NAME"
|
||
echo ""
|
||
exit 1
|
||
fi
|
||
|
||
local all_ok=true
|
||
|
||
while IFS= read -r disk_share_path; do
|
||
local disk_mount
|
||
disk_mount=$(dirname "$disk_share_path")
|
||
local disk_name
|
||
disk_name=$(basename "$disk_mount")
|
||
|
||
MOUNTED=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"mountpoint -q '$disk_mount' && echo yes || echo no" 2>/dev/null)
|
||
|
||
if [[ "$MOUNTED" == "yes" ]]; then
|
||
info "$ICON_DISK $disk_name $ICON_RUNNING — $share_name present"
|
||
else
|
||
error "$ICON_DISK $disk_name $ICON_STOPPED — $share_name missing or incomplete"
|
||
all_ok=false
|
||
fi
|
||
done <<< "$DISK_PATHS"
|
||
|
||
if [[ "$all_ok" == false ]]; then
|
||
echo ""
|
||
error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME"
|
||
warn "Aborting to prevent partial or destructive sync"
|
||
info "Hint: Check disk assignments and array status on $REMOTE_SERVER_NAME before retrying"
|
||
echo ""
|
||
exit 1
|
||
fi
|
||
|
||
success "All disks backing $share_name are online"
|
||
}
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# CONTAINER MANAGEMENT — STOP
|
||
# Stops all containers listed in CRITICAL_CONTAINER_NAMES on the remote server.
|
||
# Only stops containers that are currently running — skips those already stopped.
|
||
# Tracks stopped containers in RUNNING_CONTAINERS for restart after rsync completes.
|
||
# CRITICAL_CONTAINER_NAMES must be a bash array — rsync.sh handles conversion from profile strings.
|
||
# -----------------------------------------------------------------------------------------------
|
||
RUNNING_CONTAINERS=()
|
||
|
||
stop_containers() {
|
||
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \
|
||
[[ "${CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then
|
||
log "No containers configured for this profile, skipping stop."
|
||
return
|
||
fi
|
||
|
||
info "Stopping containers..."
|
||
RUNNING_CONTAINERS=()
|
||
|
||
for c in "${CRITICAL_CONTAINER_NAMES[@]}"; do
|
||
[[ -z "$c" ]] && continue
|
||
|
||
info "Checking $c..."
|
||
|
||
STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"docker inspect -f '{{.State.Running}}' $c 2>/dev/null" 2>/dev/null || echo "false")
|
||
|
||
if [[ "$STATUS" == "true" ]]; then
|
||
echo "$ICON_STOP Stopping $c..."
|
||
RUNNING_CONTAINERS+=("$c")
|
||
|
||
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $c" >/dev/null; then
|
||
echo "$ICON_STOPPED $c stopped"
|
||
else
|
||
error "Failed to stop $c"
|
||
fi
|
||
else
|
||
echo "$ICON_NOT_RUNNING $c is not running, skipping"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# CONTAINER MANAGEMENT — START
|
||
# Restarts only the containers that were running before rsync and were stopped by stop_containers.
|
||
# Containers listed in DELAYED_CONTAINERS receive a sleep of CONTAINER_DELAY seconds before
|
||
# starting — useful for dependencies like Authelia that need upstream services ready first.
|
||
# DELAYED_CONTAINERS must be a bash array — rsync.sh handles conversion from profile strings.
|
||
# -----------------------------------------------------------------------------------------------
|
||
start_containers() {
|
||
if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then
|
||
log "No containers to restart."
|
||
return
|
||
fi
|
||
|
||
info "Starting containers..."
|
||
|
||
for c in "${RUNNING_CONTAINERS[@]}"; do
|
||
[[ -z "$c" ]] && continue
|
||
|
||
local needs_delay=false
|
||
for d in "${DELAYED_CONTAINERS[@]}"; do
|
||
if [[ "$c" == "$d" ]]; then
|
||
needs_delay=true
|
||
break
|
||
fi
|
||
done
|
||
|
||
if [[ "$needs_delay" == true ]]; then
|
||
info "Waiting ${CONTAINER_DELAY}s before starting $c..."
|
||
sleep "$CONTAINER_DELAY"
|
||
fi
|
||
|
||
echo "$ICON_START Starting $c..."
|
||
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $c" >/dev/null; then
|
||
echo "$ICON_STARTED $c started"
|
||
else
|
||
error "Failed to start $c"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# RSYNC OPTIONS
|
||
# Loads rsync options for the current profile from PROFILE_RSYNC_OPTS in Master.conf.
|
||
# If no profile match is found, falls back to DEFAULT_RSYNC_OPTS.
|
||
# Note: profile opts do NOT inherit from defaults — all desired flags must be listed explicitly.
|
||
# Sets RSYNC_OPTS array used directly in the rsync call in rsync.sh.
|
||
# -----------------------------------------------------------------------------------------------
|
||
get_rsync_opts() {
|
||
if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then
|
||
read -r -a RSYNC_OPTS <<< "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]}"
|
||
log "Using profile rsync opts for $PROFILE_NAME: ${RSYNC_OPTS[*]}"
|
||
else
|
||
RSYNC_OPTS=("${DEFAULT_RSYNC_OPTS[@]}")
|
||
log "Using default rsync opts: ${RSYNC_OPTS[*]}"
|
||
fi
|
||
}
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# STATUS DISPLAY
|
||
# Prints a summary of the current runtime configuration.
|
||
# Triggered by --status or --summary flag passed to rsync.sh.
|
||
# Useful for verifying profile resolution and variable state before a live run.
|
||
# -----------------------------------------------------------------------------------------------
|
||
show_status() {
|
||
echo "===== $ICON_SUMMARY STATUS ====="
|
||
echo "Local: $LOCAL_SERVER_NAME"
|
||
echo "Remote: $REMOTE_SERVER_NAME"
|
||
echo "IP: $REMOTE_SERVER"
|
||
echo "Profile: $PROFILE_NAME"
|
||
echo "DryRun: $DRY_RUN"
|
||
echo "Logging: $ENABLE_LOGGING"
|
||
echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]}"
|
||
echo "Delayed: ${DELAYED_CONTAINERS[*]}"
|
||
echo "Excludes: ${EXCLUDE_DIRS[*]}"
|
||
echo "========================"
|
||
} |