#!/bin/bash
# -----------------------------------------------------------------------------------------------
# ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ----------------------------
# -----------------------------------------------------------------------------------------------
# Version: 1.2
# -----------------------------------------------------------------------------------------------
# Changelog:
# v1.0 — Initial stable framework
# v1.1 — format_duration moved here from daily_sync.sh for shared use
# SSH_KEY collision resolved — gitea key renamed GITEA_SSH_KEY in Master.conf
# Version and changelog tracking added
# v1.2 — Consistent function header comment blocks across all functions
# check_connectivity friendlier error output with tailscale hint
# check_connectivity added as standalone function
# -----------------------------------------------------------------------------------------------
# ICONS
ICON_INFO="ℹ️"
ICON_WARN="⚠️"
ICON_ERROR="❌"
ICON_SUCCESS="✅"
ICON_RUN="🚀"
ICON_STOP="🛑"
ICON_RETRY="🔁"
ICON_SYNC="🔄"
ICON_GEAR="⚙️"
# -----------------------------------------------------------------------------------------------
# OUTPUT HELPERS
# Standardised output functions used across all scripts.
# log() is gated by ENABLE_LOGGING — set in Master.conf or via --log flag.
# -----------------------------------------------------------------------------------------------
info() { echo "$ICON_INFO [INFO] $*"; }
warn() { echo "$ICON_WARN [WARN] $*"; }
error() { echo "$ICON_ERROR [ERROR] $*"; }
success() { echo "$ICON_SUCCESS [OK] $*"; }
log() {
[[ "${ENABLE_LOGGING:-false}" == true ]] && echo "[LOG] $*"
}
# -----------------------------------------------------------------------------------------------
# DURATION FORMATTER
# Converts raw seconds into a human readable string — e.g. 10m53s or 47s
# Used by rsync.sh summary and daily_sync.sh summary.
# Sourced from common.sh so both scripts share the same implementation.
# -----------------------------------------------------------------------------------------------
format_duration() {
local secs=$1
local mins=$((secs / 60))
local rem=$((secs % 60))
[[ $mins -gt 0 ]] && echo "${mins}m${rem}s" || echo "${rem}s"
}
# -----------------------------------------------------------------------------------------------
# ARG PARSER
# Processes all flags and key=value pairs passed to any script.
# Positional arguments (directory paths) are separated before calling this — see rsync.sh.
# Supported flags: --dry-run, --log, --no-log, --status, --help
# Supported key=value: LOG=true/false, or any declared variable e.g. BW_LIMIT=5000
# Unparsed positional args are returned in PARSED_ARGS array.
# -----------------------------------------------------------------------------------------------
parse_args() {
ENABLE_LOGGING=${ENABLE_LOGGING:-false}
DRY_RUN=${DRY_RUN:-false}
SHOW_STATUS=${SHOW_STATUS:-false}
CLEAN_ARGS=()
for ARG in "$@"; do
if [[ "$ARG" == *=* ]]; then
VAR="${ARG%%=*}"
VAL="${ARG#*=}"
case "$VAR" in
LOG)
[[ "$VAL" == "true" ]] && ENABLE_LOGGING=true
[[ "$VAL" == "false" ]] && ENABLE_LOGGING=false
;;
*)
if declare -p "$VAR" &>/dev/null; then
printf -v "$VAR" '%s' "$VAL"
log "Set $VAR=$VAL"
else
warn "Unknown variable: $VAR"
fi
;;
esac
else
case "$ARG" in
--dry-run|-n) DRY_RUN=true ;;
--log) ENABLE_LOGGING=true ;;
--no-log) ENABLE_LOGGING=false ;;
--status|--summary) SHOW_STATUS=true ;;
--help|-h)
echo "Usage: script
[--dry-run] [--log] [--status]"
exit 0
;;
*) CLEAN_ARGS+=("$ARG") ;;
esac
fi
done
PARSED_ARGS=("${CLEAN_ARGS[@]}")
}
# -----------------------------------------------------------------------------------------------
# VALIDATION
# Checks that a required variable is set and non-empty.
# Usage: require_var VAR_NAME
# Exits with error if the variable is missing.
# -----------------------------------------------------------------------------------------------
require_var() {
[[ -z "${!1:-}" ]] && error "Missing required: $1" && exit 1
}
# -----------------------------------------------------------------------------------------------
# HOST DETECTION
# Determines local and remote server names by comparing hostname against HOST1/HOST2.
# Sets LOCAL_SERVER_NAME, REMOTE_SERVER_NAME, and SSH_KEY for the current run direction.
# Both HOST1 and HOST2 must be defined in Master.conf.
# -----------------------------------------------------------------------------------------------
detect_hosts() {
LOCAL_HOSTNAME="$(hostname)"
if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then
LOCAL_SERVER_NAME="$HOST1"
REMOTE_SERVER_NAME="$HOST2"
elif [[ "$LOCAL_HOSTNAME" == "$HOST2" ]]; then
LOCAL_SERVER_NAME="$HOST2"
REMOTE_SERVER_NAME="$HOST1"
else
error "Unknown host: $LOCAL_HOSTNAME"
exit 1
fi
declare -A SSH_KEYS
SSH_KEYS["$HOST1|$HOST2"]="$HOST1_SSH_KEY"
SSH_KEYS["$HOST2|$HOST1"]="$HOST2_SSH_KEY"
SSH_KEY="${SSH_KEYS[$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME]}"
[[ -z "$SSH_KEY" ]] && error "Missing SSH key mapping for $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" && exit 1
info "$ICON_GEAR Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME"
}
# -----------------------------------------------------------------------------------------------
# REMOTE IP RESOLUTION
# Resolves the Tailscale IPv4 address of the remote server.
# Sets REMOTE_SERVER used by all subsequent SSH and rsync calls.
# Exits if resolution fails — likely means Tailscale is down or peer is offline.
# -----------------------------------------------------------------------------------------------
resolve_remote_ip() {
log "$ICON_SYNC Resolving remote IP for $REMOTE_SERVER_NAME..."
REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)
[[ -z "$REMOTE_SERVER" ]] && error "Failed to resolve Tailscale IP for $REMOTE_SERVER_NAME" && exit 1
info "$ICON_SYNC Remote IP: $REMOTE_SERVER"
}
# -----------------------------------------------------------------------------------------------
# CONNECTIVITY CHECK
# Pings the remote server to confirm it is reachable before starting any transfers.
# Prevents the rsync retry loop from burning all attempts against an unreachable host.
# If ping fails, prints a tailscale status hint to aid diagnosis before exiting.
# -----------------------------------------------------------------------------------------------
check_connectivity() {
log "Checking connectivity to $REMOTE_SERVER..."
if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then
error "Remote $REMOTE_SERVER ($REMOTE_SERVER_NAME) is unreachable"
info "Hint: tailscale status | grep $REMOTE_SERVER_NAME"
exit 1
fi
log "Remote is reachable"
}
# -----------------------------------------------------------------------------------------------
# CONTAINER MANAGEMENT — STOP
# Stops all containers listed in CRITICAL_CONTAINER_NAMES on the remote server.
# Only stops containers that are currently running — skips those already stopped.
# Tracks stopped containers in RUNNING_CONTAINERS for restart after rsync completes.
# CRITICAL_CONTAINER_NAMES must be a bash array — rsync.sh handles conversion from profile strings.
# -----------------------------------------------------------------------------------------------
RUNNING_CONTAINERS=()
stop_containers() {
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \
[[ "${CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then
log "No containers configured for this profile, skipping stop."
return
fi
info "Stopping containers..."
RUNNING_CONTAINERS=()
for c in "${CRITICAL_CONTAINER_NAMES[@]}"; do
[[ -z "$c" ]] && continue
info "Checking $c..."
STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"docker inspect -f '{{.State.Running}}' $c 2>/dev/null" 2>/dev/null || echo "false")
if [[ "$STATUS" == "true" ]]; then
echo "$ICON_STOP Stopping $c..."
RUNNING_CONTAINERS+=("$c")
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $c" >/dev/null; then
success "$c stopped"
else
error "Failed to stop $c"
fi
else
warn "$c is not running, skipping"
fi
done
}
# -----------------------------------------------------------------------------------------------
# CONTAINER MANAGEMENT — START
# Restarts only the containers that were running before rsync and were stopped by stop_containers.
# Containers listed in DELAYED_CONTAINERS receive a sleep of CONTAINER_DELAY seconds before
# starting — useful for dependencies like Authelia that need upstream services ready first.
# DELAYED_CONTAINERS must be a bash array — rsync.sh handles conversion from profile strings.
# -----------------------------------------------------------------------------------------------
start_containers() {
if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then
log "No containers to restart."
return
fi
info "Starting containers..."
for c in "${RUNNING_CONTAINERS[@]}"; do
[[ -z "$c" ]] && continue
local needs_delay=false
for d in "${DELAYED_CONTAINERS[@]}"; do
if [[ "$c" == "$d" ]]; then
needs_delay=true
break
fi
done
if [[ "$needs_delay" == true ]]; then
info "Waiting ${CONTAINER_DELAY}s before starting $c..."
sleep "$CONTAINER_DELAY"
fi
echo "$ICON_RUN Starting $c..."
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $c" >/dev/null; then
success "$c started"
else
error "Failed to start $c"
fi
done
}
# -----------------------------------------------------------------------------------------------
# RSYNC OPTIONS
# Loads rsync options for the current profile from PROFILE_RSYNC_OPTS in Master.conf.
# If no profile match is found, falls back to DEFAULT_RSYNC_OPTS.
# Note: profile opts do NOT inherit from defaults — all desired flags must be listed explicitly.
# Sets RSYNC_OPTS array used directly in the rsync call in rsync.sh.
# -----------------------------------------------------------------------------------------------
get_rsync_opts() {
if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then
read -r -a RSYNC_OPTS <<< "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]}"
log "Using profile rsync opts for $PROFILE_NAME: ${RSYNC_OPTS[*]}"
else
RSYNC_OPTS=("${DEFAULT_RSYNC_OPTS[@]}")
log "Using default rsync opts: ${RSYNC_OPTS[*]}"
fi
}
# -----------------------------------------------------------------------------------------------
# STATUS DISPLAY
# Prints a summary of the current runtime configuration.
# Triggered by --status or --summary flag passed to rsync.sh.
# Useful for verifying profile resolution and variable state before a live run.
# -----------------------------------------------------------------------------------------------
show_status() {
echo "===== STATUS ====="
echo "Local: $LOCAL_SERVER_NAME"
echo "Remote: $REMOTE_SERVER_NAME"
echo "IP: $REMOTE_SERVER"
echo "Profile: $PROFILE_NAME"
echo "DryRun: $DRY_RUN"
echo "Logging: $ENABLE_LOGGING"
echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]}"
echo "Delayed: ${DELAYED_CONTAINERS[*]}"
echo "Excludes: ${EXCLUDE_DIRS[*]}"
echo "=================="
}