FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added failover script and other sytem scripts

Browse Source
This commit is contained in:
FailedProxy
2026-04-11 17:46:11 -04:00
parent ee107e5df7
commit 55930a1166
7 changed files with 1388 additions and 386 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common.sh
+114 -129
View File
@@ -2,7 +2,7 @@
# -----------------------------------------------------------------------------------------------
# ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ----------------------------
# -----------------------------------------------------------------------------------------------
# Version: 2.4
# Version: 2.7
# -----------------------------------------------------------------------------------------------
# Changelog:
# v1.0 — Initial stable framework
@@ -40,6 +40,17 @@
# v2.4 — ICON_CLEAN, ICON_TRASH added for media cleaner operations
# ICON_PERMS, ICON_UNLOCKED added for media permissions operations
# ICON_REBOOT_SMART added for smart conditional reboot
# v2.5 — ICON_RAM added for ramdisk operations
# ICON_LINK added for symlink state and management
# Transcode scripts group added to ecosystem
# v2.6 — ICON_FAILOVER added for failover operations
# check_local_array added — verifies local /mnt/user is mounted and healthy
# check_remote_array added — verifies remote /mnt/user is mounted and healthy
# check_remote_docker added — verifies remote Docker daemon is responding
# ping_remote added — non-fatal ping returning status for failover use
# ping_internet added — non-fatal external ping for failover use
# v2.7 — ICON_WEBGUI added for WebGUI watchdog operations
# ICON_DOCKER_NET added for Docker network connect operations
# -----------------------------------------------------------------------------------------------
# -----------------------------------------------------------------------------------------------
@@ -59,13 +70,13 @@ ICON_HEALTH="🩺" # rootfs / share health checks
ICON_SHIELD="🛡️" # pre-flight section header
# Containers
ICON_CONTAINERS="📦" # container section anchor — paired with action icon for direction
ICON_CONTAINERS="📦" # container section anchor
ICON_STOP="⛔" # stop command being issued
ICON_STOPPED="🔴" # container confirmed stopped
ICON_START="▶️" # start command being issued
ICON_STARTED="💚" # container confirmed started
ICON_RUNNING="🟢" # container already running when checked
ICON_NOT_RUNNING="⭕" # container already stopped when checked — distinct from ICON_STOPPED
ICON_NOT_RUNNING="⭕" # container already stopped when checked
# Transfer
ICON_SYNC="🔄" # transfer section header
@@ -83,6 +94,7 @@ ICON_REBOOT="⚡" # scheduled server reboot
ICON_REBOOT_SMART="🚨" # smart conditional reboot triggered
ICON_PLUGIN="🧩" # user scripts plugin operations
ICON_PHP="👥" # PHP-FPM operations
ICON_WEBGUI="💻" # WebGUI / nginx / emhttp operations
# Diagnostics
ICON_ZFS="📊" # ZFS ARC statistics
@@ -95,6 +107,16 @@ ICON_TRASH="🗑️" # files being deleted
ICON_PERMS="🔐" # permissions operation / section header
ICON_UNLOCKED="🔓" # permissions successfully applied to a share
# Transcode Operations
ICON_RAM="💨" # ramdisk operations — fast ephemeral storage
ICON_LINK="🔗" # symlink state and management
# Failover Operations
ICON_FAILOVER="🔀" # failover state changes and operations
# Docker Network Operations
ICON_DOCKER_NET="🔌" # Docker network connect operations
# Notifications
ICON_NOTIFY="🔔" # notification operations
@@ -106,8 +128,6 @@ ICON_SUCCESS="✅"
# -----------------------------------------------------------------------------------------------
# OUTPUT HELPERS
# Standardised output functions used across all scripts.
# log() is gated by ENABLE_LOGGING — set in Master.conf or via --log flag.
# -----------------------------------------------------------------------------------------------
info() { echo "$ICON_INFO [INFO] $*"; }
warn() { echo "$ICON_WARN [WARN] $*"; }
@@ -120,14 +140,8 @@ log() {
# -----------------------------------------------------------------------------------------------
# NOTIFICATION
# Sends a notification via unRAID native system and/or Discord webhook.
# Both channels are optional and independently controlled via Master.conf.
# unRAID native: requires NOTIFY_UNRAID=true and the dynamix notify script to be present.
# Discord: requires DISCORD_WEBHOOK to be set to a valid webhook URL.
# Severity levels: normal, warning, alert — maps to unRAID notification severity.
# Usage: notify "message" "subject" "severity"
# notify "Rsync failed: Movies" "Rsync Alert" "warning"
# notify "Daily sync complete" "Daily Sync" "normal"
# Severity: normal, warning, alert
# -----------------------------------------------------------------------------------------------
notify() {
local message="$1"
@@ -136,7 +150,6 @@ notify() {
log "$ICON_NOTIFY Sending notification: $subject — $message"
# unRAID native notification
if [[ "${NOTIFY_UNRAID:-false}" == true ]]; then
local notify_script="/usr/local/emhttp/plugins/dynamix/scripts/notify"
if [[ -x "$notify_script" ]]; then
@@ -147,7 +160,6 @@ notify() {
fi
fi
# Discord webhook notification
if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then
local payload
payload=$(printf '{"content": "%s — **%s**\\n%s"}' \
@@ -163,9 +175,6 @@ notify() {
# -----------------------------------------------------------------------------------------------
# DURATION FORMATTER
# Converts raw seconds into a human readable string — e.g. 10m53s or 47s
# Used by rsync.sh summary and daily_sync.sh summary.
# Sourced from common.sh so both scripts share the same implementation.
# -----------------------------------------------------------------------------------------------
format_duration() {
local secs=$1
@@ -176,24 +185,17 @@ format_duration() {
# -----------------------------------------------------------------------------------------------
# ARG PARSER
# Processes all flags and key=value pairs passed to any script.
# Positional arguments (directory paths) are separated before calling this — see rsync.sh.
# Supported flags: --dry-run, --log, --no-log, --status, --help
# Supported key=value: LOG=true/false, or any declared variable e.g. BW_LIMIT=5000
# Unparsed positional args are returned in PARSED_ARGS array.
# -----------------------------------------------------------------------------------------------
parse_args() {
ENABLE_LOGGING=${ENABLE_LOGGING:-false}
DRY_RUN=${DRY_RUN:-false}
SHOW_STATUS=${SHOW_STATUS:-false}
CLEAN_ARGS=()
for ARG in "$@"; do
if [[ "$ARG" == *=* ]]; then
VAR="${ARG%%=*}"
VAL="${ARG#*=}"
case "$VAR" in
LOG)
[[ "$VAL" == "true" ]] && ENABLE_LOGGING=true
@@ -215,7 +217,7 @@ parse_args() {
--no-log) ENABLE_LOGGING=false ;;
--status|--summary) SHOW_STATUS=true ;;
--help|-h)
echo "Usage: script <dir> [--dry-run] [--log] [--status]"
echo "Usage: script [--dry-run] [--log] [--status]"
exit 0
;;
*) CLEAN_ARGS+=("$ARG") ;;
@@ -228,43 +230,26 @@ parse_args() {
# -----------------------------------------------------------------------------------------------
# VALIDATION
# Checks that a required variable is set and non-empty.
# Usage: require_var VAR_NAME
# Exits with error if the variable is missing.
# -----------------------------------------------------------------------------------------------
require_var() {
[[ -z "${!1:-}" ]] && error "Missing required: $1" && exit 1
}
# -----------------------------------------------------------------------------------------------
# INTEGER VALIDATION
# Checks that a variable contains a valid positive integer.
# Exits with a clear error if the value is missing, empty, or not a number.
# Usage: validate_int VAR_NAME "$VAR_VALUE"
# Example: validate_int MOVER_STOP_TIMEOUT "$MOVER_STOP_TIMEOUT"
# -----------------------------------------------------------------------------------------------
validate_int() {
local name="$1"
local value="$2"
local name="$1" value="$2"
if [[ -z "$value" ]]; then
error "$name is not set — check Master.conf"
exit 1
fi
if ! [[ "$value" =~ ^[0-9]+$ ]]; then
error "$name must be a positive integer — got: '$value'"
exit 1
fi
log "$name validated: $value"
}
# -----------------------------------------------------------------------------------------------
# HOST DETECTION
# Determines local and remote server names by comparing hostname against HOST1/HOST2.
# Sets LOCAL_SERVER_NAME, REMOTE_SERVER_NAME, and SSH_KEY for the current run direction.
# Both HOST1 and HOST2 must be defined in Master.conf.
# -----------------------------------------------------------------------------------------------
detect_hosts() {
LOCAL_HOSTNAME="$(hostname)"
@@ -283,34 +268,25 @@ detect_hosts() {
declare -A SSH_KEYS
SSH_KEYS["$HOST1|$HOST2"]="$HOST1_SSH_KEY"
SSH_KEYS["$HOST2|$HOST1"]="$HOST2_SSH_KEY"
SSH_KEY="${SSH_KEYS[$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME]}"
[[ -z "$SSH_KEY" ]] && error "Missing SSH key mapping for $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" && exit 1
[[ -z "$SSH_KEY" ]] && error "Missing SSH key mapping" && exit 1
info "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME"
}
# -----------------------------------------------------------------------------------------------
# REMOTE IP RESOLUTION
# Resolves the Tailscale IPv4 address of the remote server.
# Sets REMOTE_SERVER used by all subsequent SSH and rsync calls.
# Exits if resolution fails — likely means Tailscale is down or peer is offline.
# -----------------------------------------------------------------------------------------------
resolve_remote_ip() {
log "Resolving remote IP for $REMOTE_SERVER_NAME..."
REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)
[[ -z "$REMOTE_SERVER" ]] && error "Failed to resolve Tailscale IP for $REMOTE_SERVER_NAME" && exit 1
info "$ICON_NET Remote IP: $REMOTE_SERVER"
}
# -----------------------------------------------------------------------------------------------
# CONNECTIVITY CHECK
# Pings the remote server to confirm it is reachable before starting any transfers.
# Prevents the rsync retry loop from burning all attempts against an unreachable host.
# If ping fails, prints a tailscale status hint to aid diagnosis before exiting.
# CONNECTIVITY CHECK — fatal, used by rsync scripts
# -----------------------------------------------------------------------------------------------
check_connectivity() {
log "Checking connectivity to $REMOTE_SERVER..."
@@ -323,16 +299,78 @@ check_connectivity() {
}
# -----------------------------------------------------------------------------------------------
# REMOTE ROOTFS SPACE CHECK
# Checks the remote server's rootfs usage before any rsync runs.
# If the array is down or drives are missing, rsync writes land on rootfs instead of the array —
# this can fill the remote filesystem rapidly and crash the server.
# Threshold is set by ROOTFS_WARN in Master.conf (recommended: 75).
# Aborts cleanly with a clear error showing current usage vs threshold.
# PING REMOTE — non-fatal, used by failover
# Returns 0 if reachable, 1 if not
# -----------------------------------------------------------------------------------------------
ping_remote() {
ping -c2 -W3 "$REMOTE_SERVER" &>/dev/null
}
# -----------------------------------------------------------------------------------------------
# PING INTERNET — non-fatal, used by failover
# Returns 0 if internet reachable, 1 if not
# -----------------------------------------------------------------------------------------------
ping_internet() {
ping -c2 -W3 "${EXTERNAL_IP:-8.8.8.8}" &>/dev/null
}
# -----------------------------------------------------------------------------------------------
# LOCAL ARRAY CHECK — non-fatal, returns status
# Used by failover before starting remote containers locally
# -----------------------------------------------------------------------------------------------
check_local_array() {
log "Checking local array..."
if ! mountpoint -q /mnt/user 2>/dev/null; then
error "$ICON_DISK Local array is not started — /mnt/user is not mounted"
return 1
fi
local file_count
file_count=$(ls /mnt/user 2>/dev/null | wc -l)
if [[ "$file_count" -eq 0 ]]; then
error "$ICON_DISK Local array appears empty — shares may not be available"
return 1
fi
log "Local array is healthy"
return 0
}
# -----------------------------------------------------------------------------------------------
# REMOTE ARRAY CHECK — non-fatal, returns status
# Used before handback rsync
# -----------------------------------------------------------------------------------------------
check_remote_array() {
log "Checking remote array on $REMOTE_SERVER_NAME..."
local result
result=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
"mountpoint -q /mnt/user && echo yes || echo no" 2>/dev/null)
if [[ "$result" != "yes" ]]; then
error "$ICON_DISK Remote array not started on $REMOTE_SERVER_NAME"
return 1
fi
log "Remote array is healthy"
return 0
}
# -----------------------------------------------------------------------------------------------
# REMOTE DOCKER CHECK — non-fatal, returns status
# Used before starting containers on remote
# -----------------------------------------------------------------------------------------------
check_remote_docker() {
log "Checking remote Docker daemon on $REMOTE_SERVER_NAME..."
if ! ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
"timeout 10 docker ps" >/dev/null 2>&1; then
error "$ICON_CONTAINERS Remote Docker daemon not responding on $REMOTE_SERVER_NAME"
return 1
fi
log "Remote Docker daemon is healthy"
return 0
}
# -----------------------------------------------------------------------------------------------
# REMOTE ROOTFS SPACE CHECK — fatal
# -----------------------------------------------------------------------------------------------
check_remote_rootfs() {
log "Checking remote rootfs usage..."
REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null)
@@ -342,39 +380,27 @@ check_remote_rootfs() {
fi
if [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then
echo ""
error "$ICON_HEALTH Remote rootfs is ${REMOTE_USAGE}% full — threshold is ${ROOTFS_WARN:-75}%"
error "$ICON_HEALTH Remote rootfs ${REMOTE_USAGE}% — threshold ${ROOTFS_WARN:-75}%"
warn "Array may be down or drives missing on $REMOTE_SERVER_NAME"
info "Hint: Check array status on $REMOTE_SERVER_NAME before retrying"
echo ""
exit 1
fi
info "$ICON_HEALTH Remote rootfs: ${REMOTE_USAGE}% used (threshold: ${ROOTFS_WARN:-75}%)"
info "$ICON_HEALTH Remote rootfs: ${REMOTE_USAGE}% (threshold: ${ROOTFS_WARN:-75}%)"
}
# -----------------------------------------------------------------------------------------------
# REMOTE SHARE VALIDATION
# Verifies that the target directory exists and is not empty on the remote server.
# Catches the scenario where the array is mounted but drives are not backing the share —
# the path exists as an empty mountpoint, which would cause --delete to wipe the remote.
# Called with the specific directory being synced so each share is checked individually.
# REMOTE SHARE VALIDATION — fatal
# Usage: check_remote_share "/mnt/user/Movies"
# -----------------------------------------------------------------------------------------------
check_remote_share() {
local dir="$1"
log "Checking remote share: $dir..."
SHARE_EXISTS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null)
if [[ "$SHARE_EXISTS" != "yes" ]]; then
echo ""
error "$ICON_HEALTH Remote share does not exist: $dir"
warn "Array may not be started or share is not configured on $REMOTE_SERVER_NAME"
info "Hint: Check shares and array status on $REMOTE_SERVER_NAME before retrying"
echo ""
exit 1
fi
@@ -382,11 +408,7 @@ check_remote_share() {
"[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null)
if [[ "$SHARE_EMPTY" == "yes" ]]; then
echo ""
warn "$ICON_HEALTH Remote share exists but is empty: $dir"
warn "Drives may not be mounted on $REMOTE_SERVER_NAME — aborting to protect data"
info "Hint: Verify array and drive assignments on $REMOTE_SERVER_NAME before retrying"
echo ""
warn "$ICON_HEALTH Remote share exists but is empty: $dir — aborting to protect data"
exit 1
fi
@@ -394,12 +416,7 @@ check_remote_share() {
}
# -----------------------------------------------------------------------------------------------
# REMOTE DISK CHECK
# Verifies that all physical disks backing a share are online and mounted on the remote server.
# Discovers disk layout automatically at runtime by finding all /mnt/diskN/sharename paths —
# no configuration required, works for any share regardless of how many disks it spans.
# Aborts if any single disk backing the share is offline — partial disk failure means
# incomplete data which could result in files being deleted by --delete during sync.
# REMOTE DISK CHECK — fatal
# Usage: check_remote_disks "/mnt/user/Movies"
# -----------------------------------------------------------------------------------------------
check_remote_disks() {
@@ -413,20 +430,14 @@ check_remote_disks() {
"ls -d /mnt/disk*/$share_name 2>/dev/null" 2>/dev/null)
if [[ -z "$DISK_PATHS" ]]; then
echo ""
error "$ICON_DISK No disks found backing share $share_name on $REMOTE_SERVER_NAME"
warn "Share may not exist on any disk or array may not be started"
info "Hint: Check array and share configuration on $REMOTE_SERVER_NAME"
echo ""
error "$ICON_DISK No disks found backing $share_name on $REMOTE_SERVER_NAME"
exit 1
fi
local all_ok=true
while IFS= read -r disk_share_path; do
local disk_mount
local disk_mount disk_name
disk_mount=$(dirname "$disk_share_path")
local disk_name
disk_name=$(basename "$disk_mount")
MOUNTED=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
@@ -435,17 +446,13 @@ check_remote_disks() {
if [[ "$MOUNTED" == "yes" ]]; then
info "$ICON_DISK $disk_name $ICON_RUNNING — $share_name present"
else
error "$ICON_DISK $disk_name $ICON_STOPPED — $share_name missing or incomplete"
error "$ICON_DISK $disk_name $ICON_STOPPED — $share_name missing"
all_ok=false
fi
done <<< "$DISK_PATHS"
if [[ "$all_ok" == false ]]; then
echo ""
error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME"
warn "Aborting to prevent partial or destructive sync"
info "Hint: Check disk assignments and array status on $REMOTE_SERVER_NAME before retrying"
echo ""
exit 1
fi
@@ -453,11 +460,7 @@ check_remote_disks() {
}
# -----------------------------------------------------------------------------------------------
# CONTAINER MANAGEMENT — STOP
# Stops all containers listed in CRITICAL_CONTAINER_NAMES on the remote server.
# Only stops containers that are currently running — skips those already stopped.
# Tracks stopped containers in RUNNING_CONTAINERS for restart after rsync completes.
# CRITICAL_CONTAINER_NAMES must be a bash array — rsync.sh handles conversion from profile strings.
# CONTAINER MANAGEMENT — STOP (remote via SSH)
# -----------------------------------------------------------------------------------------------
RUNNING_CONTAINERS=()
@@ -473,16 +476,12 @@ stop_containers() {
for c in "${CRITICAL_CONTAINER_NAMES[@]}"; do
[[ -z "$c" ]] && continue
info "Checking $c..."
STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"docker inspect -f '{{.State.Running}}' $c 2>/dev/null" 2>/dev/null || echo "false")
if [[ "$STATUS" == "true" ]]; then
echo "$ICON_STOP Stopping $c..."
RUNNING_CONTAINERS+=("$c")
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $c" >/dev/null; then
echo "$ICON_STOPPED $c stopped"
else
@@ -495,11 +494,7 @@ stop_containers() {
}
# -----------------------------------------------------------------------------------------------
# CONTAINER MANAGEMENT — START
# Restarts only the containers that were running before rsync and were stopped by stop_containers.
# Containers listed in DELAYED_CONTAINERS receive a sleep of CONTAINER_DELAY seconds before
# starting — useful for dependencies like Authelia that need upstream services ready first.
# DELAYED_CONTAINERS must be a bash array — rsync.sh handles conversion from profile strings.
# CONTAINER MANAGEMENT — START (remote via SSH)
# -----------------------------------------------------------------------------------------------
start_containers() {
if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then
@@ -514,10 +509,7 @@ start_containers() {
local needs_delay=false
for d in "${DELAYED_CONTAINERS[@]}"; do
if [[ "$c" == "$d" ]]; then
needs_delay=true
break
fi
[[ "$c" == "$d" ]] && needs_delay=true && break
done
if [[ "$needs_delay" == true ]]; then
@@ -536,10 +528,6 @@ start_containers() {
# -----------------------------------------------------------------------------------------------
# RSYNC OPTIONS
# Loads rsync options for the current profile from PROFILE_RSYNC_OPTS in Master.conf.
# If no profile match is found, falls back to DEFAULT_RSYNC_OPTS.
# Note: profile opts do NOT inherit from defaults — all desired flags must be listed explicitly.
# Sets RSYNC_OPTS array used directly in the rsync call in rsync.sh.
# -----------------------------------------------------------------------------------------------
get_rsync_opts() {
if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then
@@ -553,20 +541,17 @@ get_rsync_opts() {
# -----------------------------------------------------------------------------------------------
# STATUS DISPLAY
# Prints a summary of the current runtime configuration.
# Triggered by --status or --summary flag passed to any script.
# Useful for verifying profile resolution and variable state before a live run.
# -----------------------------------------------------------------------------------------------
show_status() {
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "Local: $LOCAL_SERVER_NAME"
echo "Remote: $REMOTE_SERVER_NAME"
echo "IP: $REMOTE_SERVER"
echo "Profile: $PROFILE_NAME"
echo "Profile: ${PROFILE_NAME:-n/a}"
echo "DryRun: $DRY_RUN"
echo "Logging: $ENABLE_LOGGING"
echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]}"
echo "Delayed: ${DELAYED_CONTAINERS[*]}"
echo "Excludes: ${EXCLUDE_DIRS[*]}"
echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]:-n/a}"
echo "Delayed: ${DELAYED_CONTAINERS[*]:-n/a}"
echo "Excludes: ${EXCLUDE_DIRS[*]:-n/a}"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
}