did way to much,,,,,, mostly added monitors, but almost every file was edited in some way
This commit is contained in:
@@ -2,7 +2,7 @@
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ----------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Version: 2.7
|
||||
# Version: 2.9
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Changelog:
|
||||
# v1.0 — Initial stable framework
|
||||
@@ -51,11 +51,20 @@
|
||||
# ping_internet added — non-fatal external ping for failover use
|
||||
# v2.7 — ICON_WEBGUI added for WebGUI watchdog operations
|
||||
# ICON_DOCKER_NET added for Docker network connect operations
|
||||
# v2.8 — ICON_CERT added for SSL certificate monitoring operations
|
||||
# v2.9 — ICON_MONITOR added for monitoring section headers
|
||||
# ICON_SMART added for drive SMART health operations
|
||||
# ICON_BANDWIDTH added for bandwidth tracking operations
|
||||
# ICON_DIGEST added for health digest operations
|
||||
# ICON_EMBY added for Emby session reporting
|
||||
# ICON_VERIFY added for backup verification operations
|
||||
# Monitor/ folder added to ecosystem
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ICONS
|
||||
# Each icon has one job — do not reuse across different contexts.
|
||||
# Adding a new icon: add it to the appropriate group below with a comment describing its job.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# System / Host
|
||||
@@ -117,6 +126,17 @@ ICON_FAILOVER="🔀" # failover state changes and operations
|
||||
# Docker Network Operations
|
||||
ICON_DOCKER_NET="🔌" # Docker network connect operations
|
||||
|
||||
# Security / Certificate Operations
|
||||
ICON_CERT="🔒" # SSL certificate monitoring
|
||||
|
||||
# Monitor Operations
|
||||
ICON_MONITOR="📈" # monitoring section headers and general monitoring
|
||||
ICON_SMART="🔧" # drive SMART health attribute monitoring
|
||||
ICON_BANDWIDTH="📶" # bandwidth usage tracking and reporting
|
||||
ICON_DIGEST="📰" # health digest — aggregated system summary
|
||||
ICON_EMBY="🎬" # Emby media server session reporting
|
||||
ICON_VERIFY="✔️" # backup verification — checksum comparison
|
||||
|
||||
# Notifications
|
||||
ICON_NOTIFY="🔔" # notification operations
|
||||
|
||||
@@ -128,6 +148,8 @@ ICON_SUCCESS="✅"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# OUTPUT HELPERS
|
||||
# Standardised output functions used across all scripts.
|
||||
# log() is gated by ENABLE_LOGGING — set in Master.conf or via --log flag.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
info() { echo "$ICON_INFO [INFO] $*"; }
|
||||
warn() { echo "$ICON_WARN [WARN] $*"; }
|
||||
@@ -140,8 +162,10 @@ log() {
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# NOTIFICATION
|
||||
# Sends a notification via unRAID native system and/or Discord webhook.
|
||||
# Both channels are optional and independently controlled via Master.conf.
|
||||
# Severity levels: normal, warning, alert
|
||||
# Usage: notify "message" "subject" "severity"
|
||||
# Severity: normal, warning, alert
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
notify() {
|
||||
local message="$1"
|
||||
@@ -175,6 +199,7 @@ notify() {
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# DURATION FORMATTER
|
||||
# Converts raw seconds into a human readable string — e.g. 10m53s or 47s
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
format_duration() {
|
||||
local secs=$1
|
||||
@@ -185,6 +210,10 @@ format_duration() {
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ARG PARSER
|
||||
# Processes all flags and key=value pairs passed to any script.
|
||||
# Supported flags: --dry-run, --log, --no-log, --status, --help
|
||||
# Supported key=value: LOG=true/false, or any declared variable e.g. BW_LIMIT=5000
|
||||
# Unparsed positional args returned in PARSED_ARGS array.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
parse_args() {
|
||||
ENABLE_LOGGING=${ENABLE_LOGGING:-false}
|
||||
@@ -229,12 +258,17 @@ parse_args() {
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# VALIDATION
|
||||
# VALIDATION HELPERS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# Exits with error if a required variable is empty or unset.
|
||||
# Usage: require_var VAR_NAME
|
||||
require_var() {
|
||||
[[ -z "${!1:-}" ]] && error "Missing required: $1" && exit 1
|
||||
}
|
||||
|
||||
# Exits with error if a variable is not a valid positive integer.
|
||||
# Usage: validate_int VAR_NAME "$VAR_VALUE"
|
||||
validate_int() {
|
||||
local name="$1" value="$2"
|
||||
if [[ -z "$value" ]]; then
|
||||
@@ -250,6 +284,9 @@ validate_int() {
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# HOST DETECTION
|
||||
# Determines which server is local and which is remote by comparing hostname against
|
||||
# HOST1 and HOST2 in Master.conf. Sets LOCAL_SERVER_NAME, REMOTE_SERVER_NAME and SSH_KEY.
|
||||
# Both servers run identical scripts — this is what makes them bidirectional.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
detect_hosts() {
|
||||
LOCAL_HOSTNAME="$(hostname)"
|
||||
@@ -271,12 +308,14 @@ detect_hosts() {
|
||||
SSH_KEY="${SSH_KEYS[$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME]}"
|
||||
|
||||
[[ -z "$SSH_KEY" ]] && error "Missing SSH key mapping" && exit 1
|
||||
|
||||
info "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# REMOTE IP RESOLUTION
|
||||
# Resolves the Tailscale IPv4 address of the remote server.
|
||||
# Sets REMOTE_SERVER used by all subsequent SSH and rsync calls.
|
||||
# Exits if resolution fails — Tailscale may be down or peer offline.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
resolve_remote_ip() {
|
||||
log "Resolving remote IP for $REMOTE_SERVER_NAME..."
|
||||
@@ -287,6 +326,8 @@ resolve_remote_ip() {
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# CONNECTIVITY CHECK — fatal, used by rsync scripts
|
||||
# Pings remote and exits if unreachable.
|
||||
# For failover use ping_remote() which returns status without exiting.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_connectivity() {
|
||||
log "Checking connectivity to $REMOTE_SERVER..."
|
||||
@@ -300,15 +341,15 @@ check_connectivity() {
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# PING REMOTE — non-fatal, used by failover
|
||||
# Returns 0 if reachable, 1 if not
|
||||
# Returns 0 if reachable, 1 if not — does NOT exit.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
ping_remote() {
|
||||
ping -c2 -W3 "$REMOTE_SERVER" &>/dev/null
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# PING INTERNET — non-fatal, used by failover
|
||||
# Returns 0 if internet reachable, 1 if not
|
||||
# PING INTERNET — non-fatal external connectivity check
|
||||
# Returns 0 if internet reachable, 1 if not — does NOT exit.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
ping_internet() {
|
||||
ping -c2 -W3 "${EXTERNAL_IP:-8.8.8.8}" &>/dev/null
|
||||
@@ -316,7 +357,9 @@ ping_internet() {
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# LOCAL ARRAY CHECK — non-fatal, returns status
|
||||
# Used by failover before starting remote containers locally
|
||||
# Verifies local /mnt/user is mounted and has shares.
|
||||
# Used by failover before starting remote containers locally.
|
||||
# Returns 0 if healthy, 1 if not.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_local_array() {
|
||||
log "Checking local array..."
|
||||
@@ -336,7 +379,9 @@ check_local_array() {
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# REMOTE ARRAY CHECK — non-fatal, returns status
|
||||
# Used before handback rsync
|
||||
# Verifies remote /mnt/user is mounted via SSH.
|
||||
# Used before handback rsync — syncing to remote with no array fills rootfs.
|
||||
# Returns 0 if healthy, 1 if not.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_remote_array() {
|
||||
log "Checking remote array on $REMOTE_SERVER_NAME..."
|
||||
@@ -353,7 +398,9 @@ check_remote_array() {
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# REMOTE DOCKER CHECK — non-fatal, returns status
|
||||
# Used before starting containers on remote
|
||||
# Verifies remote Docker daemon is responding before container operations.
|
||||
# A hung daemon means start/stop commands will silently fail.
|
||||
# Returns 0 if healthy, 1 if not.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_remote_docker() {
|
||||
log "Checking remote Docker daemon on $REMOTE_SERVER_NAME..."
|
||||
@@ -368,28 +415,26 @@ check_remote_docker() {
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# REMOTE ROOTFS SPACE CHECK — fatal
|
||||
# Aborts if remote rootfs exceeds ROOTFS_WARN threshold.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_remote_rootfs() {
|
||||
log "Checking remote rootfs usage..."
|
||||
REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null)
|
||||
|
||||
if [[ -z "$REMOTE_USAGE" ]]; then
|
||||
error "Could not retrieve rootfs usage from $REMOTE_SERVER_NAME"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then
|
||||
error "$ICON_HEALTH Remote rootfs ${REMOTE_USAGE}% — threshold ${ROOTFS_WARN:-75}%"
|
||||
warn "Array may be down or drives missing on $REMOTE_SERVER_NAME"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
info "$ICON_HEALTH Remote rootfs: ${REMOTE_USAGE}% (threshold: ${ROOTFS_WARN:-75}%)"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# REMOTE SHARE VALIDATION — fatal
|
||||
# Verifies target directory exists and is not empty on remote.
|
||||
# Usage: check_remote_share "/mnt/user/Movies"
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_remote_share() {
|
||||
@@ -398,7 +443,6 @@ check_remote_share() {
|
||||
|
||||
SHARE_EXISTS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null)
|
||||
|
||||
if [[ "$SHARE_EXISTS" != "yes" ]]; then
|
||||
error "$ICON_HEALTH Remote share does not exist: $dir"
|
||||
exit 1
|
||||
@@ -406,7 +450,6 @@ check_remote_share() {
|
||||
|
||||
SHARE_EMPTY=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null)
|
||||
|
||||
if [[ "$SHARE_EMPTY" == "yes" ]]; then
|
||||
warn "$ICON_HEALTH Remote share exists but is empty: $dir — aborting to protect data"
|
||||
exit 1
|
||||
@@ -417,6 +460,8 @@ check_remote_share() {
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# REMOTE DISK CHECK — fatal
|
||||
# Verifies all physical disks backing a share are online on the remote server.
|
||||
# Skipped when PROFILE_SKIP_DISK_CHECK is true (ZFS pools have no /mnt/disk* structure).
|
||||
# Usage: check_remote_disks "/mnt/user/Movies"
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_remote_disks() {
|
||||
@@ -439,10 +484,8 @@ check_remote_disks() {
|
||||
local disk_mount disk_name
|
||||
disk_mount=$(dirname "$disk_share_path")
|
||||
disk_name=$(basename "$disk_mount")
|
||||
|
||||
MOUNTED=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"mountpoint -q '$disk_mount' && echo yes || echo no" 2>/dev/null)
|
||||
|
||||
if [[ "$MOUNTED" == "yes" ]]; then
|
||||
info "$ICON_DISK $disk_name $ICON_RUNNING — $share_name present"
|
||||
else
|
||||
@@ -455,12 +498,13 @@ check_remote_disks() {
|
||||
error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "All disks backing $share_name are online"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# CONTAINER MANAGEMENT — STOP (remote via SSH)
|
||||
# Stops containers in CRITICAL_CONTAINER_NAMES on remote server.
|
||||
# Tracks running containers in RUNNING_CONTAINERS for restart after rsync.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
RUNNING_CONTAINERS=()
|
||||
|
||||
@@ -470,15 +514,12 @@ stop_containers() {
|
||||
log "No containers configured for this profile, skipping stop."
|
||||
return
|
||||
fi
|
||||
|
||||
info "Stopping containers..."
|
||||
RUNNING_CONTAINERS=()
|
||||
|
||||
for c in "${CRITICAL_CONTAINER_NAMES[@]}"; do
|
||||
[[ -z "$c" ]] && continue
|
||||
STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"docker inspect -f '{{.State.Running}}' $c 2>/dev/null" 2>/dev/null || echo "false")
|
||||
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
echo "$ICON_STOP Stopping $c..."
|
||||
RUNNING_CONTAINERS+=("$c")
|
||||
@@ -495,28 +536,25 @@ stop_containers() {
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# CONTAINER MANAGEMENT — START (remote via SSH)
|
||||
# Restarts only containers tracked in RUNNING_CONTAINERS.
|
||||
# Delayed containers receive CONTAINER_DELAY seconds before starting.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
start_containers() {
|
||||
if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then
|
||||
log "No containers to restart."
|
||||
return
|
||||
fi
|
||||
|
||||
info "Starting containers..."
|
||||
|
||||
for c in "${RUNNING_CONTAINERS[@]}"; do
|
||||
[[ -z "$c" ]] && continue
|
||||
|
||||
local needs_delay=false
|
||||
for d in "${DELAYED_CONTAINERS[@]}"; do
|
||||
[[ "$c" == "$d" ]] && needs_delay=true && break
|
||||
done
|
||||
|
||||
if [[ "$needs_delay" == true ]]; then
|
||||
info "Waiting ${CONTAINER_DELAY}s before starting $c..."
|
||||
sleep "$CONTAINER_DELAY"
|
||||
fi
|
||||
|
||||
echo "$ICON_START Starting $c..."
|
||||
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $c" >/dev/null; then
|
||||
echo "$ICON_STARTED $c started"
|
||||
@@ -528,6 +566,8 @@ start_containers() {
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# RSYNC OPTIONS
|
||||
# Loads rsync options for current profile. Falls back to DEFAULT_RSYNC_OPTS if no match.
|
||||
# Profile opts do NOT inherit from defaults — list all desired flags explicitly.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
get_rsync_opts() {
|
||||
if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then
|
||||
@@ -541,6 +581,7 @@ get_rsync_opts() {
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# STATUS DISPLAY
|
||||
# Prints current runtime configuration — triggered by --status flag.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
show_status() {
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
|
||||
Reference in New Issue
Block a user