Gmer4Lfe
2a062e5140
Every orchestrator invoked its children differently — four near-duplicate run_job() copies, a differently-shaped run_watchdog(), or plain inline bash calls, each with its own take on path resolution, pass/fail naming, and dry-run threading. Extracted one shared run_orch_child() into common.sh so there's a single place to fix or extend this behavior going forward. Along the way: watchdog_orchestrator.sh and monthly_maintenance.sh were checking $VERBOSE, a variable nothing in the codebase ever assigns, so --log silently did nothing beyond basic logging on those two. Fixed to $ENABLE_LOGGING. watchdog_orchestrator.sh and array_started.sh had no trailing exit, so their exit codes reflected whatever the last command happened to return rather than actual success/failure. transcode_management.sh had no failure notification and no summary at all. Also made transcode_management.sh's two-script pipeline config-driven (TRANSCODE_MANAGEMENT_SCRIPTS in master.conf) instead of hardcoded, for room to extend it later without editing the orchestrator itself.
1993 lines
91 KiB
Bash
Executable File
1993 lines
91 KiB
Bash
Executable File
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ================================= COMMON LIBRARY =============================================
|
||
# ==============================================================================================
|
||
# Shared functions used by every script in the ecosystem.
|
||
# Sourced automatically by load_config.sh — do not source directly.
|
||
#
|
||
# ── WHAT THIS FILE PROVIDES ───────────────────────────────────────────────────────────────────
|
||
# Icons — consistent visual language across all script output
|
||
# Output helpers — info, warn, error, success, log, notify
|
||
# Arg parser — unified --dry-run, --log, --status, KEY=VALUE handling
|
||
# Validation — require_var, validate_int
|
||
# Host detection — detect_hosts() identifies local/remote server, sets all aliases
|
||
# Connectivity — resolve_remote_ip, check_connectivity, ping_remote, ping_internet
|
||
# Health checks — local/remote array, docker daemon, rootfs, share, disk temps
|
||
# Disk verification — check_remote_disks() — XFS, ZFS, cache pools, nested paths
|
||
# Container mgmt — stop/start local and remote containers with state tracking
|
||
# Rsync helpers — get_rsync_opts, check_rsync_enabled
|
||
# Locking — acquire_lock, acquire_rsync_lock — concurrent execution protection
|
||
# Path translation — translate_path() — container → host path for arr cleanup scripts
|
||
# Arr utilities — check_arr_version() — API version safety gate
|
||
# Status display — show_status() — runtime config dump on --status flag
|
||
#
|
||
# ── CHANGELOG ─────────────────────────────────────────────────────────────────────────────────
|
||
#
|
||
# Version What changed
|
||
# ───────────────────────────────────────────────────────────────────────────────────────────
|
||
# v1.0 Initial stable framework — output helpers, arg parser, host detection
|
||
#
|
||
# v1.1 format_duration() moved here from daily_sync_maintenance.sh for shared use
|
||
# SSH_KEY naming collision resolved — gitea key renamed GITEA_SSH_KEY in master.conf
|
||
#
|
||
# v1.2 check_connectivity() added — fatal ping with Tailscale hint on failure
|
||
# Function header comment blocks standardised across all functions
|
||
#
|
||
# v1.3 check_remote_rootfs() — aborts rsync if remote rootfs exceeds ROOTFS_WARN
|
||
# check_remote_share() — aborts if target directory missing or empty on remote
|
||
# Both protect against rsync running when remote array is down
|
||
#
|
||
# v1.4 check_remote_disks() — verifies all physical disks backing a share are mounted
|
||
# Auto-discovers disk layout at runtime from disks.ini — no config required
|
||
# Aborts if any disk backing the share is offline or unmounted
|
||
#
|
||
# v1.5-2.9 Icon set expanded progressively — each operation has its own distinct icon
|
||
# All function output updated to use correct icon per context
|
||
# notify() added — unRAID native + Discord webhook notifications
|
||
# Script locking system added — acquire_lock(), acquire_rsync_lock()
|
||
# translate_path() added — container → host path for arr cleanup scripts
|
||
# check_arr_version() added — API version safety gate before arr operations
|
||
# check_api() added — pre-flight API reachability check
|
||
# ping_remote(), ping_internet() — non-fatal ping for fallback use
|
||
# check_local_array(), check_remote_array() — array health checks
|
||
# check_remote_docker() — Docker daemon health check
|
||
# get_unraid_temp_thresholds() — reads thresholds from dynamix.cfg
|
||
# check_local_disk_temps() — pre-rsync temp check with exit codes 0/1/2
|
||
# stop/start local containers added alongside existing remote variants
|
||
#
|
||
# v3.1 Three-file config split — master.conf + host1.conf + host2.conf
|
||
# load_config.sh introduced — auto-discovers all host*.conf files
|
||
# detect_hosts() rewritten — sets MY_ID/REMOTE_ID and aliases all HOST* vars
|
||
# All scripts now source load_config.sh instead of conf files directly
|
||
# Adding a new server = add host*.conf, zero script changes required
|
||
#
|
||
# v3.2 check_remote_disks() rewritten — three-tier detection:
|
||
# Tier 1: array disk paths (/mnt/disk*/sharename)
|
||
# Tier 2: ZFS pools with find -maxdepth 2 (catches nested cache paths)
|
||
# Tier 3: shfs fallback — verifies /mnt/user is mounted
|
||
# Fixes: nested paths like /mnt/cache/appdata-Fallback/Critical-Data
|
||
# Fixes: ZFS cache pools not detected when share is not at pool root
|
||
#
|
||
# v3.3 PROFILE_REMOTE_RESTART_CONTAINERS support added
|
||
# Dirty sync profiles (critical-fallback, emby-fallback) restart remote
|
||
# containers after sync if they were running before — picks up config changes
|
||
# Was stopped → stays stopped. Was running → gets restarted. ✅
|
||
#
|
||
# v3.5 notify_emby_scan() added — triggers Emby "Clean Missing Files" task
|
||
# after arr cleanup scripts delete orphaned files
|
||
# Emby immediately removes ghost entries — no user-facing file-not-found errors
|
||
# Called by lidarr/sonarr/radarr_cleanup.sh when files are deleted
|
||
#
|
||
#
|
||
# Three new safety functions added:
|
||
# check_os_version_parity() — refuses remote ops on version mismatch
|
||
# reads OS version via platform_get_os_version() / platform_os_version_probe_cmd()
|
||
# major mismatch → abort | minor mismatch → configurable warn/abort
|
||
# check_remote_docker_daemon() — verifies remote Docker daemon before
|
||
# issuing any remote container commands — strike system → skip/retry/exit
|
||
# validate_unraid_cmd() — verifies unRAID-specific commands exist and
|
||
# produce expected output before use — notifies and exits calling script
|
||
# if command changed or disappeared after upgrade — other scripts unaffected
|
||
#
|
||
# ── VERSION ───────────────────────────────────────────────────────────────────────────────────
|
||
# Current: v3.5
|
||
# ==============================================================================================
|
||
|
||
# Cron runs with a minimal PATH (/usr/bin:/bin) that omits /usr/local/sbin where tailscale,
|
||
# and other Unraid tools live. Prepend it here so every sourcing script finds them.
|
||
export PATH="/usr/local/sbin:/usr/local/bin:$PATH"
|
||
|
||
# ==============================================================================================
|
||
# ── ICONS ─────────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Each icon has one job — do not reuse across different contexts.
|
||
# Adding a new icon: add to the appropriate group below with a comment describing its job.
|
||
|
||
# System / Host
|
||
ICON_HOST="🖥️" # host detection and identity
|
||
ICON_NET="🌐" # network / IP resolution
|
||
ICON_PING="📡" # connectivity check
|
||
ICON_GEAR="⚙️" # setup section header / profile load
|
||
|
||
# Health Checks
|
||
ICON_DISK="💾" # disk checks
|
||
ICON_HEALTH="🩺" # rootfs / share health checks
|
||
ICON_SHIELD="🛡️" # pre-flight section header
|
||
|
||
# Containers
|
||
ICON_CONTAINERS="📦" # container section anchor
|
||
ICON_STOP="⛔" # stop command being issued
|
||
ICON_STOPPED="🔴" # container confirmed stopped
|
||
ICON_START="▶️" # start command being issued
|
||
ICON_STARTED="💚" # container confirmed started
|
||
ICON_RUNNING="🟢" # container already running when checked
|
||
ICON_SKIP="⏭️" # skipping — already running healthy instance
|
||
ICON_PROTECTED="🔰" # file/item is protected — never delete
|
||
ICON_NOT_RUNNING="⭕" # container already stopped when checked
|
||
|
||
# Transfer
|
||
ICON_SYNC="🔄" # transfer section header
|
||
ICON_RUN="🚀" # sync starting / rsync attempt
|
||
ICON_RETRY="🔁" # retry attempt
|
||
ICON_DONE="🏁" # transfer complete
|
||
|
||
# Summary
|
||
ICON_SUMMARY="📋" # summary section header
|
||
ICON_TIME="⏱️" # duration line
|
||
|
||
# System Operations
|
||
ICON_MOVER="🔃" # mover operations
|
||
ICON_REBOOT="⚡" # scheduled server reboot
|
||
ICON_REBOOT_SMART="🚨" # smart conditional reboot triggered
|
||
ICON_PLUGIN="🧩" # user scripts plugin operations
|
||
ICON_PHP="👥" # PHP-FPM operations
|
||
ICON_WEBGUI="💻" # WebGUI / nginx / emhttp operations
|
||
|
||
# Diagnostics
|
||
ICON_ZFS="📊" # ZFS ARC statistics
|
||
ICON_MEM="🧠" # memory status
|
||
ICON_WATCHDOG="🐾" # docker watchdog monitoring operations
|
||
|
||
# Media Operations
|
||
ICON_CLEAN="🧹" # media cleaner operations
|
||
ICON_TRASH="🗑️" # files being deleted
|
||
ICON_PERMS="🔐" # permissions operation / section header
|
||
ICON_UNLOCKED="🔓" # permissions successfully applied to a share
|
||
ICON_LOCK="🔏" # script instance lock — acquired/released
|
||
|
||
# Transcode Operations
|
||
ICON_RAM="💨" # ramdisk operations — fast ephemeral storage
|
||
ICON_LINK="🔗" # symlink state and management
|
||
|
||
# fallback Operations
|
||
ICON_FALLBACK="🔀" # fallback state changes and operations
|
||
|
||
# Docker Network Operations
|
||
ICON_DOCKER_NET="🔌" # Docker network connect operations
|
||
|
||
# Security / Certificate Operations
|
||
ICON_CERT="🔒" # SSL certificate monitoring
|
||
|
||
# Monitor Operations
|
||
ICON_MONITOR="📈" # monitoring section headers and general monitoring
|
||
ICON_SMART="🔧" # drive SMART health attribute monitoring
|
||
ICON_BANDWIDTH="📶" # bandwidth usage tracking and reporting
|
||
ICON_DIGEST="📰" # health digest — aggregated system summary
|
||
ICON_EMBY="🎬" # Emby media server session reporting
|
||
ICON_VERIFY="✔️" # backup verification — checksum comparison
|
||
|
||
# Notifications
|
||
ICON_NOTIFY="🔔" # notification operations
|
||
|
||
# Output
|
||
ICON_INFO="ℹ️"
|
||
ICON_WARN="⚠️"
|
||
ICON_ERROR="❌"
|
||
ICON_SUCCESS="✅"
|
||
|
||
# ==============================================================================================
|
||
# ── OUTPUT HELPERS ────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Standardised output functions used across all scripts.
|
||
#
|
||
# Output tiers:
|
||
# echo — always visible — summaries, status conclusions, section headers
|
||
# info() — always visible — status confirmations, reachability, check pass
|
||
# warn() — always visible — state transitions, warnings, important events
|
||
# error() — always visible — something broke
|
||
# success() — always visible — operation completed successfully
|
||
# log() — only with --log flag — per-item detail, internal checks
|
||
#
|
||
# All output goes to stdout — callers can redirect as needed.
|
||
|
||
info() { echo "$ICON_INFO [INFO] $*"; }
|
||
warn() { echo "$ICON_WARN [WARN] $*"; }
|
||
error() { echo "$ICON_ERROR [ERROR] $*"; }
|
||
success() { echo "$ICON_DONE [OK] $*"; }
|
||
|
||
log() {
|
||
[[ "${ENABLE_LOGGING:-false}" == true ]] && echo "[LOG] $*"
|
||
return 0
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── NOTIFICATION ──────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Sends a notification via unRAID native system and/or Discord webhook.
|
||
# Both channels are optional and independently controlled:
|
||
# NOTIFY_UNRAID — shared toggle in master.conf
|
||
# MY_DISCORD_WEBHOOK — per-host in host*.conf, set by detect_hosts()
|
||
#
|
||
# Severity levels: normal, warning, alert
|
||
# Usage: notify "message" "subject" "severity"
|
||
|
||
notify() {
|
||
local message="$1"
|
||
local subject="${2:-unRAID Notification}"
|
||
local severity="${3:-normal}"
|
||
|
||
log "$ICON_NOTIFY Sending notification: $subject — $message"
|
||
|
||
if [[ "${NOTIFY_UNRAID:-false}" == true ]]; then
|
||
if platform_send_os_notification "$message" "$subject" "$severity"; then
|
||
log "$ICON_NOTIFY OS notification sent"
|
||
else
|
||
log "$ICON_NOTIFY OS notify not available — skipping"
|
||
fi
|
||
fi
|
||
|
||
# Uses MY_DISCORD_WEBHOOK — set by detect_hosts() from HOST*_DISCORD_WEBHOOK
|
||
if [[ -n "${MY_DISCORD_WEBHOOK:-}" ]]; then
|
||
local payload
|
||
payload=$(printf '{"content": "%s — **%s**\\n%s"}' \
|
||
"$ICON_NOTIFY" "$subject" "$message")
|
||
if curl -s -H "Content-Type: application/json" \
|
||
-d "$payload" "$MY_DISCORD_WEBHOOK" >/dev/null 2>&1; then
|
||
log "$ICON_NOTIFY Discord notification sent"
|
||
else
|
||
warn "Discord notification failed — check HOST*_DISCORD_WEBHOOK in host*.conf"
|
||
fi
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── DURATION FORMATTER ────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Converts raw seconds into a human readable string.
|
||
# Examples: 47 → "47s" | 653 → "10m53s"
|
||
# Usage: format_duration $SECONDS
|
||
|
||
format_duration() {
|
||
local secs=$1
|
||
local days=$(( secs / 86400 ))
|
||
local hrs=$(( (secs % 86400) / 3600 ))
|
||
local mins=$(( (secs % 3600) / 60 ))
|
||
local rem=$(( secs % 60 ))
|
||
if [[ $days -gt 0 ]]; then echo "${days}d ${hrs}h ${mins}m"
|
||
elif [[ $hrs -gt 0 ]]; then echo "${hrs}h ${mins}m"
|
||
elif [[ $mins -gt 0 ]]; then echo "${mins}m${rem}s"
|
||
else echo "${rem}s"; fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── ARG PARSER ────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Processes all flags and key=value pairs passed to any script.
|
||
# Call early in every script: parse_args "$@"
|
||
#
|
||
# Supported flags:
|
||
# --dry-run | -n → DRY_RUN=true
|
||
# --log → ENABLE_LOGGING=true
|
||
# --no-log → ENABLE_LOGGING=false
|
||
# --status → SHOW_STATUS=true
|
||
# --help | -h → print usage and exit
|
||
#
|
||
# Supported key=value:
|
||
# LOG=true/false → toggle logging
|
||
# BW_LIMIT=5000 → override any declared master.conf variable for this run
|
||
#
|
||
# Unparsed positional args returned in PARSED_ARGS array.
|
||
|
||
parse_args() {
|
||
ENABLE_LOGGING=${ENABLE_LOGGING:-false}
|
||
DRY_RUN=${DRY_RUN:-false}
|
||
SHOW_STATUS=${SHOW_STATUS:-false}
|
||
CLEAN_ARGS=()
|
||
|
||
for ARG in "$@"; do
|
||
if [[ "$ARG" == *=* ]]; then
|
||
VAR="${ARG%%=*}"
|
||
VAL="${ARG#*=}"
|
||
case "$VAR" in
|
||
LOG)
|
||
[[ "$VAL" == "true" ]] && ENABLE_LOGGING=true
|
||
[[ "$VAL" == "false" ]] && ENABLE_LOGGING=false
|
||
;;
|
||
*)
|
||
if declare -p "$VAR" &>/dev/null; then
|
||
printf -v "$VAR" '%s' "$VAL"
|
||
log "Set $VAR=$VAL"
|
||
else
|
||
warn "Unknown variable: $VAR"
|
||
fi
|
||
;;
|
||
esac
|
||
else
|
||
case "$ARG" in
|
||
--dry-run|-n) DRY_RUN=true ;;
|
||
--log) ENABLE_LOGGING=true ;;
|
||
--no-log) ENABLE_LOGGING=false ;;
|
||
--status|--summary) SHOW_STATUS=true ;;
|
||
--help|-h)
|
||
echo "Usage: script [--dry-run] [--log] [--status]"
|
||
exit 0
|
||
;;
|
||
*) CLEAN_ARGS+=("$ARG") ;;
|
||
esac
|
||
fi
|
||
done
|
||
|
||
PARSED_ARGS=("${CLEAN_ARGS[@]}")
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── VALIDATION HELPERS ────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# Exits with error if a required variable is empty or unset.
|
||
# Usage: require_var VAR_NAME
|
||
require_var() {
|
||
[[ -z "${!1:-}" ]] && error "Missing required: $1" && exit 1
|
||
}
|
||
|
||
# Exits with error if a variable is not a valid positive integer.
|
||
# Usage: validate_int VAR_NAME "$VAR_VALUE"
|
||
validate_int() {
|
||
local name="$1" value="$2"
|
||
if [[ -z "$value" ]]; then
|
||
error "$name is not set — check master.conf"
|
||
exit 1
|
||
fi
|
||
if ! [[ "$value" =~ ^[0-9]+$ ]]; then
|
||
error "$name must be a positive integer — got: '$value'"
|
||
exit 1
|
||
fi
|
||
log "$name validated: $value"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── HOST DETECTION ────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Determines which server is local and which is remote by comparing hostname against
|
||
# all HOST* values discovered from host*.conf files.
|
||
#
|
||
# Sets:
|
||
# MY_ID — "HOST1" or "HOST2" (the role key, not the hostname)
|
||
# REMOTE_ID — the peer's role key
|
||
# LOCAL_SERVER_NAME — local hostname string (e.g. "unRAID-Gmer4Lfe")
|
||
# REMOTE_SERVER_NAME — remote hostname string
|
||
# SSH_KEY — SSH key for server-to-server operations
|
||
# REMOTE_STORAGE_PATH — remote's storage root from HOST*_STORAGE_PATH in host*.conf;
|
||
# falls back to platform_storage_path() for unconfigured pairs
|
||
#
|
||
# Also sets aliases for all host-specific arrays so scripts use unprefixed names:
|
||
# DAILY_SYNC_SHARES ← HOST*_DAILY_SYNC_SHARES
|
||
# INTERMEDIATE_SYNC_SHARES ← HOST*_INTERMEDIATE_SYNC_SHARES
|
||
# WEEKLY_SYNC_SHARES ← HOST*_WEEKLY_SYNC_SHARES
|
||
# CRITICAL_SYNC_SHARES ← HOST*_CRITICAL_SYNC_SHARES
|
||
# DAILY_RESTART_CONTAINERS ← HOST*_DAILY_RESTART_CONTAINERS
|
||
# WEEKLY_RESTART_CONTAINERS ← HOST*_WEEKLY_RESTART_CONTAINERS
|
||
# WATCHDOG_CONTAINERS ← HOST*_WATCHDOG_CONTAINERS (associative)
|
||
# WATCHDOG_CONTAINER_URLS ← HOST*_WATCHDOG_CONTAINER_URLS (associative)
|
||
# WATCHDOG_REQUIRED_CONTAINERS ← HOST*_WATCHDOG_REQUIRED_CONTAINERS
|
||
# WATCHDOG_SCAN_IGNORE ← HOST*_WATCHDOG_SCAN_IGNORE
|
||
# WATCHDOG_DEPENDENCIES ← HOST*_WATCHDOG_DEPENDENCIES (associative)
|
||
# WATCHDOG_APPDATA_SIZES ← HOST*_WATCHDOG_APPDATA_SIZES (associative)
|
||
# NETWORK_CONNECT_CONTAINERS ← HOST*_NETWORK_CONNECT_CONTAINERS
|
||
# NETWORK_CONNECT_NETWORKS ← HOST*_NETWORK_CONNECT_NETWORKS
|
||
# MEDIA_PERMISSION_SHARES ← HOST*_MEDIA_PERMISSION_SHARES
|
||
# ANIME_CLEAN_FOLDERS ← HOST*_ANIME_CLEAN_FOLDERS
|
||
# MEDIA_CLEAN_FOLDERS ← HOST*_MEDIA_CLEAN_FOLDERS
|
||
# CERT_MONITOR_DOMAINS ← HOST*_CERT_MONITOR_DOMAINS
|
||
# SMART_IGNORE_DRIVES ← HOST*_SMART_IGNORE_DRIVES
|
||
# ZFS_REPORT_IGNORE_POOLS ← HOST*_ZFS_REPORT_IGNORE_POOLS
|
||
# TRANSCODE_SSD ← HOST*_TRANSCODE_SSD
|
||
# TRANSCODE_SERVERS ← HOST*_TRANSCODE_SERVERS
|
||
# RAMDISK_SIZE ← HOST*_RAMDISK_SIZE
|
||
# RAMDISK_WARN_GB ← HOST*_RAMDISK_WARN_GB
|
||
# RAMDISK_LOW_GB ← HOST*_RAMDISK_LOW_GB
|
||
# BACKUP_VERIFY_SHARES ← HOST*_BACKUP_VERIFY_SHARES
|
||
# DDNS_CONTAINERS ← HOST*_DDNS_CONTAINERS
|
||
# PARTNERSHIP_AUTH_WEBUIS ← HOST*_PARTNERSHIP_AUTH_WEBUIS
|
||
# PARTNERSHIP_MIRROR_BACKUPS ← HOST*_PARTNERSHIP_MIRROR_BACKUPS
|
||
# MY_DISCORD_WEBHOOK ← HOST*_DISCORD_WEBHOOK
|
||
# EMBY_CONTAINER ← HOST*_EMBY_CONTAINER
|
||
# EMBY_URL ← HOST*_EMBY_URL
|
||
# EMBY_API_KEY ← HOST*_EMBY_API_KEY
|
||
# GITEA_API_TOKEN ← HOST*_GITEA_API_TOKEN
|
||
#
|
||
# Arr-specific vars set by detect_hosts() when needed by arr scripts:
|
||
# LIDARR_URL / LIDARR_API_KEY / LIDARR_MUSIC_ROOT (MY_ID only — Lidarr is HOST1 only)
|
||
# SONARR_URL / SONARR_API_KEY / SONARR_TV_ROOT
|
||
# RADARR_URL / RADARR_API_KEY / RADARR_MOVIES_ROOT
|
||
# SLSKD_URL / SLSKD_API_KEY / SLSKD_FAILED_IMPORTS_DIR (MY_ID only)
|
||
# SABNZBD_URL / SABNZBD_API_KEY (MY_ID only)
|
||
# QBIT_URL / QBIT_USERNAME / QBIT_PASSWORD (MY_ID only)
|
||
|
||
detect_hosts() {
|
||
local local_hostname
|
||
local_hostname="$(hostname)"
|
||
|
||
# ── Find MY_ID by matching hostname against all HOST* vars ────────────────
|
||
MY_ID=""
|
||
local host_var host_val
|
||
|
||
# Scan all HOST* vars that hold a hostname value
|
||
# HOST1, HOST2, HOST3 etc. — all defined in host*.conf files
|
||
for host_var in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
||
host_val="${!host_var:-}"
|
||
[[ -z "$host_val" ]] && continue
|
||
if [[ "$local_hostname" == "$host_val" ]]; then
|
||
MY_ID="$host_var"
|
||
break
|
||
fi
|
||
done
|
||
|
||
if [[ -z "$MY_ID" ]]; then
|
||
error "Unknown host: $local_hostname"
|
||
error "Hostname must match a HOST* value in host*.conf"
|
||
error "Available: $(for h in HOST1 HOST2 HOST3 HOST4; do
|
||
[[ -n "${!h:-}" ]] && echo -n "${!h} "; done)"
|
||
exit 1
|
||
fi
|
||
|
||
# ── Find REMOTE_ID — first HOST* that isn't MY_ID ────────────────────────
|
||
REMOTE_ID=""
|
||
for host_var in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
||
[[ "$host_var" == "$MY_ID" ]] && continue
|
||
host_val="${!host_var:-}"
|
||
[[ -z "$host_val" ]] && continue
|
||
REMOTE_ID="$host_var"
|
||
break
|
||
done
|
||
|
||
# ── Set host name strings ─────────────────────────────────────────────────
|
||
LOCAL_SERVER_NAME="${!MY_ID}"
|
||
[[ -n "$REMOTE_ID" ]] && REMOTE_SERVER_NAME="${!REMOTE_ID}" || REMOTE_SERVER_NAME="unknown"
|
||
|
||
# ── Set SSH key ───────────────────────────────────────────────────────────
|
||
local ssh_key_var="${MY_ID}_SSH_KEY"
|
||
SSH_KEY="${!ssh_key_var:-}"
|
||
if [[ -z "$SSH_KEY" ]]; then
|
||
error "Missing SSH key: ${ssh_key_var} not set in host*.conf"
|
||
exit 1
|
||
fi
|
||
|
||
# ── Set scalar aliases ────────────────────────────────────────────────────
|
||
MY_DISCORD_WEBHOOK_VAR="${MY_ID}_DISCORD_WEBHOOK"
|
||
MY_DISCORD_WEBHOOK="${!MY_DISCORD_WEBHOOK_VAR:-}"
|
||
|
||
OWNER_NAME_VAR="${MY_ID}_OWNER"
|
||
OWNER_NAME="${!OWNER_NAME_VAR:-}"
|
||
OWNER_EMAIL_VAR="${MY_ID}_OWNER_EMAIL"
|
||
OWNER_EMAIL="${!OWNER_EMAIL_VAR:-}"
|
||
|
||
EMBY_CONTAINER_VAR="${MY_ID}_EMBY_CONTAINER"
|
||
EMBY_CONTAINER="${!EMBY_CONTAINER_VAR:-}"
|
||
EMBY_URL_VAR="${MY_ID}_EMBY_URL"
|
||
EMBY_URL="${!EMBY_URL_VAR:-}"
|
||
EMBY_API_KEY_VAR="${MY_ID}_EMBY_API_KEY"
|
||
EMBY_API_KEY="${!EMBY_API_KEY_VAR:-}"
|
||
|
||
GITEA_API_TOKEN_VAR="${MY_ID}_GITEA_API_TOKEN"
|
||
GITEA_API_TOKEN="${!GITEA_API_TOKEN_VAR:-}"
|
||
|
||
local _prov_var
|
||
for _prov_var in \
|
||
PARTNERSHIP_PROVISION_EMBY_ADMIN \
|
||
PARTNERSHIP_EMBY_ADMIN_USER \
|
||
PARTNERSHIP_EMBY_ADMIN_PASS \
|
||
PARTNERSHIP_EMBY_PORT; do
|
||
local _src="${MY_ID}_${_prov_var}"
|
||
printf -v "$_prov_var" '%s' "${!_src:-}"
|
||
done
|
||
|
||
# ── System watchdog check toggles — per-host ─────────────────────────────
|
||
# Aliased as unprefixed SYS_WATCHDOG_CHECK_* for use in system_watchdog.sh
|
||
local _wd_checks=(
|
||
SYS_WATCHDOG_NIC
|
||
SYS_WATCHDOG_CHECK_DOCKER_DAEMON SYS_WATCHDOG_CHECK_ROOTFS
|
||
SYS_WATCHDOG_CHECK_KERNEL_OOPS SYS_WATCHDOG_CHECK_FD
|
||
SYS_WATCHDOG_CHECK_BOOT SYS_WATCHDOG_CHECK_OOM
|
||
SYS_WATCHDOG_CHECK_RAM SYS_WATCHDOG_CHECK_LOG
|
||
SYS_WATCHDOG_CHECK_ARC SYS_WATCHDOG_CHECK_CPU_TEMP
|
||
SYS_WATCHDOG_CHECK_LOAD SYS_WATCHDOG_CHECK_ZOMBIES
|
||
SYS_WATCHDOG_CHECK_CONTAINERS SYS_WATCHDOG_CHECK_TMP
|
||
SYS_WATCHDOG_CHECK_MDSTAT SYS_WATCHDOG_CHECK_NETWORK
|
||
SYS_WATCHDOG_CHECK_SSHD SYS_WATCHDOG_CHECK_RUNAWAY
|
||
)
|
||
for _wd_var in "${_wd_checks[@]}"; do
|
||
local _wd_src="${MY_ID}_${_wd_var}"
|
||
# Only alias if the host-specific var is set — preserves master.conf defaults
|
||
[[ -n "${!_wd_src+x}" ]] && eval "${_wd_var}=\"\${${_wd_src}}\""
|
||
done
|
||
|
||
TRANSCODE_SSD_VAR="${MY_ID}_TRANSCODE_SSD"
|
||
TRANSCODE_SSD="${!TRANSCODE_SSD_VAR:-}"
|
||
RAMDISK_SIZE_VAR="${MY_ID}_RAMDISK_SIZE"
|
||
RAMDISK_SIZE="${!RAMDISK_SIZE_VAR:-8G}"
|
||
RAMDISK_WARN_GB_VAR="${MY_ID}_RAMDISK_WARN_GB"
|
||
RAMDISK_WARN_GB="${!RAMDISK_WARN_GB_VAR:-6.8}"
|
||
RAMDISK_LOW_GB_VAR="${MY_ID}_RAMDISK_LOW_GB"
|
||
RAMDISK_LOW_GB="${!RAMDISK_LOW_GB_VAR:-5.5}"
|
||
|
||
# Arr credentials — MY_ID only (remote arr accessed via its own host)
|
||
LIDARR_URL_VAR="${MY_ID}_LIDARR_URL"; LIDARR_URL="${!LIDARR_URL_VAR:-}"
|
||
LIDARR_API_KEY_VAR="${MY_ID}_LIDARR_API_KEY"; LIDARR_API_KEY="${!LIDARR_API_KEY_VAR:-}"
|
||
LIDARR_MUSIC_ROOT_VAR="${MY_ID}_LIDARR_MUSIC_ROOT"; LIDARR_MUSIC_ROOT="${!LIDARR_MUSIC_ROOT_VAR:-}"
|
||
LIDARR_RECOVERY_VAR="${MY_ID}_LIDARR_RECOVERY"; LIDARR_RECOVERY="${!LIDARR_RECOVERY_VAR:-false}"
|
||
FANART_API_KEY_VAR="${MY_ID}_FANART_API_KEY"; FANART_API_KEY="${!FANART_API_KEY_VAR:-}"
|
||
LASTFM_API_KEY_VAR="${MY_ID}_LASTFM_API_KEY"; LASTFM_API_KEY="${!LASTFM_API_KEY_VAR:-}"
|
||
TMDB_API_KEY_VAR="${MY_ID}_TMDB_API_KEY"; TMDB_API_KEY="${!TMDB_API_KEY_VAR:-}"
|
||
|
||
SONARR_URL_VAR="${MY_ID}_SONARR_URL"; SONARR_URL="${!SONARR_URL_VAR:-}"
|
||
SONARR_API_KEY_VAR="${MY_ID}_SONARR_API_KEY"; SONARR_API_KEY="${!SONARR_API_KEY_VAR:-}"
|
||
SONARR_TV_ROOT_VAR="${MY_ID}_SONARR_TV_ROOT"; SONARR_TV_ROOT="${!SONARR_TV_ROOT_VAR:-}"
|
||
SONARR_RECOVERY_VAR="${MY_ID}_SONARR_RECOVERY"; SONARR_RECOVERY="${!SONARR_RECOVERY_VAR:-true}"
|
||
|
||
RADARR_URL_VAR="${MY_ID}_RADARR_URL"; RADARR_URL="${!RADARR_URL_VAR:-}"
|
||
RADARR_API_KEY_VAR="${MY_ID}_RADARR_API_KEY"; RADARR_API_KEY="${!RADARR_API_KEY_VAR:-}"
|
||
RADARR_MOVIES_ROOT_VAR="${MY_ID}_RADARR_MOVIES_ROOT"; RADARR_MOVIES_ROOT="${!RADARR_MOVIES_ROOT_VAR:-}"
|
||
RADARR_RECOVERY_VAR="${MY_ID}_RADARR_RECOVERY"; RADARR_RECOVERY="${!RADARR_RECOVERY_VAR:-true}"
|
||
|
||
SLSKD_URL_VAR="${MY_ID}_SLSKD_URL"; SLSKD_URL="${!SLSKD_URL_VAR:-}"
|
||
SLSKD_API_KEY_VAR="${MY_ID}_SLSKD_API_KEY"; SLSKD_API_KEY="${!SLSKD_API_KEY_VAR:-}"
|
||
SLSKD_FAILED_IMPORTS_DIR_VAR="${MY_ID}_SLSKD_FAILED_IMPORTS_DIR"
|
||
SLSKD_FAILED_IMPORTS_DIR="${!SLSKD_FAILED_IMPORTS_DIR_VAR:-}"
|
||
|
||
SABNZBD_URL_VAR="${MY_ID}_SABNZBD_URL"; SABNZBD_URL="${!SABNZBD_URL_VAR:-}"
|
||
SABNZBD_API_KEY_VAR="${MY_ID}_SABNZBD_API_KEY"; SABNZBD_API_KEY="${!SABNZBD_API_KEY_VAR:-}"
|
||
|
||
QBIT_URL_VAR="${MY_ID}_QBIT_URL"; QBIT_URL="${!QBIT_URL_VAR:-}"
|
||
QBIT_USERNAME_VAR="${MY_ID}_QBIT_USERNAME"; QBIT_USERNAME="${!QBIT_USERNAME_VAR:-}"
|
||
QBIT_PASSWORD_VAR="${MY_ID}_QBIT_PASSWORD"; QBIT_PASSWORD="${!QBIT_PASSWORD_VAR:-}"
|
||
|
||
# ── Set array aliases — indexed arrays ────────────────────────────────────
|
||
# Each eval copies the host-specific array into the unprefixed name scripts use
|
||
|
||
_alias_array() {
|
||
local alias_name="$1"
|
||
local source_var="${MY_ID}_${alias_name}"
|
||
eval "${alias_name}=(\"\${${source_var}[@]}\")"
|
||
}
|
||
|
||
_alias_array "DAILY_SYNC_SHARES"
|
||
_alias_array "PERSONAL_SHARES"
|
||
_alias_array "INTERMEDIATE_SYNC_SHARES"
|
||
_alias_array "WEEKLY_SYNC_SHARES"
|
||
_alias_array "CRITICAL_SYNC_SHARES"
|
||
_alias_array "BACKUP_VERIFY_SHARES"
|
||
_alias_array "DAILY_RESTART_CONTAINERS"
|
||
_alias_array "WEEKLY_RESTART_CONTAINERS"
|
||
_alias_array "WATCHDOG_REQUIRED_CONTAINERS"
|
||
_alias_array "WATCHDOG_SCAN_IGNORE"
|
||
_alias_array "NETWORK_CONNECT_CONTAINERS"
|
||
_alias_array "NETWORK_CONNECT_NETWORKS"
|
||
_alias_array "MEDIA_PERMISSION_SHARES"
|
||
_alias_array "ANIME_CLEAN_FOLDERS"
|
||
_alias_array "MEDIA_CLEAN_FOLDERS"
|
||
_alias_array "CERT_MONITOR_DOMAINS"
|
||
_alias_array "SMART_IGNORE_DRIVES"
|
||
_alias_array "ZFS_REPORT_IGNORE_POOLS"
|
||
_alias_array "TRANSCODE_SERVERS"
|
||
_alias_array "DDNS_CONTAINERS"
|
||
_alias_array "PARTNERSHIP_AUTH_WEBUIS"
|
||
_alias_array "PARTNERSHIP_MIRROR_BACKUPS"
|
||
_alias_array "PARTNERSHIP_OWN_CONTAINERS"
|
||
_alias_array "PARTNERSHIP_AUTH_STACK"
|
||
_alias_array "PARTNERSHIP_REPLACE_CONTAINERS"
|
||
_alias_array "PARTNERSHIP_ARR_STACK"
|
||
_alias_array "PARTNERSHIP_ARR_REPLACE_CONTAINERS"
|
||
_alias_array "PARTNERSHIP_SERVICES_STACK"
|
||
_alias_array "PARTNERSHIP_SERVICES_REPLACE_CONTAINERS"
|
||
_alias_array "RW_PAUSE_CONTAINERS"
|
||
_alias_array "RW_STOP_CONTAINERS"
|
||
|
||
# ── Set array aliases — associative arrays ────────────────────────────────
|
||
# Associative arrays cannot be copied with eval — must be rebuilt key by key
|
||
|
||
_alias_assoc() {
|
||
local alias_name="$1"
|
||
local source_name="${MY_ID}_${alias_name}"
|
||
eval "declare -gA ${alias_name}"
|
||
local -n _assoc_src="${source_name}" 2>/dev/null || return 0
|
||
local -n _assoc_dst="${alias_name}"
|
||
local _assoc_key
|
||
for _assoc_key in "${!_assoc_src[@]}"; do
|
||
_assoc_dst["$_assoc_key"]="${_assoc_src[$_assoc_key]}"
|
||
done
|
||
}
|
||
|
||
_alias_assoc "WATCHDOG_CONTAINERS"
|
||
_alias_assoc "WATCHDOG_CONTAINER_URLS"
|
||
_alias_assoc "WATCHDOG_CONTAINER_API_CHECKS"
|
||
_alias_assoc "WATCHDOG_DEPENDENCIES"
|
||
_alias_assoc "WATCHDOG_APPDATA_SIZES"
|
||
|
||
# ── Remote storage path ───────────────────────────────────────────────────
|
||
# Resolved from REMOTE_ID's host*.conf so SSH commands use the correct path
|
||
# even when the remote is a different platform. Falls back to the local
|
||
# platform_storage_path() for unconfigured same-platform pairs.
|
||
local _rsp_var="${REMOTE_ID}_STORAGE_PATH"
|
||
REMOTE_STORAGE_PATH="${!_rsp_var:-$(platform_storage_path)}"
|
||
|
||
# ── Output ────────────────────────────────────────────────────────────────
|
||
log "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── REMOTE IP RESOLUTION ──────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Resolves the Tailscale IPv4 address of the remote server.
|
||
# Sets REMOTE_SERVER used by all subsequent SSH and rsync calls.
|
||
# Exits if resolution fails — Tailscale may be down or peer offline.
|
||
|
||
resolve_remote_ip() {
|
||
log "Resolving remote IP for $REMOTE_SERVER_NAME..."
|
||
local ts_name="${REMOTE_SERVER_NAME,,}"
|
||
local attempts=3
|
||
|
||
for ((i=1; i<=attempts; i++)); do
|
||
# Direct lookup — works when MagicDNS short-name resolution is active
|
||
REMOTE_SERVER=$(tailscale ip -4 "$ts_name" 2>/dev/null)
|
||
|
||
# Fallback — parse tailscale status for FQDN entries (e.g. hostname.tailXXXX.ts.net)
|
||
if [[ -z "$REMOTE_SERVER" ]]; then
|
||
REMOTE_SERVER=$(tailscale status 2>/dev/null | awk -v name="$ts_name" '$2 ~ "^" name { print $1; exit }')
|
||
fi
|
||
|
||
[[ -n "$REMOTE_SERVER" ]] && break
|
||
[[ $i -lt $attempts ]] && { warn "Tailscale resolution attempt $i/$attempts failed — retrying in 5s..."; sleep 5; }
|
||
done
|
||
|
||
if [[ -z "$REMOTE_SERVER" ]]; then
|
||
error "Failed to resolve Tailscale IP for $REMOTE_SERVER_NAME after $attempts attempts"
|
||
error "Check: tailscale status | grep $ts_name"
|
||
exit 1
|
||
fi
|
||
info "$ICON_NET Remote IP: $REMOTE_SERVER"
|
||
}
|
||
|
||
# Resolve any hostname to a Tailscale IPv4 — tries direct lookup, falls back to status parse.
|
||
# Usage: ip=$(resolve_tailscale_ip "hostname") — returns empty string on failure.
|
||
resolve_tailscale_ip() {
|
||
local hostname="${1,,}"
|
||
local ip
|
||
ip=$(tailscale ip -4 "$hostname" 2>/dev/null)
|
||
[[ -z "$ip" ]] && \
|
||
ip=$(tailscale status 2>/dev/null | awk -v name="$hostname" '$2 ~ "^" name { print $1; exit }')
|
||
echo "$ip"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── CONNECTIVITY CHECKS ───────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# Fatal connectivity check — used by rsync and other scripts that must abort if unreachable.
|
||
# For fallback use ping_remote() which returns status without exiting.
|
||
check_connectivity() {
|
||
log "Checking connectivity to $REMOTE_SERVER..."
|
||
if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then
|
||
error "$ICON_PING Remote $REMOTE_SERVER ($REMOTE_SERVER_NAME) is unreachable"
|
||
info "Hint: tailscale status | grep $REMOTE_SERVER_NAME"
|
||
exit 1
|
||
fi
|
||
info "$ICON_PING $REMOTE_SERVER_NAME is reachable"
|
||
}
|
||
|
||
# Non-fatal ping — used by fallback.sh which handles its own state machine.
|
||
# Returns 0 if reachable, 1 if not — does NOT exit.
|
||
ping_remote() {
|
||
ping -c2 -W3 "$REMOTE_SERVER" &>/dev/null
|
||
}
|
||
|
||
# Non-fatal external connectivity check — used by fallback.sh.
|
||
# Returns 0 if internet reachable, 1 if not — does NOT exit.
|
||
ping_internet() {
|
||
ping -c2 -W3 "${EXTERNAL_IP:-8.8.8.8}" &>/dev/null
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── RSYNC GATE CHECK ──────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Two-tier rsync enable/disable system.
|
||
# Tier 1: RSYNC_ENABLED — global gate, overrides everything
|
||
# Tier 2: orchestrator-specific flag — fine-grained control per window
|
||
#
|
||
# Usage:
|
||
# check_rsync_enabled "CRITICAL" ← checks RSYNC_ENABLED + CRITICAL_RSYNC_ENABLED
|
||
# check_rsync_enabled "INTERMEDIATE" ← checks RSYNC_ENABLED + INTERMEDIATE_RSYNC_ENABLED
|
||
# check_rsync_enabled "DAILY" ← checks RSYNC_ENABLED + DAILY_RSYNC_ENABLED
|
||
# check_rsync_enabled "WEEKLY" ← checks RSYNC_ENABLED + WEEKLY_RSYNC_ENABLED
|
||
# check_rsync_enabled "FALLBACK" ← checks RSYNC_ENABLED + FALLBACK_RSYNC_ENABLED
|
||
# check_rsync_enabled ← checks RSYNC_ENABLED only (direct rsync.sh call)
|
||
#
|
||
# Returns: 0 = enabled, proceed | 1 = disabled, skip cleanly
|
||
|
||
check_rsync_enabled() {
|
||
local orchestrator="${1:-}"
|
||
|
||
# Tier 1 — global gate
|
||
if [[ "${RSYNC_ENABLED:-true}" == false ]]; then
|
||
warn "RSYNC_ENABLED=false — rsync globally disabled, skipping all syncs"
|
||
return 1
|
||
fi
|
||
|
||
# Tier 2 — per-orchestrator
|
||
if [[ -n "$orchestrator" ]]; then
|
||
local var_name="${orchestrator}_RSYNC_ENABLED"
|
||
local var_value="${!var_name:-true}"
|
||
if [[ "$var_value" == false ]]; then
|
||
info "${var_name}=false — rsync disabled for $orchestrator orchestrator"
|
||
info "All other $orchestrator jobs will still run normally"
|
||
return 1
|
||
fi
|
||
fi
|
||
|
||
return 0
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── PLATFORM SERVICE STATE ────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Check whether Docker and VM Manager are enabled on this host.
|
||
# Delegates to platform_is_service_enabled() from the platform adapter.
|
||
|
||
is_docker_enabled() {
|
||
platform_is_service_enabled docker
|
||
}
|
||
|
||
is_vm_manager_enabled() {
|
||
platform_is_service_enabled libvirt
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── LOCAL HEALTH CHECKS ───────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# Verifies local storage is mounted and has shares.
|
||
# Non-fatal — returns status for caller to decide.
|
||
# Used by fallback before starting remote containers locally.
|
||
check_local_array() {
|
||
log "Checking local array..."
|
||
if ! platform_storage_healthy; then
|
||
error "$ICON_DISK Local array is not started — storage not mounted"
|
||
return 1
|
||
fi
|
||
local _storage_path file_count
|
||
_storage_path=$(platform_storage_path)
|
||
file_count=$(ls "$_storage_path" 2>/dev/null | wc -l)
|
||
if [[ "$file_count" -eq 0 ]]; then
|
||
error "$ICON_DISK Local array appears empty — shares may not be available"
|
||
return 1
|
||
fi
|
||
log "Local array is healthy"
|
||
return 0
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── REMOTE HEALTH CHECKS ─────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# Verifies remote storage is mounted via SSH.
|
||
# Non-fatal — returns status. Used before handback rsync.
|
||
# Syncing to remote with no array fills rootfs rapidly.
|
||
check_remote_array() {
|
||
log "Checking remote array on $REMOTE_SERVER_NAME..."
|
||
local _spath result
|
||
_spath="$REMOTE_STORAGE_PATH"
|
||
result=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
||
"mountpoint -q '$_spath' && echo yes || echo no" 2>/dev/null)
|
||
if [[ "$result" != "yes" ]]; then
|
||
error "$ICON_DISK Remote array not started on $REMOTE_SERVER_NAME"
|
||
return 1
|
||
fi
|
||
log "Remote array is healthy"
|
||
return 0
|
||
}
|
||
|
||
# Verifies remote Docker daemon is responding.
|
||
# Non-fatal — returns status. A hung daemon means container commands silently fail.
|
||
check_remote_docker() {
|
||
log "Checking remote Docker daemon on $REMOTE_SERVER_NAME..."
|
||
if ! ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
||
"timeout 10 docker ps" >/dev/null 2>&1; then
|
||
error "$ICON_CONTAINERS Remote Docker daemon not responding on $REMOTE_SERVER_NAME"
|
||
return 1
|
||
fi
|
||
log "Remote Docker daemon is healthy"
|
||
return 0
|
||
}
|
||
|
||
# Aborts if remote rootfs (/) usage is at or above ROOTFS_WARN threshold.
|
||
# Fatal — exits the calling script.
|
||
# When remote array is down rsync writes land on rootfs and fill it rapidly.
|
||
check_remote_rootfs() {
|
||
log "Checking remote rootfs usage..."
|
||
local remote_usage
|
||
remote_usage=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null)
|
||
if [[ -z "$remote_usage" ]]; then
|
||
error "Could not retrieve rootfs usage from $REMOTE_SERVER_NAME"
|
||
exit 1
|
||
fi
|
||
if [[ "$remote_usage" -ge "${ROOTFS_WARN:-75}" ]]; then
|
||
error "$ICON_HEALTH Remote rootfs ${remote_usage}% — threshold ${ROOTFS_WARN:-75}%"
|
||
exit 1
|
||
fi
|
||
info "$ICON_HEALTH Remote rootfs: ${remote_usage}% (threshold: ${ROOTFS_WARN:-75}%)"
|
||
}
|
||
|
||
# Verifies target directory exists and is not empty on remote.
|
||
# Fatal — exits the calling script.
|
||
# Usage: check_remote_share "/mnt/user/Movies"
|
||
check_remote_share() {
|
||
local dir="$1"
|
||
log "Checking remote share: $dir..."
|
||
|
||
local share_exists
|
||
share_exists=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null)
|
||
if [[ "$share_exists" != "yes" ]]; then
|
||
error "$ICON_HEALTH Remote share does not exist: $dir"
|
||
exit 1
|
||
fi
|
||
|
||
local share_empty
|
||
share_empty=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null)
|
||
if [[ "$share_empty" == "yes" ]]; then
|
||
warn "$ICON_HEALTH Remote share exists but is empty: $dir — aborting to protect data"
|
||
exit 1
|
||
fi
|
||
|
||
info "$ICON_HEALTH Remote share verified: $dir"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── DISK TEMPERATURE CHECKS ───────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# Returns 0 (true) if device is non-rotational (SSD/NVMe), 1 (false) if HDD.
|
||
# Checks /sys/block/<device>/queue/rotational — 0=SSD, 1=HDD.
|
||
is_ssd() {
|
||
local base
|
||
base=$(basename "$1")
|
||
[[ "$(cat "/sys/block/$base/queue/rotational" 2>/dev/null)" == "0" ]]
|
||
}
|
||
|
||
# Sets globals: UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX
|
||
# Falls back to master.conf values (SMART_TEMP_WARN/CRIT) if platform returns defaults.
|
||
get_unraid_temp_thresholds() {
|
||
read -r UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX \
|
||
<<< "$(platform_get_temp_thresholds)"
|
||
UNRAID_DISK_HOT="${UNRAID_DISK_HOT:-${SMART_TEMP_WARN:-45}}"
|
||
UNRAID_DISK_MAX="${UNRAID_DISK_MAX:-${SMART_TEMP_CRIT:-55}}"
|
||
UNRAID_SSD_HOT="${UNRAID_SSD_HOT:-60}"
|
||
UNRAID_SSD_MAX="${UNRAID_SSD_MAX:-70}"
|
||
}
|
||
|
||
# Checks local disk temperatures before rsync.
|
||
# Reads temps and rotational flag from /var/local/emhttp/disks.ini.
|
||
# Uses unRAID's own thresholds from dynamix.cfg via get_unraid_temp_thresholds().
|
||
#
|
||
# Returns:
|
||
# 0 = all temps OK — proceed
|
||
# 1 = warn threshold exceeded — skip this profile only
|
||
# 2 = critical threshold exceeded — abort all remaining profiles
|
||
#
|
||
# Sets TEMP_CHECK_RESULT with human readable summary for logging.
|
||
check_local_disk_temps() {
|
||
get_unraid_temp_thresholds
|
||
|
||
local disks_ini_content
|
||
if ! disks_ini_content=$(platform_get_disk_states); then
|
||
warn "disk state unavailable — skipping temp check"
|
||
TEMP_CHECK_RESULT="temp check skipped (disk state unavailable)"
|
||
return 0
|
||
fi
|
||
|
||
local worst_result=0
|
||
local hot_drives=()
|
||
local crit_drives=()
|
||
local current_name=""
|
||
local current_device=""
|
||
local current_rotational=""
|
||
local current_temp=""
|
||
|
||
check_drive() {
|
||
[[ -z "$current_name" ]] || [[ -z "$current_temp" ]] && return
|
||
[[ "$current_temp" -eq 0 ]] && return # spun down — skip
|
||
|
||
local warn_thresh crit_thresh
|
||
if [[ "$current_rotational" == "0" ]]; then
|
||
warn_thresh="$UNRAID_SSD_HOT"
|
||
crit_thresh="$UNRAID_SSD_MAX"
|
||
else
|
||
warn_thresh="$UNRAID_DISK_HOT"
|
||
crit_thresh="$UNRAID_DISK_MAX"
|
||
fi
|
||
|
||
if [[ "$current_temp" -ge "$crit_thresh" ]]; then
|
||
crit_drives+=("${current_name}(${current_device}):${current_temp}°C≥${crit_thresh}°C")
|
||
[[ $worst_result -lt 2 ]] && worst_result=2
|
||
elif [[ "$current_temp" -ge "$warn_thresh" ]]; then
|
||
hot_drives+=("${current_name}(${current_device}):${current_temp}°C≥${warn_thresh}°C")
|
||
[[ $worst_result -lt 1 ]] && worst_result=1
|
||
fi
|
||
}
|
||
|
||
while IFS= read -r ini_line; do
|
||
if echo "$ini_line" | grep -qE '^\["(disk[0-9]+|parity[0-9]?|cache[0-9]?)"\]'; then
|
||
check_drive
|
||
current_name=$(echo "$ini_line" | grep -o '"[^"]*"' | head -1 | tr -d '"')
|
||
current_device=""
|
||
current_rotational="1" # default HDD
|
||
current_temp=""
|
||
elif echo "$ini_line" | grep -q '^device='; then
|
||
current_device=$(echo "$ini_line" | cut -d= -f2 | tr -d '"')
|
||
elif echo "$ini_line" | grep -q '^rotational='; then
|
||
current_rotational=$(echo "$ini_line" | cut -d= -f2 | tr -d '"')
|
||
elif echo "$ini_line" | grep -q '^temp='; then
|
||
current_temp=$(echo "$ini_line" | cut -d= -f2 | tr -d '"')
|
||
current_temp="${current_temp//[^0-9]/}"
|
||
current_temp="${current_temp:-0}"
|
||
fi
|
||
done <<< "$disks_ini_content"
|
||
check_drive # process last drive
|
||
|
||
if [[ ${#crit_drives[@]} -gt 0 ]]; then
|
||
TEMP_CHECK_RESULT="CRITICAL temps: ${crit_drives[*]}"
|
||
error "Drive temp CRITICAL — aborting all remaining syncs: ${crit_drives[*]}"
|
||
notify "Rsync aborted on $(hostname) — drive temp CRITICAL: ${crit_drives[*]}" "Rsync Temp Check" "warning"
|
||
return 2
|
||
elif [[ ${#hot_drives[@]} -gt 0 ]]; then
|
||
TEMP_CHECK_RESULT="HOT drives: ${hot_drives[*]}"
|
||
warn "Drive temp WARNING — skipping this profile: ${hot_drives[*]}"
|
||
notify "Rsync profile skipped on $(hostname) — drive temp WARNING: ${hot_drives[*]}" "Rsync Temp Check" "normal"
|
||
return 1
|
||
else
|
||
TEMP_CHECK_RESULT="all normal (HDD warn:${UNRAID_DISK_HOT}°C crit:${UNRAID_DISK_MAX}°C SSD warn:${UNRAID_SSD_HOT}°C crit:${UNRAID_SSD_MAX}°C)"
|
||
return 0
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── REMOTE DISK VERIFICATION ──────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Verifies all disks backing a share are healthy on the remote server.
|
||
# Auto-detects filesystem type from disks.ini — handles XFS, ZFS, and cache pools.
|
||
# No configuration required — discovers disk layout at runtime.
|
||
#
|
||
# Three-tier detection handles all share locations:
|
||
# Tier 1: Array disk paths — /mnt/disk*/sharename (XFS or ZFS per-disk)
|
||
# Tier 2: ZFS standalone pools — finds nested paths up to 2 levels deep
|
||
# catches /mnt/cache/appdata-Fallback/Critical-Data correctly
|
||
# Tier 3: shfs fallback — if share exists under /mnt/user, resolve backing pool
|
||
# or verify shfs itself is mounted if pool can't be determined
|
||
#
|
||
# Fatal — exits the calling script if any disk is offline.
|
||
# Usage: check_remote_disks "/mnt/user/Movies"
|
||
|
||
check_remote_disks() {
|
||
local dir="$1"
|
||
local share_name
|
||
share_name=$(basename "$dir")
|
||
|
||
info "$ICON_DISK Checking disks backing $share_name on $REMOTE_SERVER_NAME..."
|
||
|
||
# Read remote disk state for fsType mapping
|
||
local disks_ini_content _disk_states_path
|
||
_disk_states_path=$(platform_disk_states_path)
|
||
disks_ini_content=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"cat '$_disk_states_path' 2>/dev/null" 2>/dev/null)
|
||
|
||
if [[ -z "$disks_ini_content" ]]; then
|
||
error "$ICON_DISK Cannot read disk state from $REMOTE_SERVER_NAME"
|
||
exit 1
|
||
fi
|
||
|
||
# Tier 1 — array disk paths (/mnt/disk*/sharename)
|
||
local backing_disks
|
||
backing_disks=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"ls -d /mnt/disk*/$share_name 2>/dev/null | awk -F/ '{print \$3}'" 2>/dev/null)
|
||
|
||
# Tier 2 — ZFS standalone pools, including nested paths
|
||
local zfs_pool_paths
|
||
zfs_pool_paths=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"zpool list -H -o name 2>/dev/null | while read pool; do
|
||
[[ -d \"/mnt/\${pool}/$share_name\" ]] && echo \"\$pool\" && continue
|
||
find \"/mnt/\${pool}\" -maxdepth 2 -name '$share_name' -type d 2>/dev/null | \
|
||
grep -q . && echo \"\$pool\"
|
||
done | sort -u" 2>/dev/null)
|
||
|
||
# Tier 3 — storage root fallback
|
||
local _spath="$REMOTE_STORAGE_PATH"
|
||
if [[ -z "$backing_disks" ]] && [[ -z "$zfs_pool_paths" ]]; then
|
||
local on_user
|
||
on_user=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"find '$_spath' -maxdepth 3 -name '$share_name' -type d 2>/dev/null | head -1" \
|
||
2>/dev/null)
|
||
if [[ -n "$on_user" ]]; then
|
||
local actual_path
|
||
actual_path=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"readlink -f '$on_user' 2>/dev/null || echo '$on_user'" 2>/dev/null)
|
||
local pool_name
|
||
pool_name=$(echo "$actual_path" | awk -F/ '{print $3}')
|
||
if [[ -n "$pool_name" ]] && [[ "$pool_name" != "user" ]]; then
|
||
zfs_pool_paths="$pool_name"
|
||
info "$ICON_DISK $share_name found at $actual_path on $REMOTE_SERVER_NAME"
|
||
else
|
||
# Can't determine pool — verify storage root itself is mounted
|
||
local shfs_ok
|
||
shfs_ok=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"mountpoint -q '$_spath' && echo yes || echo no" 2>/dev/null)
|
||
if [[ "$shfs_ok" == "yes" ]]; then
|
||
success "All disks backing $share_name are online ✅ (via storage root)"
|
||
return 0
|
||
fi
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
if [[ -z "$backing_disks" ]] && [[ -z "$zfs_pool_paths" ]]; then
|
||
error "$ICON_DISK No disks found backing $share_name on $REMOTE_SERVER_NAME"
|
||
exit 1
|
||
fi
|
||
|
||
local all_ok=true
|
||
|
||
# Check array disks (XFS or ZFS single-disk-in-array)
|
||
if [[ -n "$backing_disks" ]]; then
|
||
while IFS= read -r disk_name; do
|
||
[[ -z "$disk_name" ]] && continue
|
||
|
||
local fs_type
|
||
fs_type=$(echo "$disks_ini_content" | awk -F= -v disk="$disk_name" '
|
||
/^\["'"'"'?/ { current=substr($0,3,length($0)-4) }
|
||
current==disk && /^fsType=/ { print $2; exit }
|
||
' | tr -d '"')
|
||
fs_type="${fs_type:-xfs}"
|
||
|
||
if [[ "$fs_type" == "zfs" ]]; then
|
||
local pool_health
|
||
pool_health=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"zpool list -H -o health '$disk_name' 2>/dev/null" 2>/dev/null)
|
||
if [[ "$pool_health" == "ONLINE" ]]; then
|
||
info "$ICON_DISK $disk_name (ZFS) $ICON_RUNNING — $share_name ONLINE"
|
||
else
|
||
error "$ICON_DISK $disk_name (ZFS) $ICON_STOPPED — pool ${pool_health:-offline}"
|
||
all_ok=false
|
||
fi
|
||
else
|
||
local mounted
|
||
mounted=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"mountpoint -q '/mnt/$disk_name' && echo yes || echo no" 2>/dev/null)
|
||
if [[ "$mounted" == "yes" ]]; then
|
||
info "$ICON_DISK $disk_name (XFS) $ICON_RUNNING — $share_name present"
|
||
else
|
||
error "$ICON_DISK $disk_name (XFS) $ICON_STOPPED — not mounted"
|
||
all_ok=false
|
||
fi
|
||
fi
|
||
done <<< "$backing_disks"
|
||
fi
|
||
|
||
# Check ZFS standalone pools (cache, gaming, media-servers etc.)
|
||
if [[ -n "$zfs_pool_paths" ]]; then
|
||
while IFS= read -r pool_name; do
|
||
[[ -z "$pool_name" ]] && continue
|
||
local pool_health
|
||
pool_health=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"zpool list -H -o health '$pool_name' 2>/dev/null" 2>/dev/null)
|
||
if [[ "$pool_health" == "ONLINE" ]]; then
|
||
info "$ICON_DISK $pool_name (ZFS pool) $ICON_RUNNING — $share_name ONLINE"
|
||
else
|
||
error "$ICON_DISK $pool_name (ZFS pool) $ICON_STOPPED — pool ${pool_health:-offline}"
|
||
all_ok=false
|
||
fi
|
||
done <<< "$zfs_pool_paths"
|
||
fi
|
||
|
||
if [[ "$all_ok" == false ]]; then
|
||
error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME"
|
||
exit 1
|
||
fi
|
||
success "All disks backing $share_name are online ✅"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── CONTAINER MANAGEMENT ──────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Four functions — stop/start on both local and remote.
|
||
# State tracking: only containers that WERE running get restarted.
|
||
# Was stopped → stays stopped. Was running → gets restarted. ✅
|
||
|
||
# Tracks which remote containers were running before stop — used by start_containers()
|
||
RUNNING_CONTAINERS=()
|
||
|
||
# ── Docker command helpers (used by docker_daily_restart, docker_weekly_restart, docker_update) ──
|
||
|
||
# Default timeouts — scripts can override before calling.
|
||
DOCKER_TIMEOUT=${DOCKER_TIMEOUT:-30} # seconds before docker command is killed
|
||
RESTART_VERIFY_WAIT=${RESTART_VERIFY_WAIT:-5} # seconds to wait before checking state post-restart
|
||
|
||
# Run a docker command with a hard timeout. Returns 124 on timeout, else docker's exit code.
|
||
docker_cmd() {
|
||
timeout "$DOCKER_TIMEOUT" "$@"
|
||
local exit_code=$?
|
||
if [[ "$exit_code" -eq 124 ]]; then
|
||
error "Docker command timed out after ${DOCKER_TIMEOUT}s: $*"
|
||
return 1
|
||
fi
|
||
return "$exit_code"
|
||
}
|
||
|
||
# Verify a container is still running after a restart.
|
||
# Waits RESTART_VERIFY_WAIT seconds to let it settle, then checks State.Running.
|
||
verify_running() {
|
||
local container="$1"
|
||
sleep "$RESTART_VERIFY_WAIT"
|
||
local state
|
||
state=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||
if [[ "$state" != "true" ]]; then
|
||
error "$container failed to stay running after restart — may have crashed"
|
||
return 1
|
||
fi
|
||
return 0
|
||
}
|
||
|
||
# Retry a docker command up to RETRY_COUNT times, sleeping SLEEP seconds between attempts.
|
||
retry_docker() {
|
||
local attempt=1
|
||
while [[ "$attempt" -le "$RETRY_COUNT" ]]; do
|
||
log "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
|
||
if docker_cmd "$@"; then
|
||
log "Succeeded on attempt $attempt"
|
||
return 0
|
||
else
|
||
warn "Attempt $attempt failed"
|
||
(( attempt++ ))
|
||
[[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP"
|
||
fi
|
||
done
|
||
error "Command failed after $RETRY_COUNT attempts: $*"
|
||
return 1
|
||
}
|
||
|
||
# ── Emby API helper ───────────────────────────────────────────────────────────
|
||
# General Emby REST call. Uses EMBY_URL and EMBY_API_KEY (aliased by detect_hosts).
|
||
# Usage: emby_api <endpoint> [timeout_seconds]
|
||
# Prints response body on 200; returns 1 on HTTP error or curl failure.
|
||
emby_api() {
|
||
local endpoint="$1" max_time="${2:-30}"
|
||
local response http_code body
|
||
response=$(curl -sf --max-time "$max_time" \
|
||
-H "X-Emby-Token: $EMBY_API_KEY" \
|
||
-w "\n%{http_code}" \
|
||
"${EMBY_URL}/${endpoint}" 2>/dev/null)
|
||
http_code=$(echo "$response" | tail -1)
|
||
body=$(echo "$response" | head -n -1)
|
||
[[ "$http_code" != "200" ]] && { error "Emby API HTTP $http_code: $endpoint"; return 1; }
|
||
echo "$body"
|
||
}
|
||
|
||
# Stop containers on the REMOTE server via SSH.
|
||
# Reads CRITICAL_CONTAINER_NAMES — set from profile arrays by rsync.sh.
|
||
# Tracks running state in RUNNING_CONTAINERS for restart after sync.
|
||
stop_containers() {
|
||
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \
|
||
[[ "${CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then
|
||
log "No remote containers configured for this profile, skipping remote stop."
|
||
return
|
||
fi
|
||
echo "━━━ $ICON_STOP $ICON_CONTAINERS Containers ━━━"
|
||
info "Stopping remote containers..."
|
||
RUNNING_CONTAINERS=()
|
||
for c in "${CRITICAL_CONTAINER_NAMES[@]}"; do
|
||
[[ -z "$c" ]] && continue
|
||
local status
|
||
status=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"docker inspect -f '{{.State.Running}}' $c 2>/dev/null" 2>/dev/null || echo "unknown")
|
||
if [[ "$status" == "true" ]]; then
|
||
echo "$ICON_STOP Stopping remote $c..."
|
||
RUNNING_CONTAINERS+=("$c")
|
||
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $c" >/dev/null; then
|
||
echo "$ICON_STOPPED $c stopped"
|
||
else
|
||
error "Failed to stop remote $c"
|
||
fi
|
||
elif [[ "$status" == "false" ]]; then
|
||
echo "$ICON_NOT_RUNNING $c is not running — skipping"
|
||
else
|
||
log "$c not found on remote — skipping"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# Tracks which local containers were running before stop — used by start_local_containers()
|
||
LOCAL_RUNNING_CONTAINERS=()
|
||
|
||
# Stop containers on the LOCAL server.
|
||
# Reads LOCAL_CRITICAL_CONTAINER_NAMES — set from profile arrays by rsync.sh.
|
||
# Flushes local databases cleanly before pushing data to remote.
|
||
stop_local_containers() {
|
||
if [[ ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \
|
||
[[ "${LOCAL_CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then
|
||
log "No local containers configured for this profile, skipping local stop."
|
||
return
|
||
fi
|
||
info "Stopping local containers..."
|
||
LOCAL_RUNNING_CONTAINERS=()
|
||
for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]}"; do
|
||
[[ -z "$c" ]] && continue
|
||
local status
|
||
status=$(docker inspect -f '{{.State.Running}}' "$c" 2>/dev/null || echo "unknown")
|
||
if [[ "$status" == "true" ]]; then
|
||
echo "$ICON_STOP Stopping local $c..."
|
||
LOCAL_RUNNING_CONTAINERS+=("$c")
|
||
if docker stop "$c" >/dev/null; then
|
||
echo "$ICON_STOPPED $c stopped"
|
||
else
|
||
error "Failed to stop local $c"
|
||
fi
|
||
elif [[ "$status" == "false" ]]; then
|
||
echo "$ICON_NOT_RUNNING $c is not running — skipping"
|
||
else
|
||
log "$c not found locally — skipping"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# Restart containers on the REMOTE server that were running before sync.
|
||
# Reads RUNNING_CONTAINERS set by stop_containers().
|
||
# Respects DELAYED_CONTAINERS — waits CONTAINER_DELAY seconds before starting them.
|
||
start_containers() {
|
||
if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then
|
||
log "No remote containers to restart."
|
||
return
|
||
fi
|
||
info "Starting remote containers..."
|
||
for c in "${RUNNING_CONTAINERS[@]}"; do
|
||
[[ -z "$c" ]] && continue
|
||
local needs_delay=false
|
||
for d in "${DELAYED_CONTAINERS[@]}"; do
|
||
[[ "$c" == "$d" ]] && needs_delay=true && break
|
||
done
|
||
if [[ "$needs_delay" == true ]]; then
|
||
info "Waiting ${CONTAINER_DELAY}s before starting $c..."
|
||
sleep "$CONTAINER_DELAY"
|
||
fi
|
||
echo "$ICON_START Starting remote $c..."
|
||
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $c" >/dev/null 2>&1; then
|
||
echo "$ICON_STARTED $c started"
|
||
else
|
||
error "Failed to start remote $c — start manually if needed"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# Restart containers on the LOCAL server that were running before sync.
|
||
# Reads LOCAL_RUNNING_CONTAINERS set by stop_local_containers().
|
||
# Respects DELAYED_CONTAINERS and CONTAINER_DELAY same as remote.
|
||
start_local_containers() {
|
||
if [[ ${#LOCAL_RUNNING_CONTAINERS[@]} -eq 0 ]]; then
|
||
log "No local containers to restart."
|
||
return
|
||
fi
|
||
info "Starting local containers..."
|
||
for c in "${LOCAL_RUNNING_CONTAINERS[@]}"; do
|
||
[[ -z "$c" ]] && continue
|
||
local needs_delay=false
|
||
for d in "${DELAYED_CONTAINERS[@]}"; do
|
||
[[ "$c" == "$d" ]] && needs_delay=true && break
|
||
done
|
||
if [[ "$needs_delay" == true ]]; then
|
||
info "Waiting ${CONTAINER_DELAY}s before starting local $c..."
|
||
sleep "$CONTAINER_DELAY"
|
||
fi
|
||
echo "$ICON_START Starting local $c..."
|
||
if docker start "$c" >/dev/null 2>&1; then
|
||
echo "$ICON_STARTED $c started"
|
||
else
|
||
error "Failed to start local $c — start manually if needed"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── RSYNC OPTIONS ─────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Loads rsync options for the current profile.
|
||
# Falls back to DEFAULT_RSYNC_OPTS from master.conf if no profile match found.
|
||
# Profile opts do NOT inherit from defaults — list all desired flags explicitly in master.conf.
|
||
|
||
get_rsync_opts() {
|
||
if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then
|
||
read -r -a RSYNC_OPTS <<< "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]}"
|
||
log "Using profile rsync opts for $PROFILE_NAME: ${RSYNC_OPTS[*]}"
|
||
else
|
||
RSYNC_OPTS=("${DEFAULT_RSYNC_OPTS[@]}")
|
||
log "Using default rsync opts: ${RSYNC_OPTS[*]}"
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── SCRIPT LOCKING ────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Prevents multiple instances of the same script running simultaneously.
|
||
# All lock files live in /tmp/unraid_locks/ — auto-cleared on reboot.
|
||
#
|
||
# Modes:
|
||
# strict — exit immediately if already running (default)
|
||
# wait — wait LOCK_WAIT_TIMEOUT seconds then exit if still locked
|
||
# continuous — for long-running scripts: skip gracefully if healthy instance running
|
||
#
|
||
# Features:
|
||
# Stale lock detection — if lock PID is dead, clears and proceeds
|
||
# PID reuse protection — lock stores PID:scriptname, validates both
|
||
# Age warning — warns if lock older than LOCK_WARN_AGE (skipped for continuous)
|
||
# EXIT trap — lock always released on exit, crash, or kill signal
|
||
|
||
LOCK_DIR="/tmp/unraid_locks"
|
||
|
||
# Ensure DATA_DIR and STATE_DIR exist — created here so every script that sources common.sh
|
||
# can safely write to either without checking first.
|
||
if [[ -n "${DATA_DIR:-}" ]] && [[ ! -d "$DATA_DIR" ]]; then
|
||
mkdir -p "$DATA_DIR" 2>/dev/null || true
|
||
fi
|
||
if [[ -n "${STATE_DIR:-}" ]] && [[ ! -d "$STATE_DIR" ]]; then
|
||
mkdir -p "$STATE_DIR" 2>/dev/null || true
|
||
fi
|
||
|
||
RSYNC_COUNT_FILE="$LOCK_DIR/rsync_active_count"
|
||
RSYNC_MAX_CONCURRENT=3
|
||
LOCK_WARN_AGE=300 # seconds — warn if lock older than this (5min default)
|
||
LOCK_WAIT_TIMEOUT=30 # seconds — how long "wait" mode waits before giving up
|
||
|
||
# Registry of all lock files acquired by this process — released together on exit.
|
||
# Prevents the single-trap-per-acquire problem: each new acquire_lock/acquire_rsync_lock
|
||
# call used to overwrite the EXIT trap, orphaning the previous lock file.
|
||
declare -ga _LOCK_FILES=()
|
||
_release_all_locks() {
|
||
local lf
|
||
for lf in "${_LOCK_FILES[@]}"; do
|
||
[[ -f "$lf" ]] && rm -f "$lf"
|
||
done
|
||
}
|
||
_register_lock() {
|
||
_LOCK_FILES+=("$1")
|
||
trap '_release_all_locks' EXIT
|
||
}
|
||
|
||
# Internal — script name used as lock identifier
|
||
_lock_name() {
|
||
basename "${BASH_SOURCE[1]:-$0}" .sh
|
||
}
|
||
|
||
# Internal — lock file path for this script
|
||
_lock_file() {
|
||
echo "$LOCK_DIR/${1:-$(_lock_name)}.lock"
|
||
}
|
||
|
||
# Internal — release lock on exit (registered via EXIT trap)
|
||
_release_on_exit() {
|
||
local lockfile="$1"
|
||
[[ -f "$lockfile" ]] && rm -f "$lockfile"
|
||
}
|
||
|
||
# Acquire exclusive lock for this script.
|
||
# Usage: acquire_lock [strict|wait|continuous]
|
||
acquire_lock() {
|
||
local mode="${1:-strict}"
|
||
local script_name
|
||
script_name=$(basename "${BASH_SOURCE[1]:-$0}" .sh)
|
||
local lockfile
|
||
lockfile="$(_lock_file "$script_name")"
|
||
|
||
mkdir -p "$LOCK_DIR"
|
||
|
||
if [[ -f "$lockfile" ]]; then
|
||
local lock_content existing_pid locked_name
|
||
lock_content=$(cat "$lockfile" 2>/dev/null)
|
||
existing_pid="${lock_content%%:*}"
|
||
locked_name="${lock_content##*:}"
|
||
|
||
# Stale lock — PID dead
|
||
if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then
|
||
warn "Stale lock detected for $script_name (PID $existing_pid gone) — clearing"
|
||
rm -f "$lockfile"
|
||
|
||
# PID reuse — PID alive but belongs to different process
|
||
elif [[ "$locked_name" != "$script_name" ]]; then
|
||
warn "Lock PID $existing_pid reused by different process ($locked_name ≠ $script_name) — clearing"
|
||
rm -f "$lockfile"
|
||
|
||
else
|
||
# Lock is genuinely active — check age
|
||
local lock_age
|
||
lock_age=$(( $(date +%s) - $(stat -c %Y "$lockfile" 2>/dev/null || echo 0) ))
|
||
if [[ "$lock_age" -gt "$LOCK_WARN_AGE" ]] && [[ "$mode" != "continuous" ]]; then
|
||
warn "$script_name has been running for ${lock_age}s — may be stuck (PID $existing_pid)"
|
||
fi
|
||
|
||
if [[ "$mode" == "continuous" ]]; then
|
||
log "$ICON_SKIP $script_name already running healthy (PID $existing_pid) — skipping"
|
||
exit 0
|
||
elif [[ "$mode" == "wait" ]]; then
|
||
info "Another instance of $script_name is running — waiting up to ${LOCK_WAIT_TIMEOUT}s"
|
||
local waited=0
|
||
while [[ -f "$lockfile" ]] && [[ "$waited" -lt "$LOCK_WAIT_TIMEOUT" ]]; do
|
||
sleep 1
|
||
((waited++))
|
||
lock_content=$(cat "$lockfile" 2>/dev/null)
|
||
existing_pid="${lock_content%%:*}"
|
||
locked_name="${lock_content##*:}"
|
||
if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then
|
||
warn "Lock became stale while waiting — clearing"
|
||
rm -f "$lockfile"
|
||
break
|
||
elif [[ "$locked_name" != "$script_name" ]]; then
|
||
warn "Lock PID reused while waiting — clearing"
|
||
rm -f "$lockfile"
|
||
break
|
||
fi
|
||
done
|
||
if [[ -f "$lockfile" ]]; then
|
||
error "$script_name still locked after ${LOCK_WAIT_TIMEOUT}s — exiting"
|
||
exit 1
|
||
fi
|
||
else
|
||
error "Another instance of $script_name is already running (PID $existing_pid) — exiting"
|
||
case "$script_name" in
|
||
fallback|transcode_management|daily_sync_maintenance|stability_watchdog)
|
||
notify "$script_name lock collision on $(hostname) — concurrent instance detected" "$script_name" "warning"
|
||
;;
|
||
esac
|
||
exit 1
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
# Acquire lock — store PID:scriptname to prevent PID reuse false positives
|
||
echo "$$:$script_name" > "$lockfile"
|
||
_register_lock "$lockfile"
|
||
log "$ICON_LOCK Lock acquired: $script_name (PID $$)"
|
||
}
|
||
|
||
# Acquire per-profile rsync lock + enforce global concurrent limit.
|
||
# Prevents same profile running twice and limits total concurrent rsync instances.
|
||
# Usage: acquire_rsync_lock "$PROFILE_NAME"
|
||
acquire_rsync_lock() {
|
||
local profile="$1"
|
||
local profile_lock
|
||
profile_lock="$(_lock_file "rsync_${profile}")"
|
||
|
||
mkdir -p "$LOCK_DIR"
|
||
|
||
# Per-profile lock — same profile cannot run twice
|
||
if [[ -f "$profile_lock" ]]; then
|
||
local lock_content existing_pid locked_name
|
||
lock_content=$(cat "$profile_lock" 2>/dev/null)
|
||
existing_pid="${lock_content%%:*}"
|
||
locked_name="${lock_content##*:}"
|
||
if [[ -n "$existing_pid" ]] && kill -0 "$existing_pid" 2>/dev/null && \
|
||
[[ "$locked_name" == "rsync_${profile}" ]]; then
|
||
error "rsync profile '$profile' is already running (PID $existing_pid) — exiting"
|
||
exit 1
|
||
else
|
||
warn "Stale rsync lock for profile '$profile' — clearing"
|
||
rm -f "$profile_lock"
|
||
fi
|
||
fi
|
||
|
||
# Global concurrent limit — validate count against actual live locks
|
||
local current_count=0
|
||
if [[ -f "$RSYNC_COUNT_FILE" ]]; then
|
||
current_count=$(cat "$RSYNC_COUNT_FILE" 2>/dev/null || echo 0)
|
||
local actual_count=0
|
||
for lf in "$LOCK_DIR"/rsync_*.lock; do
|
||
[[ -f "$lf" ]] || continue
|
||
local lpid lf_content
|
||
lf_content=$(cat "$lf" 2>/dev/null)
|
||
lpid="${lf_content%%:*}"
|
||
kill -0 "$lpid" 2>/dev/null && ((actual_count++))
|
||
done
|
||
if [[ "$actual_count" -ne "$current_count" ]]; then
|
||
log "rsync count corrected: $current_count → $actual_count"
|
||
current_count=$actual_count
|
||
echo "$current_count" > "$RSYNC_COUNT_FILE"
|
||
fi
|
||
fi
|
||
|
||
if [[ "$current_count" -ge "$RSYNC_MAX_CONCURRENT" ]]; then
|
||
error "Maximum concurrent rsync limit ($RSYNC_MAX_CONCURRENT) reached — exiting"
|
||
info "Active rsync locks: $(ls "$LOCK_DIR"/rsync_*.lock 2>/dev/null | \
|
||
xargs -I{} basename {} .lock | tr '\n' ' ')"
|
||
exit 1
|
||
fi
|
||
|
||
# Acquire profile lock and increment counter
|
||
echo "$$:rsync_${profile}" > "$profile_lock"
|
||
echo $(( current_count + 1 )) > "$RSYNC_COUNT_FILE"
|
||
_register_lock "$profile_lock"
|
||
log "$ICON_LOCK rsync lock acquired: profile '$profile' (PID $$, active: $(( current_count + 1 ))/$RSYNC_MAX_CONCURRENT)"
|
||
}
|
||
|
||
# Internal — release rsync lock and decrement global counter on exit
|
||
_release_rsync_on_exit() {
|
||
local profile_lock="$1"
|
||
[[ -f "$profile_lock" ]] && rm -f "$profile_lock"
|
||
if [[ -f "$RSYNC_COUNT_FILE" ]]; then
|
||
local count
|
||
count=$(cat "$RSYNC_COUNT_FILE" 2>/dev/null || echo 1)
|
||
count=$(( count - 1 ))
|
||
[[ "$count" -lt 0 ]] && count=0
|
||
echo "$count" > "$RSYNC_COUNT_FILE"
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── ORCHESTRATOR CHILD EXECUTION ──────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Shared child-script runner for Orchestrators/*.sh. Replaces the run_job()/run_watchdog()
|
||
# copies that used to be hand-duplicated (with small inconsistencies) into each orchestrator —
|
||
# one implementation now, one place to fix a bug or extend behavior (e.g. per-child logging).
|
||
#
|
||
# Usage — caller declares its own tracking arrays before the loop:
|
||
# JOB_PASS=()
|
||
# JOB_FAIL=()
|
||
# for entry in "${SOME_SCRIPTS[@]}"; do
|
||
# run_orch_child "$entry"
|
||
# done
|
||
#
|
||
# $entry is "relative/path.sh [extra args...]" — the same format every master.conf
|
||
# script-list array already uses. Resolved against $ECOSYSTEM_ROOT, which the caller
|
||
# must set before calling this (the absolute repo root — "$(cd "$SCRIPT_DIR/.." && pwd)").
|
||
#
|
||
# --dry-run / --log are threaded down automatically from $DRY_RUN / $ENABLE_LOGGING —
|
||
# never $VERBOSE, which nothing in this codebase ever assigns.
|
||
|
||
run_orch_child() {
|
||
local entry="$1"
|
||
local script_args script_path script_name label extra_args run_args
|
||
|
||
read -r -a script_args <<< "$entry"
|
||
script_path="$ECOSYSTEM_ROOT/${script_args[0]}"
|
||
script_name=$(basename "${script_args[0]}")
|
||
extra_args=("${script_args[@]:1}")
|
||
label="$script_name"
|
||
[[ -n "${extra_args[*]}" ]] && label="$script_name ${extra_args[*]}"
|
||
|
||
if [[ ! -f "$script_path" ]]; then
|
||
error "$script_name — not found at $script_path"
|
||
JOB_FAIL+=("$label")
|
||
return 1
|
||
fi
|
||
[[ ! -x "$script_path" ]] && chmod +x "$script_path"
|
||
|
||
run_args=("${extra_args[@]}")
|
||
[[ "$DRY_RUN" == true ]] && run_args+=("--dry-run")
|
||
[[ "$ENABLE_LOGGING" == true ]] && run_args+=("--log")
|
||
|
||
local _start _ec
|
||
_start=$(date +%s)
|
||
log "Running: $label"
|
||
if bash "$script_path" "${run_args[@]}"; then
|
||
log "$script_name — done in $(format_duration $(( $(date +%s) - _start )))"
|
||
JOB_PASS+=("$label")
|
||
return 0
|
||
else
|
||
_ec=$?
|
||
error "$label — failed (exit $_ec, $(format_duration $(( $(date +%s) - _start ))))"
|
||
JOB_FAIL+=("$label")
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── PATH TRANSLATION ──────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Translates container-internal file paths to host paths for arr cleanup scripts.
|
||
# Arr APIs return file paths as seen inside the container — scripts need host paths to scan.
|
||
#
|
||
# Uses ARR_PATH_MAP associative array — must be declared before calling.
|
||
# Longest-match wins — prevents partial path collisions.
|
||
#
|
||
# Usage:
|
||
# declare -A ARR_PATH_MAP=(["/ext-music"]="/mnt/user/Music")
|
||
# translate_path "/ext-music/Artist/Album/track.flac"
|
||
# Returns: /mnt/user/Music/Artist/Album/track.flac
|
||
#
|
||
# If no match found — returns path unchanged (safe fallback).
|
||
|
||
translate_path() {
|
||
local api_path="$1"
|
||
local best_match=""
|
||
local best_len=0
|
||
|
||
for container_path in "${!ARR_PATH_MAP[@]}"; do
|
||
if [[ "$api_path" == "$container_path"* ]]; then
|
||
if [[ "${#container_path}" -gt "$best_len" ]]; then
|
||
best_match="$container_path"
|
||
best_len="${#container_path}"
|
||
fi
|
||
fi
|
||
done
|
||
|
||
if [[ -n "$best_match" ]]; then
|
||
echo "${ARR_PATH_MAP[$best_match]}${api_path#$best_match}"
|
||
else
|
||
echo "$api_path" # no match — return unchanged
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── ARR VERSION CHECK ─────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Verifies arr major version matches tested version in master.conf.
|
||
# Exits if version doesn't match — prevents running against untested API structure.
|
||
# Warns and proceeds if version endpoint is unreachable.
|
||
#
|
||
# Usage:
|
||
# check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" "Radarr"
|
||
#
|
||
# Arguments:
|
||
# $1 = arr base URL
|
||
# $2 = API key
|
||
# $3 = API path prefix (v3 or v1)
|
||
# $4 = expected major version number from master.conf
|
||
# $5 = arr name for error messages
|
||
|
||
check_arr_version() {
|
||
local url="$1"
|
||
local api_key="$2"
|
||
local api_prefix="$3"
|
||
local expected_major="$4"
|
||
local arr_name="${5:-Arr}"
|
||
|
||
local status_response version major_version
|
||
|
||
status_response=$(curl -sf --max-time 10 \
|
||
-H "X-Api-Key: $api_key" \
|
||
"${url}/api/${api_prefix}/system/status" 2>/dev/null)
|
||
|
||
if [[ -z "$status_response" ]]; then
|
||
warn "$arr_name version check failed — could not reach system/status endpoint"
|
||
warn "Proceeding without version verification — monitor for API errors"
|
||
return 0
|
||
fi
|
||
|
||
version=$(echo "$status_response" | \
|
||
grep -o '"version": *"[^"]*"' | \
|
||
grep -o '[0-9][^"]*' | head -1)
|
||
|
||
if [[ -z "$version" ]]; then
|
||
warn "$arr_name version check failed — could not parse version from response"
|
||
warn "Proceeding without version verification — monitor for API errors"
|
||
return 0
|
||
fi
|
||
|
||
major_version="${version%%.*}"
|
||
|
||
if [[ "$major_version" == "$expected_major" ]]; then
|
||
success "$arr_name version: $version (major $major_version — tested ✅)"
|
||
return 0
|
||
else
|
||
error "$arr_name version mismatch — running v${major_version}, tested against v${expected_major}"
|
||
error "The API endpoint structure may have changed — exiting to protect your library"
|
||
error "Update ${arr_name^^}_VERSION_MAJOR in master.conf after verifying the script works with v${major_version}"
|
||
notify "$arr_name version mismatch on $(hostname) — running v${major_version}, script tested against v${expected_major}" "$arr_name Cleanup" "warning"
|
||
exit 1
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── EMBY LIBRARY SCAN ─────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Triggers Emby "Clean Missing Files" scheduled task after arr cleanup scripts delete files.
|
||
# Without this Emby keeps showing deleted files as ghost entries until its next scheduled scan.
|
||
# User clicks a ghost entry → file not found error. This prevents that entirely.
|
||
#
|
||
# Uses EMBY_URL and EMBY_API_KEY — aliased by detect_hosts() from HOST*_EMBY_* vars.
|
||
# Skips cleanly if Emby is not configured on this host (empty URL or API key).
|
||
#
|
||
# Emby task discovery:
|
||
# GET /ScheduledTasks → find task where Name contains "Clean Missing"
|
||
# POST /ScheduledTasks/Running/{taskId} → trigger it
|
||
#
|
||
# Non-fatal — logs warning if task cannot be triggered but does not exit.
|
||
# Clean Missing Files runs across all libraries at once — one call handles
|
||
# music, TV, and movies together regardless of which arr triggered the cleanup.
|
||
#
|
||
# Usage: notify_emby_scan (no args — always triggers Clean Missing Files)
|
||
|
||
notify_emby_scan() {
|
||
# Skip if Emby not configured on this host
|
||
if [[ -z "${EMBY_URL:-}" ]] || [[ -z "${EMBY_API_KEY:-}" ]]; then
|
||
log "Emby not configured on $MY_ID — skipping library scan notification"
|
||
return 0
|
||
fi
|
||
|
||
log "Notifying Emby to clean missing files..."
|
||
|
||
# Get all scheduled tasks
|
||
local tasks_response
|
||
tasks_response=$(curl -sf --max-time 15 \
|
||
-H "X-Emby-Token: $EMBY_API_KEY" \
|
||
"${EMBY_URL}/ScheduledTasks" 2>/dev/null)
|
||
|
||
if [[ -z "$tasks_response" ]]; then
|
||
warn "Could not reach Emby scheduled tasks API at $EMBY_URL — skipping scan"
|
||
return 0
|
||
fi
|
||
|
||
# Find the "Clean Missing Files" task ID
|
||
local task_id
|
||
task_id=$(echo "$tasks_response" | \
|
||
grep -o '"Id":"[^"]*","Name":"[^"]*Clean Missing[^"]*"' | \
|
||
grep -o '"Id":"[^"]*"' | \
|
||
sed 's/"Id":"//;s/"//' | head -1)
|
||
|
||
# Fallback — try "Scan Media Library" if Clean Missing not found
|
||
if [[ -z "$task_id" ]]; then
|
||
task_id=$(echo "$tasks_response" | \
|
||
grep -o '"Id":"[^"]*","Name":"[^"]*Scan Media Library[^"]*"' | \
|
||
grep -o '"Id":"[^"]*"' | \
|
||
sed 's/"Id":"//;s/"//' | head -1)
|
||
[[ -n "$task_id" ]] && log "Clean Missing Files not found — using Scan Media Library"
|
||
fi
|
||
|
||
if [[ -z "$task_id" ]]; then
|
||
warn "Could not find Emby Clean Missing Files or Scan Media Library task"
|
||
warn "Check Emby scheduled tasks — ghost entries will persist until next Emby scan"
|
||
return 0
|
||
fi
|
||
|
||
log "Found Emby task ID: $task_id"
|
||
|
||
# Trigger the task
|
||
local http_code
|
||
http_code=$(curl -sf --max-time 15 \
|
||
-X POST \
|
||
-H "X-Emby-Token: $EMBY_API_KEY" \
|
||
-w "%{http_code}" \
|
||
-o /dev/null \
|
||
"${EMBY_URL}/ScheduledTasks/Running/${task_id}" 2>/dev/null)
|
||
|
||
if [[ "$http_code" == "204" ]] || [[ "$http_code" == "200" ]]; then
|
||
warn "$ICON_EMBY Emby Clean Missing Files triggered — ghost entries will be removed"
|
||
else
|
||
warn "Emby task trigger returned HTTP $http_code — ghost entries may persist"
|
||
warn "Check Emby dashboard — manually run Clean Missing Files if needed"
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── API REACHABILITY CHECK ────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Pre-flight check — verifies an API endpoint is reachable before attempting operations.
|
||
# Non-fatal — returns 0 if reachable, 1 if not. Caller decides whether to exit.
|
||
#
|
||
# Usage: check_api "http://localhost:8989" "Sonarr" || exit 1
|
||
|
||
check_api() {
|
||
local url="$1"
|
||
local service="${2:-API}"
|
||
local timeout="${3:-10}"
|
||
|
||
if curl -sf --max-time "$timeout" "$url" >/dev/null 2>&1; then
|
||
log "$service API reachable: $url"
|
||
return 0
|
||
else
|
||
error "$service API not reachable: $url"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── UNRAID VERSION PARITY CHECK ───────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Verifies local and remote servers are running compatible unRAID versions before any
|
||
# remote operation. Version mismatches can mean changed APIs, commands, or behaviours
|
||
# that silently break remote container operations, rsync, or fallback logic.
|
||
#
|
||
# Reads platform OS version on both sides via adapter.
|
||
#
|
||
# Mismatch behaviour (UNRAID_VERSION_MISMATCH_ACTION in master.conf):
|
||
# "warn" — log warning and continue (default for minor/patch differences)
|
||
# "abort" — exit the calling script (default for major version differences)
|
||
# Major version mismatch always aborts regardless of setting
|
||
#
|
||
# Usage: check_os_version_parity || exit 1
|
||
|
||
check_os_version_parity() {
|
||
local local_version remote_version
|
||
|
||
# Read local version
|
||
local_version=$(platform_get_os_version 2>/dev/null)
|
||
if [[ -z "$local_version" ]]; then
|
||
warn "Cannot read local OS version — skipping version parity check"
|
||
return 0
|
||
fi
|
||
|
||
# Read remote version via SSH
|
||
local _probe_cmd
|
||
_probe_cmd=$(platform_os_version_probe_cmd)
|
||
remote_version=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
||
"$_probe_cmd" 2>/dev/null)
|
||
if [[ -z "$remote_version" ]]; then
|
||
warn "Cannot read remote OS version from $REMOTE_SERVER_NAME — skipping parity check"
|
||
return 0
|
||
fi
|
||
|
||
log "Version parity: local=$local_version remote=$remote_version"
|
||
|
||
if [[ "$local_version" == "$remote_version" ]]; then
|
||
log "OS versions match: $local_version ✅"
|
||
return 0
|
||
fi
|
||
|
||
# Parse major versions
|
||
local local_major remote_major
|
||
local_major="${local_version%%.*}"
|
||
remote_major="${remote_version%%.*}"
|
||
|
||
if [[ "$local_major" != "$remote_major" ]]; then
|
||
error "OS MAJOR version mismatch — local: $local_version remote: $remote_version"
|
||
error "Major version differences may break remote APIs, commands, and behaviours"
|
||
error "Update both servers to the same major version before running remote operations"
|
||
notify "OS major version mismatch on $(hostname) — local: $local_version remote: $remote_version — remote operations aborted" \
|
||
"Version Parity" "warning"
|
||
return 1
|
||
fi
|
||
|
||
# Minor/patch mismatch — action depends on config
|
||
local action="${UNRAID_VERSION_MISMATCH_ACTION:-warn}"
|
||
warn "OS version mismatch — local: $local_version remote: $remote_version"
|
||
|
||
if [[ "$action" == "abort" ]]; then
|
||
error "UNRAID_VERSION_MISMATCH_ACTION=abort — refusing to continue"
|
||
notify "OS version mismatch on $(hostname) — local: $local_version remote: $remote_version — aborted" \
|
||
"Version Parity" "warning"
|
||
return 1
|
||
fi
|
||
|
||
warn "UNRAID_VERSION_MISMATCH_ACTION=warn — continuing despite mismatch (monitor for issues)"
|
||
return 0
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── REMOTE DOCKER DAEMON CHECK ────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Verifies the remote Docker daemon is responsive before issuing any remote container commands.
|
||
# A hung remote daemon causes docker commands to hang or silently fail — checking first
|
||
# prevents cascading failures and gives clear diagnostic output.
|
||
#
|
||
# Strike system:
|
||
# Each consecutive failed check adds a strike to REMOTE_DOCKER_STRIKES
|
||
# Below REMOTE_DOCKER_STRIKE_LIMIT → warn and return 1 (caller skips operation)
|
||
# At limit → notify critical and return 1 (caller should exit or escalate)
|
||
# Strikes reset when daemon recovers
|
||
#
|
||
# Usage: check_remote_docker_daemon || return 1
|
||
|
||
REMOTE_DOCKER_STRIKES=0
|
||
|
||
check_remote_docker_daemon() {
|
||
local timeout="${1:-10}"
|
||
|
||
if ssh -i "$SSH_KEY" -o ConnectTimeout="$timeout" root@"$REMOTE_SERVER" \
|
||
"timeout $timeout docker info" >/dev/null 2>&1; then
|
||
# Daemon healthy — clear strikes if we had issues
|
||
if [[ "$REMOTE_DOCKER_STRIKES" -gt 0 ]]; then
|
||
info "Remote Docker daemon on $REMOTE_SERVER_NAME recovered — clearing strikes"
|
||
REMOTE_DOCKER_STRIKES=0
|
||
fi
|
||
log "Remote Docker daemon healthy on $REMOTE_SERVER_NAME"
|
||
return 0
|
||
fi
|
||
|
||
REMOTE_DOCKER_STRIKES=$(( REMOTE_DOCKER_STRIKES + 1 ))
|
||
local limit="${REMOTE_DOCKER_STRIKE_LIMIT:-3}"
|
||
warn "Remote Docker daemon not responding on $REMOTE_SERVER_NAME (strike $REMOTE_DOCKER_STRIKES/$limit)"
|
||
|
||
if [[ "$REMOTE_DOCKER_STRIKES" -ge "$limit" ]]; then
|
||
error "Remote Docker daemon unresponsive on $REMOTE_SERVER_NAME after $limit consecutive checks"
|
||
error "Skipping all remote container operations — check Docker on $REMOTE_SERVER_NAME"
|
||
notify "Remote Docker daemon unresponsive on $REMOTE_SERVER_NAME — remote operations skipped on $(hostname)" "Remote Docker Daemon" "warning"
|
||
fi
|
||
|
||
return 1
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── UNRAID COMMAND VALIDATION ─────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Verifies a unRAID-specific command exists and produces expected output before use.
|
||
# Protects against commands moving, changing format, or disappearing after upgrades.
|
||
#
|
||
# If validation fails — notifies and returns 1. Caller exits its own section only.
|
||
# Other scripts that pass validation are unaffected.
|
||
#
|
||
# Usage:
|
||
# validate_unraid_cmd # "/usr/local/emhttp/plugins/dynamix/scripts/notify" # "--help" # "Usage" # "unRAID notify script"
|
||
#
|
||
# Arguments:
|
||
# $1 = full path to command
|
||
# $2 = test argument to pass (use "" for no argument)
|
||
# $3 = expected string in output (use "" to skip output check)
|
||
# $4 = human readable name for error messages
|
||
|
||
validate_unraid_cmd() {
|
||
local cmd_path="$1"
|
||
local test_arg="$2"
|
||
local expected_output="$3"
|
||
local cmd_name="${4:-$1}"
|
||
|
||
# Check command exists and is executable
|
||
if [[ ! -x "$cmd_path" ]]; then
|
||
error "unRAID command not found or not executable: $cmd_path"
|
||
error "$cmd_name may have moved or been removed — check after recent unRAID upgrade"
|
||
notify "$cmd_name not found on $(hostname) at $cmd_path — check after unRAID upgrade" "Command Validation" "warning"
|
||
return 1
|
||
fi
|
||
|
||
# Check output matches expected pattern if provided
|
||
if [[ -n "$expected_output" ]]; then
|
||
local actual_output
|
||
actual_output=$(timeout 10 "$cmd_path" $test_arg 2>&1 || true)
|
||
if ! echo "$actual_output" | grep -q "$expected_output"; then
|
||
error "$cmd_name output format changed — expected '$expected_output' not found"
|
||
error "Command may have changed after unRAID upgrade — review $cmd_path"
|
||
notify "$cmd_name output format changed on $(hostname) — may need script update after unRAID upgrade" "Command Validation" "warning"
|
||
return 1
|
||
fi
|
||
fi
|
||
|
||
log "$cmd_name validated: $cmd_path ✅"
|
||
return 0
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── STATUS DISPLAY ────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Prints current runtime configuration — triggered by --status flag in any script.
|
||
# Useful for verifying detect_hosts() resolved correctly and profile loaded as expected.
|
||
|
||
show_status() {
|
||
local local_ver
|
||
local_ver=$(platform_get_os_version 2>/dev/null || echo "unknown")
|
||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||
echo "Local: $LOCAL_SERVER_NAME"
|
||
echo "Remote: $REMOTE_SERVER_NAME"
|
||
echo "IP: ${REMOTE_SERVER:-not resolved}"
|
||
echo "My ID: ${MY_ID:-not set}"
|
||
echo "Remote ID: ${REMOTE_ID:-not set}"
|
||
echo "OS ver: $local_ver"
|
||
echo "Profile: ${PROFILE_NAME:-n/a}"
|
||
|
||
echo "DryRun: ${DRY_RUN:-false}"
|
||
echo "Logging: ${ENABLE_LOGGING:-false}"
|
||
echo "SSH Key: ${SSH_KEY:-not set}"
|
||
echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]:-n/a}"
|
||
echo "Delayed: ${DELAYED_CONTAINERS[*]:-n/a}"
|
||
echo "Excludes: ${EXCLUDE_DIRS[*]:-n/a}"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
}
|
||
|
||
require_partnership() {
|
||
[[ "${PARTNERSHIP_ENABLED:-false}" == "true" ]] && return 0
|
||
log "PARTNERSHIP_ENABLED=false — skipping cross-server operation"
|
||
exit 0
|
||
} |