#!/bin/bash # ============================================================================================== # ================================= COMMON LIBRARY ============================================= # ============================================================================================== # Shared functions used by every script in the ecosystem. # Sourced automatically by load_config.sh — do not source directly. # # ── WHAT THIS FILE PROVIDES ─────────────────────────────────────────────────────────────────── # Icons — consistent visual language across all script output # Output helpers — info, warn, error, success, log, notify # Arg parser — unified --dry-run, --log, --status, KEY=VALUE handling # Validation — require_var, validate_int # Host detection — detect_hosts() identifies local/remote server, sets all aliases # Connectivity — resolve_remote_ip, check_connectivity, ping_remote, ping_internet # Health checks — local/remote array, docker daemon, rootfs, share, disk temps # Disk verification — check_remote_disks() — XFS, ZFS, cache pools, nested paths # Container mgmt — stop/start local and remote containers with state tracking # Rsync helpers — get_rsync_opts, check_rsync_enabled # Locking — acquire_lock, acquire_rsync_lock — concurrent execution protection # Path translation — translate_path() — container → host path for arr cleanup scripts # Arr utilities — check_arr_version() — API version safety gate # Status display — show_status() — runtime config dump on --status flag # # ── CHANGELOG ───────────────────────────────────────────────────────────────────────────────── # # Version What changed # ─────────────────────────────────────────────────────────────────────────────────────────── # v1.0 Initial stable framework — output helpers, arg parser, host detection # # v1.1 format_duration() moved here from daily_sync_maintenance.sh for shared use # SSH_KEY naming collision resolved — gitea key renamed GITEA_SSH_KEY in master.conf # # v1.2 check_connectivity() added — fatal ping with Tailscale hint on failure # Function header comment blocks standardised across all functions # # v1.3 check_remote_rootfs() — aborts rsync if remote rootfs exceeds ROOTFS_WARN # check_remote_share() — aborts if target directory missing or empty on remote # Both protect against rsync running when remote array is down # # v1.4 check_remote_disks() — verifies all physical disks backing a share are mounted # Auto-discovers disk layout at runtime from disks.ini — no config required # Aborts if any disk backing the share is offline or unmounted # # v1.5-2.9 Icon set expanded progressively — each operation has its own distinct icon # All function output updated to use correct icon per context # notify() added — unRAID native + Discord webhook notifications # Script locking system added — acquire_lock(), acquire_rsync_lock() # translate_path() added — container → host path for arr cleanup scripts # check_arr_version() added — API version safety gate before arr operations # check_api() added — pre-flight API reachability check # ping_remote(), ping_internet() — non-fatal ping for failover use # check_local_array(), check_remote_array() — array health checks # check_remote_docker() — Docker daemon health check # get_unraid_temp_thresholds() — reads thresholds from dynamix.cfg # check_local_disk_temps() — pre-rsync temp check with exit codes 0/1/2 # stop/start local containers added alongside existing remote variants # # v3.1 Three-file config split — master.conf + host1.conf + host2.conf # load_config.sh introduced — auto-discovers all host*.conf files # detect_hosts() rewritten — sets MY_ID/REMOTE_ID and aliases all HOST* vars # All scripts now source load_config.sh instead of conf files directly # Adding a new server = add host*.conf, zero script changes required # # v3.2 check_remote_disks() rewritten — three-tier detection: # Tier 1: array disk paths (/mnt/disk*/sharename) # Tier 2: ZFS pools with find -maxdepth 2 (catches nested cache paths) # Tier 3: shfs fallback — verifies /mnt/user is mounted # Fixes: nested paths like /mnt/cache/appdata-Fallback/Critical-Data # Fixes: ZFS cache pools not detected when share is not at pool root # # v3.3 PROFILE_REMOTE_RESTART_CONTAINERS support added # Dirty sync profiles (critical-fallback, emby-fallback) restart remote # containers after sync if they were running before — picks up config changes # Was stopped → stays stopped. Was running → gets restarted. ✅ # # v3.5 notify_emby_scan() added — triggers Emby "Clean Missing Files" task # after arr cleanup scripts delete orphaned files # Emby immediately removes ghost entries — no user-facing file-not-found errors # Called by lidarr/sonarr/radarr_cleanup.sh when files are deleted # # # Three new safety functions added: # check_unraid_version_parity() — refuses remote ops on version mismatch # reads /etc/unraid-version local and remote via SSH # major mismatch → abort | minor mismatch → configurable warn/abort # check_remote_docker_daemon() — verifies remote Docker daemon before # issuing any remote container commands — strike system → skip/retry/exit # validate_unraid_cmd() — verifies unRAID-specific commands exist and # produce expected output before use — notifies and exits calling script # if command changed or disappeared after upgrade — other scripts unaffected # # ── VERSION ─────────────────────────────────────────────────────────────────────────────────── # Current: v3.5 # ============================================================================================== # Cron runs with a minimal PATH (/usr/bin:/bin) that omits /usr/local/sbin where tailscale, # and other Unraid tools live. Prepend it here so every sourcing script finds them. export PATH="/usr/local/sbin:/usr/local/bin:$PATH" # ============================================================================================== # ── ICONS ───────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Each icon has one job — do not reuse across different contexts. # Adding a new icon: add to the appropriate group below with a comment describing its job. # System / Host ICON_HOST="🖥️" # host detection and identity ICON_NET="🌐" # network / IP resolution ICON_PING="📡" # connectivity check ICON_GEAR="⚙️" # setup section header / profile load # Health Checks ICON_DISK="💾" # disk checks ICON_HEALTH="🩺" # rootfs / share health checks ICON_SHIELD="🛡️" # pre-flight section header # Containers ICON_CONTAINERS="📦" # container section anchor ICON_STOP="⛔" # stop command being issued ICON_STOPPED="🔴" # container confirmed stopped ICON_START="▶️" # start command being issued ICON_STARTED="💚" # container confirmed started ICON_RUNNING="🟢" # container already running when checked ICON_SKIP="⏭️" # skipping — already running healthy instance ICON_PROTECTED="🔰" # file/item is protected — never delete ICON_NOT_RUNNING="⭕" # container already stopped when checked # Transfer ICON_SYNC="🔄" # transfer section header ICON_RUN="🚀" # sync starting / rsync attempt ICON_RETRY="🔁" # retry attempt ICON_DONE="🏁" # transfer complete # Summary ICON_SUMMARY="📋" # summary section header ICON_TIME="⏱️" # duration line # System Operations ICON_MOVER="🔃" # mover operations ICON_REBOOT="⚡" # scheduled server reboot ICON_REBOOT_SMART="🚨" # smart conditional reboot triggered ICON_PLUGIN="🧩" # user scripts plugin operations ICON_PHP="👥" # PHP-FPM operations ICON_WEBGUI="💻" # WebGUI / nginx / emhttp operations # Diagnostics ICON_ZFS="📊" # ZFS ARC statistics ICON_MEM="🧠" # memory status ICON_WATCHDOG="🐾" # docker watchdog monitoring operations # Media Operations ICON_CLEAN="🧹" # media cleaner operations ICON_TRASH="🗑️" # files being deleted ICON_PERMS="🔐" # permissions operation / section header ICON_UNLOCKED="🔓" # permissions successfully applied to a share ICON_LOCK="🔏" # script instance lock — acquired/released # Transcode Operations ICON_RAM="💨" # ramdisk operations — fast ephemeral storage ICON_LINK="🔗" # symlink state and management # Failover Operations ICON_FALLBACK="🔀" # fallback state changes and operations # Docker Network Operations ICON_DOCKER_NET="🔌" # Docker network connect operations # Security / Certificate Operations ICON_CERT="🔒" # SSL certificate monitoring # Monitor Operations ICON_MONITOR="📈" # monitoring section headers and general monitoring ICON_SMART="🔧" # drive SMART health attribute monitoring ICON_BANDWIDTH="📶" # bandwidth usage tracking and reporting ICON_DIGEST="📰" # health digest — aggregated system summary ICON_EMBY="🎬" # Emby media server session reporting ICON_VERIFY="✔️" # backup verification — checksum comparison # Notifications ICON_NOTIFY="🔔" # notification operations # Output ICON_INFO="ℹ️" ICON_WARN="⚠️" ICON_ERROR="❌" ICON_SUCCESS="✅" # ============================================================================================== # ── OUTPUT HELPERS ──────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Standardised output functions used across all scripts. # # Output tiers: # echo — always visible — summaries, status conclusions, section headers # info() — always visible — status confirmations, reachability, check pass # warn() — always visible — state transitions, warnings, important events # error() — always visible — something broke # success() — always visible — operation completed successfully # log() — only with --log flag — per-item detail, internal checks # # All output goes to stdout — callers can redirect as needed. info() { echo "$ICON_INFO [INFO] $*"; } warn() { echo "$ICON_WARN [WARN] $*"; } error() { echo "$ICON_ERROR [ERROR] $*"; } success() { echo "$ICON_DONE [OK] $*"; } log() { [[ "${ENABLE_LOGGING:-false}" == true ]] && echo "[LOG] $*" return 0 } # ============================================================================================== # ── NOTIFICATION ────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Sends a notification via unRAID native system and/or Discord webhook. # Both channels are optional and independently controlled: # NOTIFY_UNRAID — shared toggle in master.conf # MY_DISCORD_WEBHOOK — per-host in host*.conf, set by detect_hosts() # # Severity levels: normal, warning, alert # Usage: notify "message" "subject" "severity" notify() { local message="$1" local subject="${2:-unRAID Notification}" local severity="${3:-normal}" log "$ICON_NOTIFY Sending notification: $subject — $message" if [[ "${NOTIFY_UNRAID:-false}" == true ]]; then local notify_script="/usr/local/emhttp/plugins/dynamix/scripts/notify" if [[ -x "$notify_script" ]]; then "$notify_script" -s "$subject" -d "$message" -i "$severity" 2>/dev/null log "$ICON_NOTIFY unRAID notification sent" else log "$ICON_NOTIFY unRAID notify script not found — skipping" fi fi # Uses MY_DISCORD_WEBHOOK — set by detect_hosts() from HOST*_DISCORD_WEBHOOK if [[ -n "${MY_DISCORD_WEBHOOK:-}" ]]; then local payload payload=$(printf '{"content": "%s — **%s**\\n%s"}' \ "$ICON_NOTIFY" "$subject" "$message") if curl -s -H "Content-Type: application/json" \ -d "$payload" "$MY_DISCORD_WEBHOOK" >/dev/null 2>&1; then log "$ICON_NOTIFY Discord notification sent" else warn "Discord notification failed — check HOST*_DISCORD_WEBHOOK in host*.conf" fi fi } # ============================================================================================== # ── DURATION FORMATTER ──────────────────────────────────────────────────────────────────────── # ============================================================================================== # Converts raw seconds into a human readable string. # Examples: 47 → "47s" | 653 → "10m53s" # Usage: format_duration $SECONDS format_duration() { local secs=$1 local mins=$((secs / 60)) local rem=$((secs % 60)) [[ $mins -gt 0 ]] && echo "${mins}m${rem}s" || echo "${rem}s" } # ============================================================================================== # ── ARG PARSER ──────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Processes all flags and key=value pairs passed to any script. # Call early in every script: parse_args "$@" # # Supported flags: # --dry-run | -n → DRY_RUN=true # --log → ENABLE_LOGGING=true # --no-log → ENABLE_LOGGING=false # --status → SHOW_STATUS=true # --help | -h → print usage and exit # # Supported key=value: # LOG=true/false → toggle logging # BW_LIMIT=5000 → override any declared master.conf variable for this run # # Unparsed positional args returned in PARSED_ARGS array. parse_args() { ENABLE_LOGGING=${ENABLE_LOGGING:-false} DRY_RUN=${DRY_RUN:-false} SHOW_STATUS=${SHOW_STATUS:-false} CLEAN_ARGS=() for ARG in "$@"; do if [[ "$ARG" == *=* ]]; then VAR="${ARG%%=*}" VAL="${ARG#*=}" case "$VAR" in LOG) [[ "$VAL" == "true" ]] && ENABLE_LOGGING=true [[ "$VAL" == "false" ]] && ENABLE_LOGGING=false ;; *) if declare -p "$VAR" &>/dev/null; then printf -v "$VAR" '%s' "$VAL" log "Set $VAR=$VAL" else warn "Unknown variable: $VAR" fi ;; esac else case "$ARG" in --dry-run|-n) DRY_RUN=true ;; --log) ENABLE_LOGGING=true ;; --no-log) ENABLE_LOGGING=false ;; --status|--summary) SHOW_STATUS=true ;; --help|-h) echo "Usage: script [--dry-run] [--log] [--status]" exit 0 ;; *) CLEAN_ARGS+=("$ARG") ;; esac fi done PARSED_ARGS=("${CLEAN_ARGS[@]}") } # ============================================================================================== # ── VALIDATION HELPERS ──────────────────────────────────────────────────────────────────────── # ============================================================================================== # Exits with error if a required variable is empty or unset. # Usage: require_var VAR_NAME require_var() { [[ -z "${!1:-}" ]] && error "Missing required: $1" && exit 1 } # Exits with error if a variable is not a valid positive integer. # Usage: validate_int VAR_NAME "$VAR_VALUE" validate_int() { local name="$1" value="$2" if [[ -z "$value" ]]; then error "$name is not set — check master.conf" exit 1 fi if ! [[ "$value" =~ ^[0-9]+$ ]]; then error "$name must be a positive integer — got: '$value'" exit 1 fi log "$name validated: $value" } # ============================================================================================== # ── HOST DETECTION ──────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Determines which server is local and which is remote by comparing hostname against # all HOST* values discovered from host*.conf files. # # Sets: # MY_ID — "HOST1" or "HOST2" (the role key, not the hostname) # REMOTE_ID — the peer's role key # LOCAL_SERVER_NAME — local hostname string (e.g. "unRAID-Gmer4Lfe") # REMOTE_SERVER_NAME — remote hostname string # SSH_KEY — SSH key for server-to-server operations # # Also sets aliases for all host-specific arrays so scripts use unprefixed names: # DAILY_SYNC_SHARES ← HOST*_DAILY_SYNC_SHARES # INTERMEDIATE_SYNC_SHARES ← HOST*_INTERMEDIATE_SYNC_SHARES # WEEKLY_SYNC_SHARES ← HOST*_WEEKLY_SYNC_SHARES # CRITICAL_SYNC_SHARES ← HOST*_CRITICAL_SYNC_SHARES # DAILY_RESTART_CONTAINERS ← HOST*_DAILY_RESTART_CONTAINERS # WEEKLY_RESTART_CONTAINERS ← HOST*_WEEKLY_RESTART_CONTAINERS # WATCHDOG_CONTAINERS ← HOST*_WATCHDOG_CONTAINERS (associative) # WATCHDOG_CONTAINER_URLS ← HOST*_WATCHDOG_CONTAINER_URLS (associative) # WATCHDOG_REQUIRED_CONTAINERS ← HOST*_WATCHDOG_REQUIRED_CONTAINERS # WATCHDOG_SCAN_IGNORE ← HOST*_WATCHDOG_SCAN_IGNORE # WATCHDOG_DEPENDENCIES ← HOST*_WATCHDOG_DEPENDENCIES (associative) # WATCHDOG_APPDATA_SIZES ← HOST*_WATCHDOG_APPDATA_SIZES (associative) # NETWORK_CONNECT_CONTAINERS ← HOST*_NETWORK_CONNECT_CONTAINERS # NETWORK_CONNECT_NETWORKS ← HOST*_NETWORK_CONNECT_NETWORKS # MEDIA_PERMISSION_SHARES ← HOST*_MEDIA_PERMISSION_SHARES # ANIME_CLEAN_FOLDERS ← HOST*_ANIME_CLEAN_FOLDERS # MEDIA_CLEAN_FOLDERS ← HOST*_MEDIA_CLEAN_FOLDERS # CERT_MONITOR_DOMAINS ← HOST*_CERT_MONITOR_DOMAINS # SMART_IGNORE_DRIVES ← HOST*_SMART_IGNORE_DRIVES # ZFS_REPORT_IGNORE_POOLS ← HOST*_ZFS_REPORT_IGNORE_POOLS # TRANSCODE_SSD ← HOST*_TRANSCODE_SSD # TRANSCODE_SERVERS ← HOST*_TRANSCODE_SERVERS # RAMDISK_SIZE ← HOST*_RAMDISK_SIZE # RAMDISK_WARN_GB ← HOST*_RAMDISK_WARN_GB # RAMDISK_LOW_GB ← HOST*_RAMDISK_LOW_GB # BACKUP_VERIFY_SHARES ← HOST*_BACKUP_VERIFY_SHARES # DDNS_CONTAINERS ← HOST*_DDNS_CONTAINERS # PARTNERSHIP_AUTH_WEBUIS ← HOST*_PARTNERSHIP_AUTH_WEBUIS # PARTNERSHIP_MIRROR_BACKUPS ← HOST*_PARTNERSHIP_MIRROR_BACKUPS # MY_DISCORD_WEBHOOK ← HOST*_DISCORD_WEBHOOK # EMBY_CONTAINER ← HOST*_EMBY_CONTAINER # EMBY_URL ← HOST*_EMBY_URL # EMBY_API_KEY ← HOST*_EMBY_API_KEY # GITEA_API_TOKEN ← HOST*_GITEA_API_TOKEN # # Arr-specific vars set by detect_hosts() when needed by arr scripts: # LIDARR_URL / LIDARR_API_KEY / LIDARR_MUSIC_ROOT (MY_ID only — Lidarr is HOST1 only) # SONARR_URL / SONARR_API_KEY / SONARR_TV_ROOT # RADARR_URL / RADARR_API_KEY / RADARR_MOVIES_ROOT # SLSKD_URL / SLSKD_API_KEY / SLSKD_FAILED_IMPORTS_DIR (MY_ID only) # SABNZBD_URL / SABNZBD_API_KEY (MY_ID only) # QBIT_URL / QBIT_USERNAME / QBIT_PASSWORD (MY_ID only) detect_hosts() { local local_hostname local_hostname="$(hostname)" # ── Find MY_ID by matching hostname against all HOST* vars ──────────────── MY_ID="" local host_var host_val # Scan all HOST* vars that hold a hostname value # HOST1, HOST2, HOST3 etc. — all defined in host*.conf files for host_var in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do host_val="${!host_var:-}" [[ -z "$host_val" ]] && continue if [[ "$local_hostname" == "$host_val" ]]; then MY_ID="$host_var" break fi done if [[ -z "$MY_ID" ]]; then error "Unknown host: $local_hostname" error "Hostname must match a HOST* value in host*.conf" error "Available: $(for h in HOST1 HOST2 HOST3 HOST4; do [[ -n "${!h:-}" ]] && echo -n "${!h} "; done)" exit 1 fi # ── Find REMOTE_ID — first HOST* that isn't MY_ID ──────────────────────── REMOTE_ID="" for host_var in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do [[ "$host_var" == "$MY_ID" ]] && continue host_val="${!host_var:-}" [[ -z "$host_val" ]] && continue REMOTE_ID="$host_var" break done # ── Set host name strings ───────────────────────────────────────────────── LOCAL_SERVER_NAME="${!MY_ID}" [[ -n "$REMOTE_ID" ]] && REMOTE_SERVER_NAME="${!REMOTE_ID}" || REMOTE_SERVER_NAME="unknown" # ── Set SSH key ─────────────────────────────────────────────────────────── local ssh_key_var="${MY_ID}_SSH_KEY" SSH_KEY="${!ssh_key_var:-}" if [[ -z "$SSH_KEY" ]]; then error "Missing SSH key: ${ssh_key_var} not set in host*.conf" exit 1 fi # ── Set scalar aliases ──────────────────────────────────────────────────── MY_DISCORD_WEBHOOK_VAR="${MY_ID}_DISCORD_WEBHOOK" MY_DISCORD_WEBHOOK="${!MY_DISCORD_WEBHOOK_VAR:-}" OWNER_NAME_VAR="${MY_ID}_OWNER" OWNER_NAME="${!OWNER_NAME_VAR:-}" OWNER_EMAIL_VAR="${MY_ID}_OWNER_EMAIL" OWNER_EMAIL="${!OWNER_EMAIL_VAR:-}" EMBY_CONTAINER_VAR="${MY_ID}_EMBY_CONTAINER" EMBY_CONTAINER="${!EMBY_CONTAINER_VAR:-}" EMBY_URL_VAR="${MY_ID}_EMBY_URL" EMBY_URL="${!EMBY_URL_VAR:-}" EMBY_API_KEY_VAR="${MY_ID}_EMBY_API_KEY" EMBY_API_KEY="${!EMBY_API_KEY_VAR:-}" GITEA_API_TOKEN_VAR="${MY_ID}_GITEA_API_TOKEN" GITEA_API_TOKEN="${!GITEA_API_TOKEN_VAR:-}" local _prov_var for _prov_var in \ PARTNERSHIP_PROVISION_EMBY_ADMIN \ PARTNERSHIP_EMBY_ADMIN_USER \ PARTNERSHIP_EMBY_ADMIN_PASS \ PARTNERSHIP_EMBY_PORT; do local _src="${MY_ID}_${_prov_var}" printf -v "$_prov_var" '%s' "${!_src:-}" done # ── System watchdog check toggles — per-host ───────────────────────────── # Aliased as unprefixed SYS_WATCHDOG_CHECK_* for use in system_watchdog.sh local _wd_checks=( SYS_WATCHDOG_NIC SYS_WATCHDOG_CHECK_DOCKER_DAEMON SYS_WATCHDOG_CHECK_ROOTFS SYS_WATCHDOG_CHECK_KERNEL_OOPS SYS_WATCHDOG_CHECK_FD SYS_WATCHDOG_CHECK_BOOT SYS_WATCHDOG_CHECK_OOM SYS_WATCHDOG_CHECK_RAM SYS_WATCHDOG_CHECK_LOG SYS_WATCHDOG_CHECK_ARC SYS_WATCHDOG_CHECK_CPU_TEMP SYS_WATCHDOG_CHECK_LOAD SYS_WATCHDOG_CHECK_ZOMBIES SYS_WATCHDOG_CHECK_CONTAINERS SYS_WATCHDOG_CHECK_TMP SYS_WATCHDOG_CHECK_MDSTAT SYS_WATCHDOG_CHECK_NETWORK SYS_WATCHDOG_CHECK_SSHD SYS_WATCHDOG_CHECK_RUNAWAY ) for _wd_var in "${_wd_checks[@]}"; do local _wd_src="${MY_ID}_${_wd_var}" # Only alias if the host-specific var is set — preserves master.conf defaults [[ -n "${!_wd_src+x}" ]] && eval "${_wd_var}=\"\${${_wd_src}}\"" done TRANSCODE_SSD_VAR="${MY_ID}_TRANSCODE_SSD" TRANSCODE_SSD="${!TRANSCODE_SSD_VAR:-}" RAMDISK_SIZE_VAR="${MY_ID}_RAMDISK_SIZE" RAMDISK_SIZE="${!RAMDISK_SIZE_VAR:-8G}" RAMDISK_WARN_GB_VAR="${MY_ID}_RAMDISK_WARN_GB" RAMDISK_WARN_GB="${!RAMDISK_WARN_GB_VAR:-6.8}" RAMDISK_LOW_GB_VAR="${MY_ID}_RAMDISK_LOW_GB" RAMDISK_LOW_GB="${!RAMDISK_LOW_GB_VAR:-5.5}" # Arr credentials — MY_ID only (remote arr accessed via its own host) LIDARR_URL_VAR="${MY_ID}_LIDARR_URL"; LIDARR_URL="${!LIDARR_URL_VAR:-}" LIDARR_API_KEY_VAR="${MY_ID}_LIDARR_API_KEY"; LIDARR_API_KEY="${!LIDARR_API_KEY_VAR:-}" LIDARR_MUSIC_ROOT_VAR="${MY_ID}_LIDARR_MUSIC_ROOT"; LIDARR_MUSIC_ROOT="${!LIDARR_MUSIC_ROOT_VAR:-}" FANART_API_KEY_VAR="${MY_ID}_FANART_API_KEY"; FANART_API_KEY="${!FANART_API_KEY_VAR:-}" LASTFM_API_KEY_VAR="${MY_ID}_LASTFM_API_KEY"; LASTFM_API_KEY="${!LASTFM_API_KEY_VAR:-}" TMDB_API_KEY_VAR="${MY_ID}_TMDB_API_KEY"; TMDB_API_KEY="${!TMDB_API_KEY_VAR:-}" SONARR_URL_VAR="${MY_ID}_SONARR_URL"; SONARR_URL="${!SONARR_URL_VAR:-}" SONARR_API_KEY_VAR="${MY_ID}_SONARR_API_KEY"; SONARR_API_KEY="${!SONARR_API_KEY_VAR:-}" SONARR_TV_ROOT_VAR="${MY_ID}_SONARR_TV_ROOT"; SONARR_TV_ROOT="${!SONARR_TV_ROOT_VAR:-}" RADARR_URL_VAR="${MY_ID}_RADARR_URL"; RADARR_URL="${!RADARR_URL_VAR:-}" RADARR_API_KEY_VAR="${MY_ID}_RADARR_API_KEY"; RADARR_API_KEY="${!RADARR_API_KEY_VAR:-}" RADARR_MOVIES_ROOT_VAR="${MY_ID}_RADARR_MOVIES_ROOT"; RADARR_MOVIES_ROOT="${!RADARR_MOVIES_ROOT_VAR:-}" SLSKD_URL_VAR="${MY_ID}_SLSKD_URL"; SLSKD_URL="${!SLSKD_URL_VAR:-}" SLSKD_API_KEY_VAR="${MY_ID}_SLSKD_API_KEY"; SLSKD_API_KEY="${!SLSKD_API_KEY_VAR:-}" SLSKD_FAILED_IMPORTS_DIR_VAR="${MY_ID}_SLSKD_FAILED_IMPORTS_DIR" SLSKD_FAILED_IMPORTS_DIR="${!SLSKD_FAILED_IMPORTS_DIR_VAR:-}" SABNZBD_URL_VAR="${MY_ID}_SABNZBD_URL"; SABNZBD_URL="${!SABNZBD_URL_VAR:-}" SABNZBD_API_KEY_VAR="${MY_ID}_SABNZBD_API_KEY"; SABNZBD_API_KEY="${!SABNZBD_API_KEY_VAR:-}" QBIT_URL_VAR="${MY_ID}_QBIT_URL"; QBIT_URL="${!QBIT_URL_VAR:-}" QBIT_USERNAME_VAR="${MY_ID}_QBIT_USERNAME"; QBIT_USERNAME="${!QBIT_USERNAME_VAR:-}" QBIT_PASSWORD_VAR="${MY_ID}_QBIT_PASSWORD"; QBIT_PASSWORD="${!QBIT_PASSWORD_VAR:-}" # ── Set array aliases — indexed arrays ──────────────────────────────────── # Each eval copies the host-specific array into the unprefixed name scripts use _alias_array() { local alias_name="$1" local source_var="${MY_ID}_${alias_name}" eval "${alias_name}=(\"\${${source_var}[@]}\")" } _alias_array "DAILY_SYNC_SHARES" _alias_array "PERSONAL_SHARES" _alias_array "INTERMEDIATE_SYNC_SHARES" _alias_array "WEEKLY_SYNC_SHARES" _alias_array "CRITICAL_SYNC_SHARES" _alias_array "BACKUP_VERIFY_SHARES" _alias_array "DAILY_RESTART_CONTAINERS" _alias_array "WEEKLY_RESTART_CONTAINERS" _alias_array "WATCHDOG_REQUIRED_CONTAINERS" _alias_array "WATCHDOG_SCAN_IGNORE" _alias_array "NETWORK_CONNECT_CONTAINERS" _alias_array "NETWORK_CONNECT_NETWORKS" _alias_array "MEDIA_PERMISSION_SHARES" _alias_array "ANIME_CLEAN_FOLDERS" _alias_array "MEDIA_CLEAN_FOLDERS" _alias_array "CERT_MONITOR_DOMAINS" _alias_array "SMART_IGNORE_DRIVES" _alias_array "ZFS_REPORT_IGNORE_POOLS" _alias_array "TRANSCODE_SERVERS" _alias_array "DDNS_CONTAINERS" _alias_array "PARTNERSHIP_AUTH_WEBUIS" _alias_array "PARTNERSHIP_MIRROR_BACKUPS" _alias_array "PARTNERSHIP_OWN_CONTAINERS" _alias_array "PARTNERSHIP_AUTH_STACK" _alias_array "PARTNERSHIP_REPLACE_CONTAINERS" _alias_array "PARTNERSHIP_ARR_STACK" _alias_array "PARTNERSHIP_ARR_REPLACE_CONTAINERS" _alias_array "RW_PAUSE_CONTAINERS" _alias_array "RW_STOP_CONTAINERS" # ── Set array aliases — associative arrays ──────────────────────────────── # Associative arrays cannot be copied with eval — must be rebuilt key by key _alias_assoc() { local alias_name="$1" local source_name="${MY_ID}_${alias_name}" eval "declare -gA ${alias_name}" local -n _assoc_src="${source_name}" 2>/dev/null || return 0 local -n _assoc_dst="${alias_name}" local _assoc_key for _assoc_key in "${!_assoc_src[@]}"; do _assoc_dst["$_assoc_key"]="${_assoc_src[$_assoc_key]}" done } _alias_assoc "WATCHDOG_CONTAINERS" _alias_assoc "WATCHDOG_CONTAINER_URLS" _alias_assoc "WATCHDOG_DEPENDENCIES" _alias_assoc "WATCHDOG_APPDATA_SIZES" # ── Output ──────────────────────────────────────────────────────────────── log "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" } # ============================================================================================== # ── REMOTE IP RESOLUTION ────────────────────────────────────────────────────────────────────── # ============================================================================================== # Resolves the Tailscale IPv4 address of the remote server. # Sets REMOTE_SERVER used by all subsequent SSH and rsync calls. # Exits if resolution fails — Tailscale may be down or peer offline. resolve_remote_ip() { log "Resolving remote IP for $REMOTE_SERVER_NAME..." local ts_name="${REMOTE_SERVER_NAME,,}" local attempts=3 for ((i=1; i<=attempts; i++)); do # Direct lookup — works when MagicDNS short-name resolution is active REMOTE_SERVER=$(tailscale ip -4 "$ts_name" 2>/dev/null) # Fallback — parse tailscale status for FQDN entries (e.g. hostname.tailXXXX.ts.net) if [[ -z "$REMOTE_SERVER" ]]; then REMOTE_SERVER=$(tailscale status 2>/dev/null | awk -v name="$ts_name" '$2 ~ "^" name { print $1; exit }') fi [[ -n "$REMOTE_SERVER" ]] && break [[ $i -lt $attempts ]] && { warn "Tailscale resolution attempt $i/$attempts failed — retrying in 5s..."; sleep 5; } done if [[ -z "$REMOTE_SERVER" ]]; then error "Failed to resolve Tailscale IP for $REMOTE_SERVER_NAME after $attempts attempts" error "Check: tailscale status | grep $ts_name" exit 1 fi info "$ICON_NET Remote IP: $REMOTE_SERVER" } # Resolve any hostname to a Tailscale IPv4 — tries direct lookup, falls back to status parse. # Usage: ip=$(resolve_tailscale_ip "hostname") — returns empty string on failure. resolve_tailscale_ip() { local hostname="${1,,}" local ip ip=$(tailscale ip -4 "$hostname" 2>/dev/null) [[ -z "$ip" ]] && \ ip=$(tailscale status 2>/dev/null | awk -v name="$hostname" '$2 ~ "^" name { print $1; exit }') echo "$ip" } # ============================================================================================== # ── CONNECTIVITY CHECKS ─────────────────────────────────────────────────────────────────────── # ============================================================================================== # Fatal connectivity check — used by rsync and other scripts that must abort if unreachable. # For failover use ping_remote() which returns status without exiting. check_connectivity() { log "Checking connectivity to $REMOTE_SERVER..." if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then error "$ICON_PING Remote $REMOTE_SERVER ($REMOTE_SERVER_NAME) is unreachable" info "Hint: tailscale status | grep $REMOTE_SERVER_NAME" exit 1 fi info "$ICON_PING $REMOTE_SERVER_NAME is reachable" } # Non-fatal ping — used by fallback.sh which handles its own state machine. # Returns 0 if reachable, 1 if not — does NOT exit. ping_remote() { ping -c2 -W3 "$REMOTE_SERVER" &>/dev/null } # Non-fatal external connectivity check — used by fallback.sh. # Returns 0 if internet reachable, 1 if not — does NOT exit. ping_internet() { ping -c2 -W3 "${EXTERNAL_IP:-8.8.8.8}" &>/dev/null } # ============================================================================================== # ── RSYNC GATE CHECK ────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Two-tier rsync enable/disable system. # Tier 1: RSYNC_ENABLED — global gate, overrides everything # Tier 2: orchestrator-specific flag — fine-grained control per window # # Usage: # check_rsync_enabled "CRITICAL" ← checks RSYNC_ENABLED + CRITICAL_RSYNC_ENABLED # check_rsync_enabled "INTERMEDIATE" ← checks RSYNC_ENABLED + INTERMEDIATE_RSYNC_ENABLED # check_rsync_enabled "DAILY" ← checks RSYNC_ENABLED + DAILY_RSYNC_ENABLED # check_rsync_enabled "WEEKLY" ← checks RSYNC_ENABLED + WEEKLY_RSYNC_ENABLED # check_rsync_enabled "FALLBACK" ← checks RSYNC_ENABLED + FALLBACK_RSYNC_ENABLED # check_rsync_enabled ← checks RSYNC_ENABLED only (direct rsync.sh call) # # Returns: 0 = enabled, proceed | 1 = disabled, skip cleanly check_rsync_enabled() { local orchestrator="${1:-}" # Tier 1 — global gate if [[ "${RSYNC_ENABLED:-true}" == false ]]; then warn "RSYNC_ENABLED=false — rsync globally disabled, skipping all syncs" return 1 fi # Tier 2 — per-orchestrator if [[ -n "$orchestrator" ]]; then local var_name="${orchestrator}_RSYNC_ENABLED" local var_value="${!var_name:-true}" if [[ "$var_value" == false ]]; then info "${var_name}=false — rsync disabled for $orchestrator orchestrator" info "All other $orchestrator jobs will still run normally" return 1 fi fi return 0 } # ============================================================================================== # ── LOCAL HEALTH CHECKS ─────────────────────────────────────────────────────────────────────── # ============================================================================================== # Verifies local /mnt/user is mounted and has shares. # Non-fatal — returns status for caller to decide. # Used by failover before starting remote containers locally. check_local_array() { log "Checking local array..." if ! mountpoint -q /mnt/user 2>/dev/null; then error "$ICON_DISK Local array is not started — /mnt/user is not mounted" return 1 fi local file_count file_count=$(ls /mnt/user 2>/dev/null | wc -l) if [[ "$file_count" -eq 0 ]]; then error "$ICON_DISK Local array appears empty — shares may not be available" return 1 fi log "Local array is healthy" return 0 } # ============================================================================================== # ── REMOTE HEALTH CHECKS ───────────────────────────────────────────────────────────────────── # ============================================================================================== # Verifies remote /mnt/user is mounted via SSH. # Non-fatal — returns status. Used before handback rsync. # Syncing to remote with no array fills rootfs rapidly. check_remote_array() { log "Checking remote array on $REMOTE_SERVER_NAME..." local result result=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ "mountpoint -q /mnt/user && echo yes || echo no" 2>/dev/null) if [[ "$result" != "yes" ]]; then error "$ICON_DISK Remote array not started on $REMOTE_SERVER_NAME" return 1 fi log "Remote array is healthy" return 0 } # Verifies remote Docker daemon is responding. # Non-fatal — returns status. A hung daemon means container commands silently fail. check_remote_docker() { log "Checking remote Docker daemon on $REMOTE_SERVER_NAME..." if ! ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ "timeout 10 docker ps" >/dev/null 2>&1; then error "$ICON_CONTAINERS Remote Docker daemon not responding on $REMOTE_SERVER_NAME" return 1 fi log "Remote Docker daemon is healthy" return 0 } # Aborts if remote rootfs (/) usage is at or above ROOTFS_WARN threshold. # Fatal — exits the calling script. # When remote array is down rsync writes land on rootfs and fill it rapidly. check_remote_rootfs() { log "Checking remote rootfs usage..." local remote_usage remote_usage=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null) if [[ -z "$remote_usage" ]]; then error "Could not retrieve rootfs usage from $REMOTE_SERVER_NAME" exit 1 fi if [[ "$remote_usage" -ge "${ROOTFS_WARN:-75}" ]]; then error "$ICON_HEALTH Remote rootfs ${remote_usage}% — threshold ${ROOTFS_WARN:-75}%" exit 1 fi info "$ICON_HEALTH Remote rootfs: ${remote_usage}% (threshold: ${ROOTFS_WARN:-75}%)" } # Verifies target directory exists and is not empty on remote. # Fatal — exits the calling script. # Usage: check_remote_share "/mnt/user/Movies" check_remote_share() { local dir="$1" log "Checking remote share: $dir..." local share_exists share_exists=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null) if [[ "$share_exists" != "yes" ]]; then error "$ICON_HEALTH Remote share does not exist: $dir" exit 1 fi local share_empty share_empty=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null) if [[ "$share_empty" == "yes" ]]; then warn "$ICON_HEALTH Remote share exists but is empty: $dir — aborting to protect data" exit 1 fi info "$ICON_HEALTH Remote share verified: $dir" } # ============================================================================================== # ── DISK TEMPERATURE CHECKS ─────────────────────────────────────────────────────────────────── # ============================================================================================== # Returns 0 (true) if device is non-rotational (SSD/NVMe), 1 (false) if HDD. # Checks /sys/block//queue/rotational — 0=SSD, 1=HDD. is_ssd() { local base base=$(basename "$1") [[ "$(cat "/sys/block/$base/queue/rotational" 2>/dev/null)" == "0" ]] } # Reads disk temperature thresholds from unRAID's dynamix.cfg. # Sets globals: UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX # Falls back to values in master.conf (SMART_TEMP_WARN/CRIT) if file not found. get_unraid_temp_thresholds() { local cfg="/boot/config/plugins/dynamix/dynamix.cfg" if [[ -f "$cfg" ]]; then UNRAID_DISK_HOT=$(grep '^hot=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"') UNRAID_DISK_MAX=$(grep '^max=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"') UNRAID_SSD_HOT=$(grep '^hotssd=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"') UNRAID_SSD_MAX=$(grep '^maxssd=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"') fi # Fall back to master.conf values if dynamix.cfg not found or values empty UNRAID_DISK_HOT="${UNRAID_DISK_HOT:-${SMART_TEMP_WARN:-45}}" UNRAID_DISK_MAX="${UNRAID_DISK_MAX:-${SMART_TEMP_CRIT:-55}}" UNRAID_SSD_HOT="${UNRAID_SSD_HOT:-60}" UNRAID_SSD_MAX="${UNRAID_SSD_MAX:-70}" } # Checks local disk temperatures before rsync. # Reads temps and rotational flag from /var/local/emhttp/disks.ini. # Uses unRAID's own thresholds from dynamix.cfg via get_unraid_temp_thresholds(). # # Returns: # 0 = all temps OK — proceed # 1 = warn threshold exceeded — skip this profile only # 2 = critical threshold exceeded — abort all remaining profiles # # Sets TEMP_CHECK_RESULT with human readable summary for logging. check_local_disk_temps() { get_unraid_temp_thresholds local disks_ini="/var/local/emhttp/disks.ini" if [[ ! -f "$disks_ini" ]]; then warn "disks.ini not found — skipping temp check" TEMP_CHECK_RESULT="temp check skipped (disks.ini not found)" return 0 fi local worst_result=0 local hot_drives=() local crit_drives=() local current_name="" local current_device="" local current_rotational="" local current_temp="" check_drive() { [[ -z "$current_name" ]] || [[ -z "$current_temp" ]] && return [[ "$current_temp" -eq 0 ]] && return # spun down — skip local warn_thresh crit_thresh if [[ "$current_rotational" == "0" ]]; then warn_thresh="$UNRAID_SSD_HOT" crit_thresh="$UNRAID_SSD_MAX" else warn_thresh="$UNRAID_DISK_HOT" crit_thresh="$UNRAID_DISK_MAX" fi if [[ "$current_temp" -ge "$crit_thresh" ]]; then crit_drives+=("${current_name}(${current_device}):${current_temp}°C≥${crit_thresh}°C") [[ $worst_result -lt 2 ]] && worst_result=2 elif [[ "$current_temp" -ge "$warn_thresh" ]]; then hot_drives+=("${current_name}(${current_device}):${current_temp}°C≥${warn_thresh}°C") [[ $worst_result -lt 1 ]] && worst_result=1 fi } while IFS= read -r ini_line; do if echo "$ini_line" | grep -qE '^\["(disk[0-9]+|parity[0-9]?|cache[0-9]?)"\]'; then check_drive current_name=$(echo "$ini_line" | grep -o '"[^"]*"' | head -1 | tr -d '"') current_device="" current_rotational="1" # default HDD current_temp="" elif echo "$ini_line" | grep -q '^device='; then current_device=$(echo "$ini_line" | cut -d= -f2 | tr -d '"') elif echo "$ini_line" | grep -q '^rotational='; then current_rotational=$(echo "$ini_line" | cut -d= -f2 | tr -d '"') elif echo "$ini_line" | grep -q '^temp='; then current_temp=$(echo "$ini_line" | cut -d= -f2 | tr -d '"') current_temp="${current_temp//[^0-9]/}" current_temp="${current_temp:-0}" fi done < "$disks_ini" check_drive # process last drive if [[ ${#crit_drives[@]} -gt 0 ]]; then TEMP_CHECK_RESULT="CRITICAL temps: ${crit_drives[*]}" error "Drive temp CRITICAL — aborting all remaining syncs: ${crit_drives[*]}" notify "Rsync aborted on $(hostname) — drive temp CRITICAL: ${crit_drives[*]}" "Rsync Temp Check" "warning" return 2 elif [[ ${#hot_drives[@]} -gt 0 ]]; then TEMP_CHECK_RESULT="HOT drives: ${hot_drives[*]}" warn "Drive temp WARNING — skipping this profile: ${hot_drives[*]}" notify "Rsync profile skipped on $(hostname) — drive temp WARNING: ${hot_drives[*]}" "Rsync Temp Check" "normal" return 1 else TEMP_CHECK_RESULT="all normal (HDD warn:${UNRAID_DISK_HOT}°C crit:${UNRAID_DISK_MAX}°C SSD warn:${UNRAID_SSD_HOT}°C crit:${UNRAID_SSD_MAX}°C)" return 0 fi } # ============================================================================================== # ── REMOTE DISK VERIFICATION ────────────────────────────────────────────────────────────────── # ============================================================================================== # Verifies all disks backing a share are healthy on the remote server. # Auto-detects filesystem type from disks.ini — handles XFS, ZFS, and cache pools. # No configuration required — discovers disk layout at runtime. # # Three-tier detection handles all share locations: # Tier 1: Array disk paths — /mnt/disk*/sharename (XFS or ZFS per-disk) # Tier 2: ZFS standalone pools — finds nested paths up to 2 levels deep # catches /mnt/cache/appdata-Fallback/Critical-Data correctly # Tier 3: shfs fallback — if share exists under /mnt/user, resolve backing pool # or verify shfs itself is mounted if pool can't be determined # # Fatal — exits the calling script if any disk is offline. # Usage: check_remote_disks "/mnt/user/Movies" check_remote_disks() { local dir="$1" local share_name share_name=$(basename "$dir") info "$ICON_DISK Checking disks backing $share_name on $REMOTE_SERVER_NAME..." # Read remote disks.ini for fsType mapping local disks_ini_content disks_ini_content=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "cat /var/local/emhttp/disks.ini 2>/dev/null" 2>/dev/null) if [[ -z "$disks_ini_content" ]]; then error "$ICON_DISK Cannot read disks.ini from $REMOTE_SERVER_NAME" exit 1 fi # Tier 1 — array disk paths (/mnt/disk*/sharename) local backing_disks backing_disks=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "ls -d /mnt/disk*/$share_name 2>/dev/null | awk -F/ '{print \$3}'" 2>/dev/null) # Tier 2 — ZFS standalone pools, including nested paths local zfs_pool_paths zfs_pool_paths=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "zpool list -H -o name 2>/dev/null | while read pool; do [[ -d \"/mnt/\${pool}/$share_name\" ]] && echo \"\$pool\" && continue find \"/mnt/\${pool}\" -maxdepth 2 -name '$share_name' -type d 2>/dev/null | \ grep -q . && echo \"\$pool\" done | sort -u" 2>/dev/null) # Tier 3 — shfs fallback via /mnt/user if [[ -z "$backing_disks" ]] && [[ -z "$zfs_pool_paths" ]]; then local on_user on_user=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "find /mnt/user -maxdepth 3 -name '$share_name' -type d 2>/dev/null | head -1" \ 2>/dev/null) if [[ -n "$on_user" ]]; then local actual_path actual_path=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "readlink -f '$on_user' 2>/dev/null || echo '$on_user'" 2>/dev/null) local pool_name pool_name=$(echo "$actual_path" | awk -F/ '{print $3}') if [[ -n "$pool_name" ]] && [[ "$pool_name" != "user" ]]; then zfs_pool_paths="$pool_name" info "$ICON_DISK $share_name found at $actual_path on $REMOTE_SERVER_NAME" else # Can't determine pool — verify shfs itself is mounted local shfs_ok shfs_ok=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "mountpoint -q /mnt/user && echo yes || echo no" 2>/dev/null) if [[ "$shfs_ok" == "yes" ]]; then success "All disks backing $share_name are online ✅ (via shfs)" return 0 fi fi fi fi if [[ -z "$backing_disks" ]] && [[ -z "$zfs_pool_paths" ]]; then error "$ICON_DISK No disks found backing $share_name on $REMOTE_SERVER_NAME" exit 1 fi local all_ok=true # Check array disks (XFS or ZFS single-disk-in-array) if [[ -n "$backing_disks" ]]; then while IFS= read -r disk_name; do [[ -z "$disk_name" ]] && continue local fs_type fs_type=$(echo "$disks_ini_content" | awk -F= -v disk="$disk_name" ' /^\["'"'"'?/ { current=substr($0,3,length($0)-4) } current==disk && /^fsType=/ { print $2; exit } ' | tr -d '"') fs_type="${fs_type:-xfs}" if [[ "$fs_type" == "zfs" ]]; then local pool_health pool_health=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "zpool list -H -o health '$disk_name' 2>/dev/null" 2>/dev/null) if [[ "$pool_health" == "ONLINE" ]]; then info "$ICON_DISK $disk_name (ZFS) $ICON_RUNNING — $share_name ONLINE" else error "$ICON_DISK $disk_name (ZFS) $ICON_STOPPED — pool ${pool_health:-offline}" all_ok=false fi else local mounted mounted=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "mountpoint -q '/mnt/$disk_name' && echo yes || echo no" 2>/dev/null) if [[ "$mounted" == "yes" ]]; then info "$ICON_DISK $disk_name (XFS) $ICON_RUNNING — $share_name present" else error "$ICON_DISK $disk_name (XFS) $ICON_STOPPED — not mounted" all_ok=false fi fi done <<< "$backing_disks" fi # Check ZFS standalone pools (cache, gaming, media-servers etc.) if [[ -n "$zfs_pool_paths" ]]; then while IFS= read -r pool_name; do [[ -z "$pool_name" ]] && continue local pool_health pool_health=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "zpool list -H -o health '$pool_name' 2>/dev/null" 2>/dev/null) if [[ "$pool_health" == "ONLINE" ]]; then info "$ICON_DISK $pool_name (ZFS pool) $ICON_RUNNING — $share_name ONLINE" else error "$ICON_DISK $pool_name (ZFS pool) $ICON_STOPPED — pool ${pool_health:-offline}" all_ok=false fi done <<< "$zfs_pool_paths" fi if [[ "$all_ok" == false ]]; then error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME" exit 1 fi success "All disks backing $share_name are online ✅" } # ============================================================================================== # ── CONTAINER MANAGEMENT ────────────────────────────────────────────────────────────────────── # ============================================================================================== # Four functions — stop/start on both local and remote. # State tracking: only containers that WERE running get restarted. # Was stopped → stays stopped. Was running → gets restarted. ✅ # Tracks which remote containers were running before stop — used by start_containers() RUNNING_CONTAINERS=() # Stop containers on the REMOTE server via SSH. # Reads CRITICAL_CONTAINER_NAMES — set from profile arrays by rsync.sh. # Tracks running state in RUNNING_CONTAINERS for restart after sync. stop_containers() { if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \ [[ "${CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then log "No remote containers configured for this profile, skipping remote stop." return fi echo "━━━ $ICON_STOP $ICON_CONTAINERS Containers ━━━" info "Stopping remote containers..." RUNNING_CONTAINERS=() for c in "${CRITICAL_CONTAINER_NAMES[@]}"; do [[ -z "$c" ]] && continue local status status=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "docker inspect -f '{{.State.Running}}' $c 2>/dev/null" 2>/dev/null || echo "unknown") if [[ "$status" == "true" ]]; then echo "$ICON_STOP Stopping remote $c..." RUNNING_CONTAINERS+=("$c") if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $c" >/dev/null; then echo "$ICON_STOPPED $c stopped" else error "Failed to stop remote $c" fi elif [[ "$status" == "false" ]]; then echo "$ICON_NOT_RUNNING $c is not running — skipping" else log "$c not found on remote — skipping" fi done } # Tracks which local containers were running before stop — used by start_local_containers() LOCAL_RUNNING_CONTAINERS=() # Stop containers on the LOCAL server. # Reads LOCAL_CRITICAL_CONTAINER_NAMES — set from profile arrays by rsync.sh. # Flushes local databases cleanly before pushing data to remote. stop_local_containers() { if [[ ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \ [[ "${LOCAL_CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then log "No local containers configured for this profile, skipping local stop." return fi info "Stopping local containers..." LOCAL_RUNNING_CONTAINERS=() for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]}"; do [[ -z "$c" ]] && continue local status status=$(docker inspect -f '{{.State.Running}}' "$c" 2>/dev/null || echo "unknown") if [[ "$status" == "true" ]]; then echo "$ICON_STOP Stopping local $c..." LOCAL_RUNNING_CONTAINERS+=("$c") if docker stop "$c" >/dev/null; then echo "$ICON_STOPPED $c stopped" else error "Failed to stop local $c" fi elif [[ "$status" == "false" ]]; then echo "$ICON_NOT_RUNNING $c is not running — skipping" else log "$c not found locally — skipping" fi done } # Restart containers on the REMOTE server that were running before sync. # Reads RUNNING_CONTAINERS set by stop_containers(). # Respects DELAYED_CONTAINERS — waits CONTAINER_DELAY seconds before starting them. start_containers() { if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then log "No remote containers to restart." return fi info "Starting remote containers..." for c in "${RUNNING_CONTAINERS[@]}"; do [[ -z "$c" ]] && continue local needs_delay=false for d in "${DELAYED_CONTAINERS[@]}"; do [[ "$c" == "$d" ]] && needs_delay=true && break done if [[ "$needs_delay" == true ]]; then info "Waiting ${CONTAINER_DELAY}s before starting $c..." sleep "$CONTAINER_DELAY" fi echo "$ICON_START Starting remote $c..." if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $c" >/dev/null 2>&1; then echo "$ICON_STARTED $c started" else error "Failed to start remote $c — start manually if needed" fi done } # Restart containers on the LOCAL server that were running before sync. # Reads LOCAL_RUNNING_CONTAINERS set by stop_local_containers(). # Respects DELAYED_CONTAINERS and CONTAINER_DELAY same as remote. start_local_containers() { if [[ ${#LOCAL_RUNNING_CONTAINERS[@]} -eq 0 ]]; then log "No local containers to restart." return fi info "Starting local containers..." for c in "${LOCAL_RUNNING_CONTAINERS[@]}"; do [[ -z "$c" ]] && continue local needs_delay=false for d in "${DELAYED_CONTAINERS[@]}"; do [[ "$c" == "$d" ]] && needs_delay=true && break done if [[ "$needs_delay" == true ]]; then info "Waiting ${CONTAINER_DELAY}s before starting local $c..." sleep "$CONTAINER_DELAY" fi echo "$ICON_START Starting local $c..." if docker start "$c" >/dev/null 2>&1; then echo "$ICON_STARTED $c started" else error "Failed to start local $c — start manually if needed" fi done } # ============================================================================================== # ── RSYNC OPTIONS ───────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Loads rsync options for the current profile. # Falls back to DEFAULT_RSYNC_OPTS from master.conf if no profile match found. # Profile opts do NOT inherit from defaults — list all desired flags explicitly in master.conf. get_rsync_opts() { if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then read -r -a RSYNC_OPTS <<< "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]}" log "Using profile rsync opts for $PROFILE_NAME: ${RSYNC_OPTS[*]}" else RSYNC_OPTS=("${DEFAULT_RSYNC_OPTS[@]}") log "Using default rsync opts: ${RSYNC_OPTS[*]}" fi } # ============================================================================================== # ── SCRIPT LOCKING ──────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Prevents multiple instances of the same script running simultaneously. # All lock files live in /tmp/unraid_locks/ — auto-cleared on reboot. # # Modes: # strict — exit immediately if already running (default) # wait — wait LOCK_WAIT_TIMEOUT seconds then exit if still locked # continuous — for long-running scripts: skip gracefully if healthy instance running # # Features: # Stale lock detection — if lock PID is dead, clears and proceeds # PID reuse protection — lock stores PID:scriptname, validates both # Age warning — warns if lock older than LOCK_WARN_AGE (skipped for continuous) # EXIT trap — lock always released on exit, crash, or kill signal LOCK_DIR="/tmp/unraid_locks" # Ensure DATA_DIR and STATE_DIR exist — created here so every script that sources common.sh # can safely write to either without checking first. if [[ -n "${DATA_DIR:-}" ]] && [[ ! -d "$DATA_DIR" ]]; then mkdir -p "$DATA_DIR" 2>/dev/null || true fi if [[ -n "${STATE_DIR:-}" ]] && [[ ! -d "$STATE_DIR" ]]; then mkdir -p "$STATE_DIR" 2>/dev/null || true fi RSYNC_COUNT_FILE="$LOCK_DIR/rsync_active_count" RSYNC_MAX_CONCURRENT=3 LOCK_WARN_AGE=300 # seconds — warn if lock older than this (5min default) LOCK_WAIT_TIMEOUT=30 # seconds — how long "wait" mode waits before giving up # Registry of all lock files acquired by this process — released together on exit. # Prevents the single-trap-per-acquire problem: each new acquire_lock/acquire_rsync_lock # call used to overwrite the EXIT trap, orphaning the previous lock file. declare -ga _LOCK_FILES=() _release_all_locks() { local lf for lf in "${_LOCK_FILES[@]}"; do [[ -f "$lf" ]] && rm -f "$lf" done } _register_lock() { _LOCK_FILES+=("$1") trap '_release_all_locks' EXIT } # Internal — script name used as lock identifier _lock_name() { basename "${BASH_SOURCE[1]:-$0}" .sh } # Internal — lock file path for this script _lock_file() { echo "$LOCK_DIR/${1:-$(_lock_name)}.lock" } # Internal — release lock on exit (registered via EXIT trap) _release_on_exit() { local lockfile="$1" [[ -f "$lockfile" ]] && rm -f "$lockfile" } # Acquire exclusive lock for this script. # Usage: acquire_lock [strict|wait|continuous] acquire_lock() { local mode="${1:-strict}" local script_name script_name=$(basename "${BASH_SOURCE[1]:-$0}" .sh) local lockfile lockfile="$(_lock_file "$script_name")" mkdir -p "$LOCK_DIR" if [[ -f "$lockfile" ]]; then local lock_content existing_pid locked_name lock_content=$(cat "$lockfile" 2>/dev/null) existing_pid="${lock_content%%:*}" locked_name="${lock_content##*:}" # Stale lock — PID dead if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then warn "Stale lock detected for $script_name (PID $existing_pid gone) — clearing" rm -f "$lockfile" # PID reuse — PID alive but belongs to different process elif [[ "$locked_name" != "$script_name" ]]; then warn "Lock PID $existing_pid reused by different process ($locked_name ≠ $script_name) — clearing" rm -f "$lockfile" else # Lock is genuinely active — check age local lock_age lock_age=$(( $(date +%s) - $(stat -c %Y "$lockfile" 2>/dev/null || echo 0) )) if [[ "$lock_age" -gt "$LOCK_WARN_AGE" ]] && [[ "$mode" != "continuous" ]]; then warn "$script_name has been running for ${lock_age}s — may be stuck (PID $existing_pid)" fi if [[ "$mode" == "continuous" ]]; then log "$ICON_SKIP $script_name already running healthy (PID $existing_pid) — skipping" exit 0 elif [[ "$mode" == "wait" ]]; then info "Another instance of $script_name is running — waiting up to ${LOCK_WAIT_TIMEOUT}s" local waited=0 while [[ -f "$lockfile" ]] && [[ "$waited" -lt "$LOCK_WAIT_TIMEOUT" ]]; do sleep 1 ((waited++)) lock_content=$(cat "$lockfile" 2>/dev/null) existing_pid="${lock_content%%:*}" locked_name="${lock_content##*:}" if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then warn "Lock became stale while waiting — clearing" rm -f "$lockfile" break elif [[ "$locked_name" != "$script_name" ]]; then warn "Lock PID reused while waiting — clearing" rm -f "$lockfile" break fi done if [[ -f "$lockfile" ]]; then error "$script_name still locked after ${LOCK_WAIT_TIMEOUT}s — exiting" exit 1 fi else error "Another instance of $script_name is already running (PID $existing_pid) — exiting" case "$script_name" in fallback|transcode_management|daily_sync_maintenance|stability_watchdog) notify "$script_name lock collision on $(hostname) — concurrent instance detected" "$script_name" "warning" ;; esac exit 1 fi fi fi # Acquire lock — store PID:scriptname to prevent PID reuse false positives echo "$$:$script_name" > "$lockfile" _register_lock "$lockfile" log "$ICON_LOCK Lock acquired: $script_name (PID $$)" } # Acquire per-profile rsync lock + enforce global concurrent limit. # Prevents same profile running twice and limits total concurrent rsync instances. # Usage: acquire_rsync_lock "$PROFILE_NAME" acquire_rsync_lock() { local profile="$1" local profile_lock profile_lock="$(_lock_file "rsync_${profile}")" mkdir -p "$LOCK_DIR" # Per-profile lock — same profile cannot run twice if [[ -f "$profile_lock" ]]; then local lock_content existing_pid locked_name lock_content=$(cat "$profile_lock" 2>/dev/null) existing_pid="${lock_content%%:*}" locked_name="${lock_content##*:}" if [[ -n "$existing_pid" ]] && kill -0 "$existing_pid" 2>/dev/null && \ [[ "$locked_name" == "rsync_${profile}" ]]; then error "rsync profile '$profile' is already running (PID $existing_pid) — exiting" exit 1 else warn "Stale rsync lock for profile '$profile' — clearing" rm -f "$profile_lock" fi fi # Global concurrent limit — validate count against actual live locks local current_count=0 if [[ -f "$RSYNC_COUNT_FILE" ]]; then current_count=$(cat "$RSYNC_COUNT_FILE" 2>/dev/null || echo 0) local actual_count=0 for lf in "$LOCK_DIR"/rsync_*.lock; do [[ -f "$lf" ]] || continue local lpid lpid=$(cat "$lf" 2>/dev/null) kill -0 "$lpid" 2>/dev/null && ((actual_count++)) done if [[ "$actual_count" -ne "$current_count" ]]; then log "rsync count corrected: $current_count → $actual_count" current_count=$actual_count echo "$current_count" > "$RSYNC_COUNT_FILE" fi fi if [[ "$current_count" -ge "$RSYNC_MAX_CONCURRENT" ]]; then error "Maximum concurrent rsync limit ($RSYNC_MAX_CONCURRENT) reached — exiting" info "Active rsync locks: $(ls "$LOCK_DIR"/rsync_*.lock 2>/dev/null | \ xargs -I{} basename {} .lock | tr '\n' ' ')" exit 1 fi # Acquire profile lock and increment counter echo "$$:rsync_${profile}" > "$profile_lock" echo $(( current_count + 1 )) > "$RSYNC_COUNT_FILE" _register_lock "$profile_lock" log "$ICON_LOCK rsync lock acquired: profile '$profile' (PID $$, active: $(( current_count + 1 ))/$RSYNC_MAX_CONCURRENT)" } # Internal — release rsync lock and decrement global counter on exit _release_rsync_on_exit() { local profile_lock="$1" [[ -f "$profile_lock" ]] && rm -f "$profile_lock" if [[ -f "$RSYNC_COUNT_FILE" ]]; then local count count=$(cat "$RSYNC_COUNT_FILE" 2>/dev/null || echo 1) count=$(( count - 1 )) [[ "$count" -lt 0 ]] && count=0 echo "$count" > "$RSYNC_COUNT_FILE" fi } # ============================================================================================== # ── PATH TRANSLATION ────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Translates container-internal file paths to host paths for arr cleanup scripts. # Arr APIs return file paths as seen inside the container — scripts need host paths to scan. # # Uses ARR_PATH_MAP associative array — must be declared before calling. # Longest-match wins — prevents partial path collisions. # # Usage: # declare -A ARR_PATH_MAP=(["/ext-music"]="/mnt/user/Music-New") # translate_path "/ext-music/Artist/Album/track.flac" # Returns: /mnt/user/Music-New/Artist/Album/track.flac # # If no match found — returns path unchanged (safe fallback). translate_path() { local api_path="$1" local best_match="" local best_len=0 for container_path in "${!ARR_PATH_MAP[@]}"; do if [[ "$api_path" == "$container_path"* ]]; then if [[ "${#container_path}" -gt "$best_len" ]]; then best_match="$container_path" best_len="${#container_path}" fi fi done if [[ -n "$best_match" ]]; then echo "${ARR_PATH_MAP[$best_match]}${api_path#$best_match}" else echo "$api_path" # no match — return unchanged fi } # ============================================================================================== # ── ARR VERSION CHECK ───────────────────────────────────────────────────────────────────────── # ============================================================================================== # Verifies arr major version matches tested version in master.conf. # Exits if version doesn't match — prevents running against untested API structure. # Warns and proceeds if version endpoint is unreachable. # # Usage: # check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" "Radarr" # # Arguments: # $1 = arr base URL # $2 = API key # $3 = API path prefix (v3 or v1) # $4 = expected major version number from master.conf # $5 = arr name for error messages check_arr_version() { local url="$1" local api_key="$2" local api_prefix="$3" local expected_major="$4" local arr_name="${5:-Arr}" local status_response version major_version status_response=$(curl -sf --max-time 10 \ -H "X-Api-Key: $api_key" \ "${url}/api/${api_prefix}/system/status" 2>/dev/null) if [[ -z "$status_response" ]]; then warn "$arr_name version check failed — could not reach system/status endpoint" warn "Proceeding without version verification — monitor for API errors" return 0 fi version=$(echo "$status_response" | \ grep -o '"version": *"[^"]*"' | \ grep -o '[0-9][^"]*' | head -1) if [[ -z "$version" ]]; then warn "$arr_name version check failed — could not parse version from response" warn "Proceeding without version verification — monitor for API errors" return 0 fi major_version="${version%%.*}" if [[ "$major_version" == "$expected_major" ]]; then success "$arr_name version: $version (major $major_version — tested ✅)" return 0 else error "$arr_name version mismatch — running v${major_version}, tested against v${expected_major}" error "The API endpoint structure may have changed — exiting to protect your library" error "Update ${arr_name^^}_VERSION_MAJOR in master.conf after verifying the script works with v${major_version}" notify "$arr_name version mismatch on $(hostname) — running v${major_version}, script tested against v${expected_major}" "$arr_name Cleanup" "warning" exit 1 fi } # ============================================================================================== # ── EMBY LIBRARY SCAN ───────────────────────────────────────────────────────────────────────── # ============================================================================================== # Triggers Emby "Clean Missing Files" scheduled task after arr cleanup scripts delete files. # Without this Emby keeps showing deleted files as ghost entries until its next scheduled scan. # User clicks a ghost entry → file not found error. This prevents that entirely. # # Uses EMBY_URL and EMBY_API_KEY — aliased by detect_hosts() from HOST*_EMBY_* vars. # Skips cleanly if Emby is not configured on this host (empty URL or API key). # # Emby task discovery: # GET /ScheduledTasks → find task where Name contains "Clean Missing" # POST /ScheduledTasks/Running/{taskId} → trigger it # # Non-fatal — logs warning if task cannot be triggered but does not exit. # Clean Missing Files runs across all libraries at once — one call handles # music, TV, and movies together regardless of which arr triggered the cleanup. # # Usage: notify_emby_scan (no args — always triggers Clean Missing Files) notify_emby_scan() { # Skip if Emby not configured on this host if [[ -z "${EMBY_URL:-}" ]] || [[ -z "${EMBY_API_KEY:-}" ]]; then log "Emby not configured on $MY_ID — skipping library scan notification" return 0 fi log "Notifying Emby to clean missing files..." # Get all scheduled tasks local tasks_response tasks_response=$(curl -sf --max-time 15 \ -H "X-Api-Key: $EMBY_API_KEY" \ "${EMBY_URL}/ScheduledTasks" 2>/dev/null) if [[ -z "$tasks_response" ]]; then warn "Could not reach Emby scheduled tasks API at $EMBY_URL — skipping scan" return 0 fi # Find the "Clean Missing Files" task ID local task_id task_id=$(echo "$tasks_response" | \ grep -o '"Id":"[^"]*","Name":"[^"]*Clean Missing[^"]*"' | \ grep -o '"Id":"[^"]*"' | \ sed 's/"Id":"//;s/"//' | head -1) # Fallback — try "Scan Media Library" if Clean Missing not found if [[ -z "$task_id" ]]; then task_id=$(echo "$tasks_response" | \ grep -o '"Id":"[^"]*","Name":"[^"]*Scan Media Library[^"]*"' | \ grep -o '"Id":"[^"]*"' | \ sed 's/"Id":"//;s/"//' | head -1) [[ -n "$task_id" ]] && log "Clean Missing Files not found — using Scan Media Library" fi if [[ -z "$task_id" ]]; then warn "Could not find Emby Clean Missing Files or Scan Media Library task" warn "Check Emby scheduled tasks — ghost entries will persist until next Emby scan" return 0 fi log "Found Emby task ID: $task_id" # Trigger the task local http_code http_code=$(curl -sf --max-time 15 \ -X POST \ -H "X-Api-Key: $EMBY_API_KEY" \ -w "%{http_code}" \ -o /dev/null \ "${EMBY_URL}/ScheduledTasks/Running/${task_id}" 2>/dev/null) if [[ "$http_code" == "204" ]] || [[ "$http_code" == "200" ]]; then warn "$ICON_EMBY Emby Clean Missing Files triggered — ghost entries will be removed" else warn "Emby task trigger returned HTTP $http_code — ghost entries may persist" warn "Check Emby dashboard — manually run Clean Missing Files if needed" fi } # ============================================================================================== # ── API REACHABILITY CHECK ──────────────────────────────────────────────────────────────────── # ============================================================================================== # Pre-flight check — verifies an API endpoint is reachable before attempting operations. # Non-fatal — returns 0 if reachable, 1 if not. Caller decides whether to exit. # # Usage: check_api "http://localhost:8989" "Sonarr" || exit 1 check_api() { local url="$1" local service="${2:-API}" local timeout="${3:-10}" if curl -sf --max-time "$timeout" "$url" >/dev/null 2>&1; then log "$service API reachable: $url" return 0 else error "$service API not reachable: $url" return 1 fi } # ============================================================================================== # ── UNRAID VERSION PARITY CHECK ─────────────────────────────────────────────────────────────── # ============================================================================================== # Verifies local and remote servers are running compatible unRAID versions before any # remote operation. Version mismatches can mean changed APIs, commands, or behaviours # that silently break remote container operations, rsync, or failover logic. # # Reads /etc/unraid-version on both sides — format: version="7.2.3" # # Mismatch behaviour (UNRAID_VERSION_MISMATCH_ACTION in master.conf): # "warn" — log warning and continue (default for minor/patch differences) # "abort" — exit the calling script (default for major version differences) # Major version mismatch always aborts regardless of setting # # Usage: check_unraid_version_parity || exit 1 check_unraid_version_parity() { local local_version remote_version # Read local version if [[ ! -f /etc/unraid-version ]]; then warn "Cannot read local /etc/unraid-version — skipping version parity check" return 0 fi local_version=$(grep -oP '(?<=version=")[^"]+' /etc/unraid-version 2>/dev/null) if [[ -z "$local_version" ]]; then warn "Cannot parse local unRAID version — skipping parity check" return 0 fi # Read remote version via SSH remote_version=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ "grep -oP '(?<=version=\")[^\"]+' /etc/unraid-version 2>/dev/null" 2>/dev/null) if [[ -z "$remote_version" ]]; then warn "Cannot read remote unRAID version from $REMOTE_SERVER_NAME — skipping parity check" return 0 fi log "Version parity: local=$local_version remote=$remote_version" if [[ "$local_version" == "$remote_version" ]]; then log "unRAID versions match: $local_version ✅" return 0 fi # Parse major versions local local_major remote_major local_major="${local_version%%.*}" remote_major="${remote_version%%.*}" if [[ "$local_major" != "$remote_major" ]]; then error "unRAID MAJOR version mismatch — local: $local_version remote: $remote_version" error "Major version differences may break remote APIs, commands, and behaviours" error "Update both servers to the same major version before running remote operations" notify "unRAID major version mismatch on $(hostname) — local: $local_version remote: $remote_version — remote operations aborted" "Version Parity" "warning" return 1 fi # Minor/patch mismatch — action depends on config local action="${UNRAID_VERSION_MISMATCH_ACTION:-warn}" warn "unRAID version mismatch — local: $local_version remote: $remote_version" if [[ "$action" == "abort" ]]; then error "UNRAID_VERSION_MISMATCH_ACTION=abort — refusing to continue" notify "unRAID version mismatch on $(hostname) — local: $local_version remote: $remote_version — aborted" "Version Parity" "warning" return 1 fi warn "UNRAID_VERSION_MISMATCH_ACTION=warn — continuing despite mismatch (monitor for issues)" return 0 } # ============================================================================================== # ── REMOTE DOCKER DAEMON CHECK ──────────────────────────────────────────────────────────────── # ============================================================================================== # Verifies the remote Docker daemon is responsive before issuing any remote container commands. # A hung remote daemon causes docker commands to hang or silently fail — checking first # prevents cascading failures and gives clear diagnostic output. # # Strike system: # Each consecutive failed check adds a strike to REMOTE_DOCKER_STRIKES # Below REMOTE_DOCKER_STRIKE_LIMIT → warn and return 1 (caller skips operation) # At limit → notify critical and return 1 (caller should exit or escalate) # Strikes reset when daemon recovers # # Usage: check_remote_docker_daemon || return 1 REMOTE_DOCKER_STRIKES=0 check_remote_docker_daemon() { local timeout="${1:-10}" if ssh -i "$SSH_KEY" -o ConnectTimeout="$timeout" root@"$REMOTE_SERVER" \ "timeout $timeout docker info" >/dev/null 2>&1; then # Daemon healthy — clear strikes if we had issues if [[ "$REMOTE_DOCKER_STRIKES" -gt 0 ]]; then info "Remote Docker daemon on $REMOTE_SERVER_NAME recovered — clearing strikes" REMOTE_DOCKER_STRIKES=0 fi log "Remote Docker daemon healthy on $REMOTE_SERVER_NAME" return 0 fi REMOTE_DOCKER_STRIKES=$(( REMOTE_DOCKER_STRIKES + 1 )) local limit="${REMOTE_DOCKER_STRIKE_LIMIT:-3}" warn "Remote Docker daemon not responding on $REMOTE_SERVER_NAME (strike $REMOTE_DOCKER_STRIKES/$limit)" if [[ "$REMOTE_DOCKER_STRIKES" -ge "$limit" ]]; then error "Remote Docker daemon unresponsive on $REMOTE_SERVER_NAME after $limit consecutive checks" error "Skipping all remote container operations — check Docker on $REMOTE_SERVER_NAME" notify "Remote Docker daemon unresponsive on $REMOTE_SERVER_NAME — remote operations skipped on $(hostname)" "Remote Docker Daemon" "warning" fi return 1 } # ============================================================================================== # ── UNRAID COMMAND VALIDATION ───────────────────────────────────────────────────────────────── # ============================================================================================== # Verifies a unRAID-specific command exists and produces expected output before use. # Protects against commands moving, changing format, or disappearing after upgrades. # # If validation fails — notifies and returns 1. Caller exits its own section only. # Other scripts that pass validation are unaffected. # # Usage: # validate_unraid_cmd # "/usr/local/emhttp/plugins/dynamix/scripts/notify" # "--help" # "Usage" # "unRAID notify script" # # Arguments: # $1 = full path to command # $2 = test argument to pass (use "" for no argument) # $3 = expected string in output (use "" to skip output check) # $4 = human readable name for error messages validate_unraid_cmd() { local cmd_path="$1" local test_arg="$2" local expected_output="$3" local cmd_name="${4:-$1}" # Check command exists and is executable if [[ ! -x "$cmd_path" ]]; then error "unRAID command not found or not executable: $cmd_path" error "$cmd_name may have moved or been removed — check after recent unRAID upgrade" notify "$cmd_name not found on $(hostname) at $cmd_path — check after unRAID upgrade" "Command Validation" "warning" return 1 fi # Check output matches expected pattern if provided if [[ -n "$expected_output" ]]; then local actual_output actual_output=$(timeout 10 "$cmd_path" $test_arg 2>&1 || true) if ! echo "$actual_output" | grep -q "$expected_output"; then error "$cmd_name output format changed — expected '$expected_output' not found" error "Command may have changed after unRAID upgrade — review $cmd_path" notify "$cmd_name output format changed on $(hostname) — may need script update after unRAID upgrade" "Command Validation" "warning" return 1 fi fi log "$cmd_name validated: $cmd_path ✅" return 0 } # ============================================================================================== # ── STATUS DISPLAY ──────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Prints current runtime configuration — triggered by --status flag in any script. # Useful for verifying detect_hosts() resolved correctly and profile loaded as expected. show_status() { local local_ver local_ver=$(grep -oP '(?<=version=")[^"]+' /etc/unraid-version 2>/dev/null || echo "unknown") echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" echo "Local: $LOCAL_SERVER_NAME" echo "Remote: $REMOTE_SERVER_NAME" echo "IP: ${REMOTE_SERVER:-not resolved}" echo "My ID: ${MY_ID:-not set}" echo "Remote ID: ${REMOTE_ID:-not set}" echo "unRAID ver: $local_ver" echo "Profile: ${PROFILE_NAME:-n/a}" echo "DryRun: ${DRY_RUN:-false}" echo "Logging: ${ENABLE_LOGGING:-false}" echo "SSH Key: ${SSH_KEY:-not set}" echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]:-n/a}" echo "Delayed: ${DELAYED_CONTAINERS[*]:-n/a}" echo "Excludes: ${EXCLUDE_DIRS[*]:-n/a}" echo "━━━━━━━━━━━━━━━━━━━━━━━" }