#!/bin/bash # ============================================================================================== # ================================= COMMON LIBRARY ============================================= # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Shared functions used by every script in the ecosystem. # Sourced automatically by load_config.sh — do not source directly. # # ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # # A pure function library. It defines and returns; it starts nothing on its own. # # Load order matters and is owned by load_config.sh: # varaverk.cfg → master.conf → host*.conf → common.sh → Plugin/$PLATFORM/adapter.sh # # common.sh is sourced AFTER the conf files because detect_hosts() and the health checks # need HOST* and the thresholds to already exist. It is sourced BEFORE the adapter so the # adapter can rely on log()/warn()/error() being defined. # # Note it defines detect_hosts() but never calls it. MY_ID and REMOTE_ID stay unset until a # script calls it itself — anything building HOST*-prefixed variable names must do so after # that call, not before. # # ============================================================================================== # DESIGN PRINCIPLES # ============================================================================================== # # Define, Never Act # Sourcing this file changes no system state: no containers touched, no files written, no # locks taken, no host detected. Every script in the ecosystem sources it, including # read-only reporting ones, so anything that acted here would act everywhere. # # One Implementation of Each Shared Behaviour # Locking, host detection, notification, retry and timeout wrapping live here once. When # a safeguard needs strengthening it is strengthened for every caller at the same moment — # which is why individual scripts call acquire_lock() rather than rolling their own flock. # # Platform-Agnostic # No OS-specific paths or commands. Anything Unraid-specific belongs in the adapter, which # is why this file calls platform_*() rather than touching /etc/rc.d or /boot directly. # # Callers Own Policy # Helpers return status; they do not decide what failure means. retry_docker() reports # that a command failed after N attempts — whether that aborts a run, skips an item, or # raises a notification is the caller's call, and only the caller has the context. # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # No Root, No Lock, No detect_hosts on Load — Deliberate # This is a sourced library and must stay inert. A root check here would fire for every # script including ones that legitimately do not need it; a lock would be taken on every # source; calling detect_hosts() automatically would exit the caller on an unknown # hostname before it had a chance to handle that itself. Do not add them. # # Safe Defaults on Every Threshold # Helpers apply :- defaults (DOCKER_TIMEOUT, RESTART_VERIFY_WAIT, RETRY_COUNT and friends) # so a conf missing a key degrades to a sane value rather than an empty string that would # silently disable a timeout or a retry ceiling. # # Timeout Wrapping Provided Centrally # docker_cmd() and the remote helpers wrap calls in timeout, so no caller has to remember # to. A hung daemon or unreachable partner cannot stall a scheduled window. # # Locking Provided Centrally # acquire_lock() implements strict, wait and continuous modes in one place, so every # script gets identical semantics and a fix applies everywhere at once. # # Host Identity Is Exact-Match Only # detect_hosts() matches hostname exactly, with a single explicit NetBIOS-truncation # fallback that requires an unambiguous candidate. No fuzzy or similarity matching — a # wrong host identity would alias the wrong credentials and paths into every script. # # ============================================================================================== # CONFIGURATION # ============================================================================================== # # Defines no config of its own. It consumes what load_config.sh has already sourced from # master.conf and host*.conf, and applies defaults where a key may be absent. # # Consumed broadly: DOCKER_TIMEOUT, RESTART_VERIFY_WAIT, RETRY_COUNT, SLEEP, SSH_KEY, # SSH_TIMEOUT, STATE_DIR, DATA_DIR, LOCK_DIR, NOTIFY_UNRAID, HOST*, and the RSYNC_*/ # PARTNERSHIP_* gates. # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # None — sourced, never executed: # # source "$SCRIPT_DIR/../load_config.sh" # which sources this file # # It parses no arguments of its own. parse_args() is defined here for callers to invoke, # and --dry-run/--status/--log are honoured by the calling script, not by this file. # # ============================================================================================== # ── WHAT THIS FILE PROVIDES ─────────────────────────────────────────────────────────────────── # Icons — consistent visual language across all script output # Output helpers — info, warn, error, success, log, notify # Arg parser — unified --dry-run, --log, --status, KEY=VALUE handling # Validation — require_var, validate_int # Host detection — detect_hosts() identifies local/remote server, sets all aliases # Connectivity — resolve_remote_ip, check_connectivity, ping_remote, ping_internet # Health checks — local/remote array, docker daemon, rootfs, share, disk temps # Disk verification — check_remote_disks() — XFS, ZFS, cache pools, nested paths # Container mgmt — stop/start local and remote containers with state tracking # Rsync helpers — get_rsync_opts, check_rsync_enabled # Locking — acquire_lock, acquire_rsync_lock — concurrent execution protection # Path translation — translate_path() — container → host path for arr cleanup scripts # Arr utilities — check_arr_version() — API version safety gate # Status display — show_status() — runtime config dump on --status flag # # ── CHANGELOG ───────────────────────────────────────────────────────────────────────────────── # # Version What changed # ─────────────────────────────────────────────────────────────────────────────────────────── # v1.0 Initial stable framework — output helpers, arg parser, host detection # # v1.1 format_duration() moved here from daily_sync_maintenance.sh for shared use # SSH_KEY naming collision resolved — gitea key renamed GITEA_SSH_KEY in master.conf # # v1.2 check_connectivity() added — fatal ping with Tailscale hint on failure # Function header comment blocks standardised across all functions # # v1.3 check_remote_rootfs() — aborts rsync if remote rootfs exceeds ROOTFS_WARN # check_remote_share() — aborts if target directory missing or empty on remote # Both protect against rsync running when remote array is down # # v1.4 check_remote_disks() — verifies all physical disks backing a share are mounted # Auto-discovers disk layout at runtime from disks.ini — no config required # Aborts if any disk backing the share is offline or unmounted # # v1.5-2.9 Icon set expanded progressively — each operation has its own distinct icon # All function output updated to use correct icon per context # notify() added — unRAID native + Discord webhook notifications # Script locking system added — acquire_lock(), acquire_rsync_lock() # translate_path() added — container → host path for arr cleanup scripts # check_arr_version() added — API version safety gate before arr operations # check_api() added — pre-flight API reachability check # ping_remote(), ping_internet() — non-fatal ping for fallback use # check_remote_array() — array health check # get_unraid_temp_thresholds() — reads thresholds from dynamix.cfg # check_local_disk_temps() — pre-rsync temp check with exit codes 0/1/2 # stop/start local containers added alongside existing remote variants # # v3.1 Three-file config split — master.conf + host1.conf + host2.conf # load_config.sh introduced — auto-discovers all host*.conf files # detect_hosts() rewritten — sets MY_ID/REMOTE_ID and aliases all HOST* vars # All scripts now source load_config.sh instead of conf files directly # Adding a new server = add host*.conf, zero script changes required # # v3.2 check_remote_disks() rewritten — three-tier detection: # Tier 1: array disk paths (/mnt/disk*/sharename) # Tier 2: ZFS pools with find -maxdepth 2 (catches nested cache paths) # Tier 3: shfs fallback — verifies /mnt/user is mounted # Fixes: nested paths like /mnt/cache/appdata-Fallback/Critical-Data # Fixes: ZFS cache pools not detected when share is not at pool root # # v3.3 PROFILE_REMOTE_RESTART_CONTAINERS support added # Dirty sync profiles (critical-fallback) restart remote # containers after sync if they were running before — picks up config changes # Was stopped → stays stopped. Was running → gets restarted. ✅ # # v3.5 notify_emby_scan() added — triggers Emby "Clean Missing Files" task # after arr cleanup scripts delete orphaned files # Emby immediately removes ghost entries — no user-facing file-not-found errors # Called by lidarr/sonarr/radarr_cleanup.sh when files are deleted # # # Two new safety functions added: # check_os_version_parity() — refuses remote ops on version mismatch # reads OS version via platform_get_os_version() / platform_os_version_probe_cmd() # major mismatch → abort | minor mismatch → configurable warn/abort # check_remote_docker_daemon() — verifies remote Docker daemon before # issuing any remote container commands — strike system → skip/retry/exit # # ── VERSION ─────────────────────────────────────────────────────────────────────────────────── # Current: v3.5 # ============================================================================================== # Cron runs with a minimal PATH (/usr/bin:/bin) that omits /usr/local/sbin where tailscale, # and other Unraid tools live. Prepend it here so every sourcing script finds them. export PATH="/usr/local/sbin:/usr/local/bin:$PATH" # ============================================================================================== # ── ICONS ───────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Each icon has one job — do not reuse across different contexts. # Adding a new icon: add to the appropriate group below with a comment describing its job. # System / Host ICON_HOST="🖥️" # host detection and identity ICON_NET="🌐" # network / IP resolution ICON_PING="📡" # connectivity check ICON_GEAR="⚙️" # setup section header / profile load # Health Checks ICON_DISK="💾" # disk checks ICON_HEALTH="🩺" # rootfs / share health checks ICON_SHIELD="🛡️" # pre-flight section header # Containers ICON_CONTAINERS="📦" # container section anchor ICON_STOP="⛔" # stop command being issued ICON_STOPPED="🔴" # container confirmed stopped ICON_START="▶️" # start command being issued ICON_STARTED="💚" # container confirmed started ICON_RUNNING="🟢" # container already running when checked ICON_SKIP="⏭️" # skipping — already running healthy instance ICON_PROTECTED="🔰" # file/item is protected — never delete ICON_NOT_RUNNING="⭕" # container already stopped when checked # Transfer ICON_SYNC="🔄" # transfer section header ICON_RUN="🚀" # sync starting / rsync attempt ICON_RETRY="🔁" # retry attempt ICON_DONE="🏁" # transfer complete ICON_GIT="🌱" # git pull/push section header # Summary ICON_SUMMARY="📋" # summary section header ICON_TIME="⏱️" # duration line # System Operations ICON_MOVER="🔃" # mover operations ICON_REBOOT="⚡" # scheduled server reboot ICON_REBOOT_SMART="🚨" # smart conditional reboot triggered ICON_PLUGIN="🧩" # user scripts plugin operations ICON_PHP="👥" # PHP-FPM operations ICON_WEBGUI="💻" # WebGUI / nginx / emhttp operations # Diagnostics ICON_ZFS="📊" # ZFS ARC statistics ICON_MEM="🧠" # memory status ICON_WATCHDOG="🐾" # docker watchdog monitoring operations # Media Operations ICON_CLEAN="🧹" # media cleaner operations ICON_TRASH="🗑️" # files being deleted ICON_PERMS="🔐" # permissions operation / section header ICON_UNLOCKED="🔓" # permissions successfully applied to a share ICON_LOCK="🔏" # script instance lock — acquired/released # Transcode Operations ICON_RAM="💨" # ramdisk operations — fast ephemeral storage ICON_LINK="🔗" # symlink state and management # fallback Operations ICON_FALLBACK="🔀" # fallback state changes and operations # Docker Network Operations ICON_DOCKER_NET="🔌" # Docker network connect operations # Security / Certificate Operations ICON_CERT="🔒" # SSL certificate monitoring # Monitor Operations ICON_MONITOR="📈" # monitoring section headers and general monitoring ICON_SMART="🔧" # drive SMART health attribute monitoring ICON_BANDWIDTH="📶" # bandwidth usage tracking and reporting ICON_DIGEST="📰" # health digest — aggregated system summary ICON_EMBY="🎬" # Emby media server session reporting ICON_VERIFY="✔️" # backup verification — checksum comparison # Notifications ICON_NOTIFY="🔔" # notification operations # Output ICON_INFO="ℹ️" ICON_WARN="⚠️" ICON_ERROR="❌" ICON_SUCCESS="✅" # ============================================================================================== # ── OUTPUT HELPERS ──────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Standardised output functions used across all scripts. # # Output tiers: # echo — always visible — summaries, status conclusions, section headers # info() — always visible — status confirmations, reachability, check pass # warn() — always visible — state transitions, warnings, important events # error() — always visible — something broke # success() — always visible — operation completed successfully # log() — only with --log flag — per-item detail, internal checks # # All output goes to stdout — callers can redirect as needed. info() { echo "$ICON_INFO [INFO] $*"; } warn() { echo "$ICON_WARN [WARN] $*"; } error() { echo "$ICON_ERROR [ERROR] $*"; } success() { echo "$ICON_DONE [OK] $*"; } log() { [[ "${ENABLE_LOGGING:-false}" == true ]] && echo "[LOG] $*" return 0 } # ============================================================================================== # ── NOTIFICATION ────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Sends a notification via unRAID native system and/or Discord webhook. # Both channels are optional and independently controlled: # NOTIFY_UNRAID — shared toggle in master.conf # MY_DISCORD_WEBHOOK — per-host in host*.conf, set by detect_hosts() # # Severity levels: normal, warning, alert # Usage: notify "message" "subject" "severity" notify() { local message="$1" local subject="${2:-unRAID Notification}" local severity="${3:-normal}" log "$ICON_NOTIFY Sending notification: $subject — $message" if [[ "${NOTIFY_UNRAID:-false}" == true ]]; then if platform_send_os_notification "$message" "$subject" "$severity"; then log "$ICON_NOTIFY OS notification sent" else log "$ICON_NOTIFY OS notify not available — skipping" fi fi # Uses MY_DISCORD_WEBHOOK — set by detect_hosts() from HOST*_DISCORD_WEBHOOK if [[ -n "${MY_DISCORD_WEBHOOK:-}" ]]; then local payload payload=$(printf '{"content": "%s — **%s**\\n%s"}' \ "$ICON_NOTIFY" "$subject" "$message") if curl -s -H "Content-Type: application/json" \ -d "$payload" "$MY_DISCORD_WEBHOOK" >/dev/null 2>&1; then log "$ICON_NOTIFY Discord notification sent" else warn "Discord notification failed — check HOST*_DISCORD_WEBHOOK in host*.conf" fi fi } # ============================================================================================== # ── DURATION FORMATTER ──────────────────────────────────────────────────────────────────────── # ============================================================================================== # Converts raw seconds into a human readable string. # Examples: 47 → "47s" | 653 → "10m53s" # Usage: format_duration $SECONDS format_duration() { local secs=$1 local days=$(( secs / 86400 )) local hrs=$(( (secs % 86400) / 3600 )) local mins=$(( (secs % 3600) / 60 )) local rem=$(( secs % 60 )) if [[ $days -gt 0 ]]; then echo "${days}d ${hrs}h ${mins}m" elif [[ $hrs -gt 0 ]]; then echo "${hrs}h ${mins}m" elif [[ $mins -gt 0 ]]; then echo "${mins}m${rem}s" else echo "${rem}s"; fi } # Tiered human-readable size — picks B/MB/GB based on magnitude. # Usage: format_bytes 5368709120 → 5.0GB format_bytes() { local bytes=${1:-0} if (( bytes > 1073741824 )); then awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}" elif (( bytes > 1048576 )); then awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}" else echo "${bytes}B" fi } # Fixed-precision bytes → GB (no tiering) — for values always compared against a GB threshold. # Usage: bytes_to_gb 5368709120 [decimals=2] → 5.00 bytes_to_gb() { local bytes=${1:-0} decimals=${2:-2} awk "BEGIN {printf \"%.${decimals}f\", $bytes / 1073741824}" } # Fixed-precision KB → GB (no tiering) — df --output=used/avail reports in KB. # Usage: kb_to_gb 5242880 [decimals=2] → 5.00 kb_to_gb() { local kb=${1:-0} decimals=${2:-2} awk "BEGIN {printf \"%.${decimals}f\", $kb / 1048576}" } # ============================================================================================== # ── ARG PARSER ──────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Processes all flags and key=value pairs passed to any script. # Call early in every script: parse_args "$@" # # Supported flags: # --dry-run | -n → DRY_RUN=true # --log → ENABLE_LOGGING=true # --no-log → ENABLE_LOGGING=false # --status → SHOW_STATUS=true # --help | -h → print usage and exit # # Supported key=value: # LOG=true/false → toggle logging # BW_LIMIT=5000 → override any declared master.conf variable for this run # # Unparsed positional args returned in PARSED_ARGS array. parse_args() { ENABLE_LOGGING=${ENABLE_LOGGING:-false} DRY_RUN=${DRY_RUN:-false} SHOW_STATUS=${SHOW_STATUS:-false} CLEAN_ARGS=() for ARG in "$@"; do if [[ "$ARG" == *=* ]]; then VAR="${ARG%%=*}" VAL="${ARG#*=}" case "$VAR" in LOG) [[ "$VAL" == "true" ]] && ENABLE_LOGGING=true [[ "$VAL" == "false" ]] && ENABLE_LOGGING=false ;; *) if declare -p "$VAR" &>/dev/null; then printf -v "$VAR" '%s' "$VAL" log "Set $VAR=$VAL" else warn "Unknown variable: $VAR" fi ;; esac else case "$ARG" in --dry-run|-n) DRY_RUN=true ;; --log) ENABLE_LOGGING=true ;; --no-log) ENABLE_LOGGING=false ;; --status|--summary) SHOW_STATUS=true ;; --help|-h) echo "Usage: script [--dry-run] [--log] [--status]" exit 0 ;; *) CLEAN_ARGS+=("$ARG") ;; esac fi done PARSED_ARGS=("${CLEAN_ARGS[@]}") } # ============================================================================================== # ── VALIDATION HELPERS ──────────────────────────────────────────────────────────────────────── # ============================================================================================== # Exits with error if a required variable is empty or unset. # Usage: require_var VAR_NAME require_var() { [[ -z "${!1:-}" ]] && error "Missing required: $1" && exit 1 } # Exits with error if a variable is not a valid positive integer. # Usage: validate_int VAR_NAME "$VAR_VALUE" validate_int() { local name="$1" value="$2" if [[ -z "$value" ]]; then error "$name is not set — check master.conf" exit 1 fi if ! [[ "$value" =~ ^[0-9]+$ ]]; then error "$name must be a positive integer — got: '$value'" exit 1 fi log "$name validated: $value" } # Membership check against a list of values (e.g. an ignore list). # Usage: is_in_list "$needle" "${HAYSTACK_ARRAY[@]}" is_in_list() { local needle="$1"; shift local item for item in "$@"; do [[ "$needle" == "$item" ]] && return 0 done return 1 } # ============================================================================================== # ── HOST DETECTION ──────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Determines which server is local and which is remote by comparing hostname against # all HOST* values discovered from host*.conf files. # # Sets: # MY_ID — "HOST1" or "HOST2" (the role key, not the hostname) # REMOTE_ID — the peer's role key # LOCAL_SERVER_NAME — local hostname string (e.g. "unRAID-Gmer4Lfe") # REMOTE_SERVER_NAME — remote hostname string # SSH_KEY — SSH key for server-to-server operations # REMOTE_STORAGE_PATH — remote's storage root from HOST*_STORAGE_PATH in host*.conf; # falls back to platform_storage_path() for unconfigured pairs # # Also sets aliases for all host-specific arrays so scripts use unprefixed names: # DAILY_SYNC_SHARES ← HOST*_DAILY_SYNC_SHARES # INTERMEDIATE_SYNC_SHARES ← HOST*_INTERMEDIATE_SYNC_SHARES # WEEKLY_SYNC_SHARES ← HOST*_WEEKLY_SYNC_SHARES # CRITICAL_SYNC_SHARES ← HOST*_CRITICAL_SYNC_SHARES # DAILY_RESTART_CONTAINERS ← HOST*_DAILY_RESTART_CONTAINERS # WEEKLY_RESTART_CONTAINERS ← HOST*_WEEKLY_RESTART_CONTAINERS # WATCHDOG_CONTAINERS ← HOST*_WATCHDOG_CONTAINERS (associative) # WATCHDOG_CONTAINER_URLS ← HOST*_WATCHDOG_CONTAINER_URLS (associative) # WATCHDOG_REQUIRED_CONTAINERS ← HOST*_WATCHDOG_REQUIRED_CONTAINERS # WATCHDOG_SCAN_IGNORE ← HOST*_WATCHDOG_SCAN_IGNORE # WATCHDOG_DEPENDENCIES ← HOST*_WATCHDOG_DEPENDENCIES (associative) # WATCHDOG_APPDATA_SIZES ← HOST*_WATCHDOG_APPDATA_SIZES (associative) # NETWORK_CONNECT_CONTAINERS ← HOST*_NETWORK_CONNECT_CONTAINERS # NETWORK_CONNECT_NETWORKS ← HOST*_NETWORK_CONNECT_NETWORKS # MEDIA_PERMISSION_SHARES ← HOST*_MEDIA_PERMISSION_SHARES # ANIME_CLEAN_FOLDERS ← HOST*_ANIME_CLEAN_FOLDERS # MEDIA_CLEAN_FOLDERS ← HOST*_MEDIA_CLEAN_FOLDERS # CERT_MONITOR_DOMAINS ← HOST*_CERT_MONITOR_DOMAINS # SMART_IGNORE_DRIVES ← HOST*_SMART_IGNORE_DRIVES # ZFS_REPORT_IGNORE_POOLS ← HOST*_ZFS_REPORT_IGNORE_POOLS # TRANSCODE_SSD ← HOST*_TRANSCODE_SSD # TRANSCODE_SERVERS ← HOST*_TRANSCODE_SERVERS # RAMDISK_SIZE ← HOST*_RAMDISK_SIZE # RAMDISK_WARN_GB ← HOST*_RAMDISK_WARN_GB # RAMDISK_LOW_GB ← HOST*_RAMDISK_LOW_GB # BACKUP_VERIFY_SHARES ← HOST*_BACKUP_VERIFY_SHARES # DDNS_CONTAINERS ← HOST*_DDNS_CONTAINERS # PARTNERSHIP_AUTH_WEBUIS ← HOST*_PARTNERSHIP_AUTH_WEBUIS # PARTNERSHIP_MIRROR_BACKUPS ← HOST*_PARTNERSHIP_MIRROR_BACKUPS # MY_DISCORD_WEBHOOK ← HOST*_DISCORD_WEBHOOK # EMBY_CONTAINER ← HOST*_EMBY_CONTAINER # EMBY_URL ← HOST*_EMBY_URL # EMBY_API_KEY ← HOST*_EMBY_API_KEY # JELLYFIN_CONTAINER ← HOST*_JELLYFIN_CONTAINER # JELLYFIN_URL ← HOST*_JELLYFIN_URL # JELLYFIN_API_KEY ← HOST*_JELLYFIN_API_KEY # GITEA_API_TOKEN ← HOST*_GITEA_API_TOKEN # # Arr-specific vars set by detect_hosts() when needed by arr scripts: # LIDARR_URL / LIDARR_API_KEY / LIDARR_MUSIC_ROOT (MY_ID only — Lidarr is HOST1 only) # SONARR_URL / SONARR_API_KEY / SONARR_TV_ROOT # RADARR_URL / RADARR_API_KEY / RADARR_MOVIES_ROOT # SLSKD_URL / SLSKD_API_KEY / SLSKD_FAILED_IMPORTS_DIR (MY_ID only) # SABNZBD_URL / SABNZBD_API_KEY (MY_ID only) # QBIT_URL / QBIT_USERNAME / QBIT_PASSWORD (MY_ID only) detect_hosts() { local local_hostname local_hostname="$(hostname)" # ── Find MY_ID by matching hostname against all HOST* vars ──────────────── MY_ID="" local host_var host_val # Scan all HOST* vars that hold a hostname value # HOST1, HOST2, HOST3 etc. — all defined in host*.conf files for host_var in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do host_val="${!host_var:-}" [[ -z "$host_val" ]] && continue if [[ "${local_hostname,,}" == "${host_val,,}" ]]; then MY_ID="$host_var" break fi done # Unraid truncates the Server Name to 15 chars (NetBIOS limit). If the live hostname # is at that exact limit, a configured HOST* value may be a longer, untruncated version # (matching what Tailscale shows for this peer — resolve_tailscale_ip() keys off the same # value). Only accept the match if it's unambiguous — exactly one HOST* may qualify. if [[ -z "$MY_ID" && ${#local_hostname} -eq 15 ]]; then local candidate="" candidate_count=0 for host_var in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do host_val="${!host_var:-}" [[ -z "$host_val" ]] && continue if [[ ${#host_val} -gt 15 && "${host_val,,}" == "${local_hostname,,}"* ]]; then candidate="$host_var" (( candidate_count++ )) fi done [[ "$candidate_count" -eq 1 ]] && MY_ID="$candidate" fi if [[ -z "$MY_ID" ]]; then error "Unknown host: $local_hostname" error "Hostname must match a HOST* value in host*.conf" error "Available: $(for h in HOST1 HOST2 HOST3 HOST4; do [[ -n "${!h:-}" ]] && echo -n "${!h} "; done)" exit 1 fi # ── Find REMOTE_ID — first HOST* that isn't MY_ID ──────────────────────── REMOTE_ID="" for host_var in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do [[ "$host_var" == "$MY_ID" ]] && continue host_val="${!host_var:-}" [[ -z "$host_val" ]] && continue REMOTE_ID="$host_var" break done # ── Set host name strings ───────────────────────────────────────────────── LOCAL_SERVER_NAME="${!MY_ID}" [[ -n "$REMOTE_ID" ]] && REMOTE_SERVER_NAME="${!REMOTE_ID}" || REMOTE_SERVER_NAME="unknown" # ── Set SSH key ─────────────────────────────────────────────────────────── local ssh_key_var="${MY_ID}_SSH_KEY" SSH_KEY="${!ssh_key_var:-}" if [[ -z "$SSH_KEY" ]]; then error "Missing SSH key: ${ssh_key_var} not set in host*.conf" exit 1 fi # ── Set scalar aliases ──────────────────────────────────────────────────── MY_DISCORD_WEBHOOK_VAR="${MY_ID}_DISCORD_WEBHOOK" MY_DISCORD_WEBHOOK="${!MY_DISCORD_WEBHOOK_VAR:-}" OWNER_NAME_VAR="${MY_ID}_OWNER" OWNER_NAME="${!OWNER_NAME_VAR:-}" OWNER_EMAIL_VAR="${MY_ID}_OWNER_EMAIL" OWNER_EMAIL="${!OWNER_EMAIL_VAR:-}" EMBY_CONTAINER_VAR="${MY_ID}_EMBY_CONTAINER" EMBY_CONTAINER="${!EMBY_CONTAINER_VAR:-}" EMBY_URL_VAR="${MY_ID}_EMBY_URL" EMBY_URL="${!EMBY_URL_VAR:-}" EMBY_API_KEY_VAR="${MY_ID}_EMBY_API_KEY" EMBY_API_KEY="${!EMBY_API_KEY_VAR:-}" JELLYFIN_CONTAINER_VAR="${MY_ID}_JELLYFIN_CONTAINER" JELLYFIN_CONTAINER="${!JELLYFIN_CONTAINER_VAR:-}" JELLYFIN_URL_VAR="${MY_ID}_JELLYFIN_URL" JELLYFIN_URL="${!JELLYFIN_URL_VAR:-}" JELLYFIN_API_KEY_VAR="${MY_ID}_JELLYFIN_API_KEY" JELLYFIN_API_KEY="${!JELLYFIN_API_KEY_VAR:-}" GITEA_API_TOKEN_VAR="${MY_ID}_GITEA_API_TOKEN" GITEA_API_TOKEN="${!GITEA_API_TOKEN_VAR:-}" local _prov_var for _prov_var in \ PARTNERSHIP_PROVISION_EMBY_ADMIN \ PARTNERSHIP_EMBY_ADMIN_USER \ PARTNERSHIP_EMBY_ADMIN_PASS \ PARTNERSHIP_EMBY_PORT; do local _src="${MY_ID}_${_prov_var}" printf -v "$_prov_var" '%s' "${!_src:-}" done # ── System watchdog check toggles — per-host ───────────────────────────── # Aliased as unprefixed SYS_WATCHDOG_CHECK_* for use in system_watchdog.sh local _wd_checks=( SYS_WATCHDOG_NIC SYS_WATCHDOG_CHECK_DOCKER_DAEMON SYS_WATCHDOG_CHECK_ROOTFS SYS_WATCHDOG_CHECK_KERNEL_OOPS SYS_WATCHDOG_CHECK_FD SYS_WATCHDOG_CHECK_BOOT SYS_WATCHDOG_CHECK_OOM SYS_WATCHDOG_CHECK_RAM SYS_WATCHDOG_CHECK_LOG SYS_WATCHDOG_CHECK_ARC SYS_WATCHDOG_CHECK_CPU_TEMP SYS_WATCHDOG_CHECK_LOAD SYS_WATCHDOG_CHECK_ZOMBIES SYS_WATCHDOG_CHECK_CONTAINERS SYS_WATCHDOG_CHECK_TMP SYS_WATCHDOG_CHECK_MDSTAT SYS_WATCHDOG_CHECK_NETWORK SYS_WATCHDOG_CHECK_SSHD SYS_WATCHDOG_CHECK_RUNAWAY ) local _wd_var for _wd_var in "${_wd_checks[@]}"; do local _wd_src="${MY_ID}_${_wd_var}" # Only alias if the host-specific var is set — preserves master.conf defaults [[ -n "${!_wd_src+x}" ]] && eval "${_wd_var}=\"\${${_wd_src}}\"" done TRANSCODE_SSD_VAR="${MY_ID}_TRANSCODE_SSD" TRANSCODE_SSD="${!TRANSCODE_SSD_VAR:-}" RAMDISK_SIZE_VAR="${MY_ID}_RAMDISK_SIZE" RAMDISK_SIZE="${!RAMDISK_SIZE_VAR:-8G}" RAMDISK_WARN_GB_VAR="${MY_ID}_RAMDISK_WARN_GB" RAMDISK_WARN_GB="${!RAMDISK_WARN_GB_VAR:-6.8}" RAMDISK_LOW_GB_VAR="${MY_ID}_RAMDISK_LOW_GB" RAMDISK_LOW_GB="${!RAMDISK_LOW_GB_VAR:-5.5}" # Arr credentials — MY_ID only (remote arr accessed via its own host) LIDARR_URL_VAR="${MY_ID}_LIDARR_URL"; LIDARR_URL="${!LIDARR_URL_VAR:-}" LIDARR_API_KEY_VAR="${MY_ID}_LIDARR_API_KEY"; LIDARR_API_KEY="${!LIDARR_API_KEY_VAR:-}" LIDARR_MUSIC_ROOT_VAR="${MY_ID}_LIDARR_MUSIC_ROOT"; LIDARR_MUSIC_ROOT="${!LIDARR_MUSIC_ROOT_VAR:-}" LIDARR_RECOVERY_VAR="${MY_ID}_LIDARR_RECOVERY"; LIDARR_RECOVERY="${!LIDARR_RECOVERY_VAR:-false}" FANART_API_KEY_VAR="${MY_ID}_FANART_API_KEY"; FANART_API_KEY="${!FANART_API_KEY_VAR:-}" LASTFM_API_KEY_VAR="${MY_ID}_LASTFM_API_KEY"; LASTFM_API_KEY="${!LASTFM_API_KEY_VAR:-}" TMDB_API_KEY_VAR="${MY_ID}_TMDB_API_KEY"; TMDB_API_KEY="${!TMDB_API_KEY_VAR:-}" SONARR_URL_VAR="${MY_ID}_SONARR_URL"; SONARR_URL="${!SONARR_URL_VAR:-}" SONARR_API_KEY_VAR="${MY_ID}_SONARR_API_KEY"; SONARR_API_KEY="${!SONARR_API_KEY_VAR:-}" SONARR_TV_ROOT_VAR="${MY_ID}_SONARR_TV_ROOT"; SONARR_TV_ROOT="${!SONARR_TV_ROOT_VAR:-}" SONARR_GENERAL_ROOT_VAR="${MY_ID}_SONARR_GENERAL_ROOT"; SONARR_GENERAL_ROOT="${!SONARR_GENERAL_ROOT_VAR:-}" SONARR_KIDS_ROOT_VAR="${MY_ID}_SONARR_KIDS_ROOT"; SONARR_KIDS_ROOT="${!SONARR_KIDS_ROOT_VAR:-}" SONARR_ANIME_ROOT_VAR="${MY_ID}_SONARR_ANIME_ROOT"; SONARR_ANIME_ROOT="${!SONARR_ANIME_ROOT_VAR:-}" SONARR_RECOVERY_VAR="${MY_ID}_SONARR_RECOVERY"; SONARR_RECOVERY="${!SONARR_RECOVERY_VAR:-true}" RADARR_URL_VAR="${MY_ID}_RADARR_URL"; RADARR_URL="${!RADARR_URL_VAR:-}" RADARR_API_KEY_VAR="${MY_ID}_RADARR_API_KEY"; RADARR_API_KEY="${!RADARR_API_KEY_VAR:-}" RADARR_MOVIES_ROOT_VAR="${MY_ID}_RADARR_MOVIES_ROOT"; RADARR_MOVIES_ROOT="${!RADARR_MOVIES_ROOT_VAR:-}" RADARR_GENERAL_ROOT_VAR="${MY_ID}_RADARR_GENERAL_ROOT"; RADARR_GENERAL_ROOT="${!RADARR_GENERAL_ROOT_VAR:-}" RADARR_KIDS_ROOT_VAR="${MY_ID}_RADARR_KIDS_ROOT"; RADARR_KIDS_ROOT="${!RADARR_KIDS_ROOT_VAR:-}" RADARR_ANIME_ROOT_VAR="${MY_ID}_RADARR_ANIME_ROOT"; RADARR_ANIME_ROOT="${!RADARR_ANIME_ROOT_VAR:-}" RADARR_RECOVERY_VAR="${MY_ID}_RADARR_RECOVERY"; RADARR_RECOVERY="${!RADARR_RECOVERY_VAR:-true}" SLSKD_URL_VAR="${MY_ID}_SLSKD_URL"; SLSKD_URL="${!SLSKD_URL_VAR:-}" SLSKD_API_KEY_VAR="${MY_ID}_SLSKD_API_KEY"; SLSKD_API_KEY="${!SLSKD_API_KEY_VAR:-}" SLSKD_FAILED_IMPORTS_DIR_VAR="${MY_ID}_SLSKD_FAILED_IMPORTS_DIR" SLSKD_FAILED_IMPORTS_DIR="${!SLSKD_FAILED_IMPORTS_DIR_VAR:-}" SABNZBD_URL_VAR="${MY_ID}_SABNZBD_URL"; SABNZBD_URL="${!SABNZBD_URL_VAR:-}" SABNZBD_API_KEY_VAR="${MY_ID}_SABNZBD_API_KEY"; SABNZBD_API_KEY="${!SABNZBD_API_KEY_VAR:-}" QBIT_URL_VAR="${MY_ID}_QBIT_URL"; QBIT_URL="${!QBIT_URL_VAR:-}" QBIT_USERNAME_VAR="${MY_ID}_QBIT_USERNAME"; QBIT_USERNAME="${!QBIT_USERNAME_VAR:-}" QBIT_PASSWORD_VAR="${MY_ID}_QBIT_PASSWORD"; QBIT_PASSWORD="${!QBIT_PASSWORD_VAR:-}" # ── Set array aliases — indexed arrays ──────────────────────────────────── # Each eval copies the host-specific array into the unprefixed name scripts use _alias_array() { local alias_name="$1" local source_var="${MY_ID}_${alias_name}" eval "${alias_name}=(\"\${${source_var}[@]}\")" } _alias_array "DAILY_SYNC_SHARES" _alias_array "PERSONAL_SHARES" _alias_array "INTERMEDIATE_SYNC_SHARES" _alias_array "WEEKLY_SYNC_SHARES" _alias_array "CRITICAL_SYNC_SHARES" _alias_array "BACKUP_VERIFY_SHARES" _alias_array "DAILY_RESTART_CONTAINERS" _alias_array "WEEKLY_RESTART_CONTAINERS" _alias_array "WATCHDOG_REQUIRED_CONTAINERS" _alias_array "WATCHDOG_SCAN_IGNORE" _alias_array "NETWORK_CONNECT_CONTAINERS" _alias_array "NETWORK_CONNECT_NETWORKS" _alias_array "MEDIA_PERMISSION_SHARES" _alias_array "ANIME_CLEAN_FOLDERS" _alias_array "MEDIA_CLEAN_FOLDERS" _alias_array "CERT_MONITOR_DOMAINS" _alias_array "SMART_IGNORE_DRIVES" _alias_array "ZFS_REPORT_IGNORE_POOLS" _alias_array "TRANSCODE_SERVERS" _alias_array "DDNS_CONTAINERS" _alias_array "PARTNERSHIP_AUTH_WEBUIS" _alias_array "PARTNERSHIP_MIRROR_BACKUPS" _alias_array "PARTNERSHIP_OWN_CONTAINERS" _alias_array "PARTNERSHIP_AUTH_STACK" _alias_array "PARTNERSHIP_REPLACE_CONTAINERS" _alias_array "PARTNERSHIP_ARR_STACK" _alias_array "PARTNERSHIP_ARR_REPLACE_CONTAINERS" _alias_array "PARTNERSHIP_SERVICES_STACK" _alias_array "PARTNERSHIP_SERVICES_REPLACE_CONTAINERS" _alias_array "RW_PAUSE_CONTAINERS" _alias_array "RW_STOP_CONTAINERS" # ── Set array aliases — associative arrays ──────────────────────────────── # Associative arrays cannot be copied with eval — must be rebuilt key by key _alias_assoc() { local alias_name="$1" local source_name="${MY_ID}_${alias_name}" eval "declare -gA ${alias_name}" local -n _assoc_src="${source_name}" 2>/dev/null || return 0 local -n _assoc_dst="${alias_name}" local _assoc_key for _assoc_key in "${!_assoc_src[@]}"; do _assoc_dst["$_assoc_key"]="${_assoc_src[$_assoc_key]}" done } _alias_assoc "WATCHDOG_CONTAINERS" _alias_assoc "WATCHDOG_CONTAINER_URLS" _alias_assoc "WATCHDOG_CONTAINER_API_CHECKS" _alias_assoc "WATCHDOG_DEPENDENCIES" _alias_assoc "WATCHDOG_APPDATA_SIZES" # ── Remote storage path ─────────────────────────────────────────────────── # Resolved from REMOTE_ID's host*.conf so SSH commands use the correct path # even when the remote is a different platform. Falls back to the local # platform_storage_path() for unconfigured same-platform pairs. local _rsp_var="${REMOTE_ID}_STORAGE_PATH" REMOTE_STORAGE_PATH="${!_rsp_var:-$(platform_storage_path)}" # ── Output ──────────────────────────────────────────────────────────────── log "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" } # Derives owner/mirror roles from PARTNERSHIP_OWNER_HOST — used by every Partnership/*.sh # script. Requires detect_hosts() to have already run (needs MY_ID, SSH_KEY). # Sets: OWNER_ID MIRROR_ID OWNER MIRROR MIRROR_SSH_KEY OWNER_SSH_KEY AM_OWNER AM_MIRROR # Does NOT set any *_STATE_FILE vars — callers that need those set them after calling this, # since not every Partnership script needs the same set (transfer/onboard need fewer than # manager/offboard). partnership_resolve_roles() { OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}" MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" ) OWNER="${!OWNER_ID}" MIRROR="${!MIRROR_ID}" # SSH_KEY (set by detect_hosts) is this server's own private key. The remote accepts it # because this server's PUBLIC key was installed there via ssh_setup.sh. With sparse # checkout, each server only has its own host{N}.conf — the other server's key path is # never available here. Use SSH_KEY for all outbound SSH regardless of mode. MIRROR_SSH_KEY="$SSH_KEY" OWNER_SSH_KEY="$SSH_KEY" AM_OWNER=false AM_MIRROR=false [[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true [[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true } # Read a scalar/array var from a partner's own config via SSH — the partner sources its # own load_config.sh + detect_hosts() remotely so the value reflects ITS host*.conf, not # ours. Requires $SCRIPT_DIR to be set by the caller (one level above the repo root, same # convention every Partnership/*.sh script already uses for its own SCRIPT_DIR). # Usage: read_remote_conf_var "$mirror_ip" "VAR_NAME" # read_remote_conf_array "$mirror_ip" "ARRAY_NAME" # one element per line read_remote_conf_var() { local mirror_ip="$1" var_name="$2" timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ "source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null detect_hosts 2>/dev/null printf '%s' \"\${${var_name}:-}\"" 2>/dev/null } read_remote_conf_array() { local mirror_ip="$1" var_name="$2" timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ "source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null detect_hosts 2>/dev/null printf '%s\n' \"\${${var_name}[@]:-}\"" 2>/dev/null } # Probes a remote host's SCRIPTS_DIR (in case it differs from ours — e.g. one host runs # internal storage mode, the other flash), falling back to our own $SCRIPTS_DIR if the # probe fails or the remote isn't reachable. strict_host_key defaults to "yes" (matches # existing behavior everywhere except partnership_transfer.sh, which passes "no" since # it may be contacting a mirror for the first time during an ownership transfer). # Usage: resolve_remote_scripts_dir "$ip" ["$ssh_key"] ["no"|"yes"] resolve_remote_scripts_dir() { local ip="$1" ssh_key="${2:-$SSH_KEY}" strict_host_key="${3:-yes}" local probe_cmd result local -a opts=(-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes) [[ "$strict_host_key" == "no" ]] && opts+=(-o StrictHostKeyChecking=no) probe_cmd=$(platform_scripts_dir_probe_cmd) result=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" "${opts[@]}" root@"$ip" "$probe_cmd" \ 2>/dev/null | tr -d '[:space:]') echo "${result:-$SCRIPTS_DIR}" } # ============================================================================================== # ── MULTI-NODE DISCOVERY ────────────────────────────────────────────────────────────────────── # ============================================================================================== # Scans HOST1..HOST8 for every configured node other than this one. Requires detect_hosts() # to have already run (needs MY_ID). Sets the global REMOTE_NODES array — empty if none found. discover_remote_nodes() { REMOTE_NODES=() local _hv for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do [[ "$_hv" == "$MY_ID" ]] && continue [[ -z "${!_hv:-}" ]] && continue REMOTE_NODES+=("$_hv") done } # ============================================================================================== # ── REMOTE IP RESOLUTION ────────────────────────────────────────────────────────────────────── # ============================================================================================== # Resolves the Tailscale IPv4 address of the remote server. # Sets REMOTE_SERVER used by all subsequent SSH and rsync calls. # Exits if resolution fails — Tailscale may be down or peer offline. resolve_remote_ip() { log "Resolving remote IP for $REMOTE_SERVER_NAME..." local ts_name="${REMOTE_SERVER_NAME,,}" local attempts=3 for ((i=1; i<=attempts; i++)); do # Direct lookup — works when MagicDNS short-name resolution is active REMOTE_SERVER=$(tailscale ip -4 "$ts_name" 2>/dev/null) # Fallback — parse tailscale status for FQDN entries (e.g. hostname.tailXXXX.ts.net) if [[ -z "$REMOTE_SERVER" ]]; then REMOTE_SERVER=$(tailscale status 2>/dev/null | awk -v name="$ts_name" '$2 ~ "^" name { print $1; exit }') fi [[ -n "$REMOTE_SERVER" ]] && break [[ $i -lt $attempts ]] && { warn "Tailscale resolution attempt $i/$attempts failed — retrying in 5s..."; sleep 5; } done if [[ -z "$REMOTE_SERVER" ]]; then error "Failed to resolve Tailscale IP for $REMOTE_SERVER_NAME after $attempts attempts" error "Check: tailscale status | grep $ts_name" exit 1 fi info "$ICON_NET Remote IP: $REMOTE_SERVER" } # Resolve any hostname to a Tailscale IPv4 — tries direct lookup, falls back to status parse. # Usage: ip=$(resolve_tailscale_ip "hostname") — returns empty string on failure. resolve_tailscale_ip() { local hostname="${1,,}" local ip ip=$(tailscale ip -4 "$hostname" 2>/dev/null) if [[ -n "$ip" ]]; then echo "$ip" return fi # Fallback: unambiguous prefix match against tailscale status (either direction) — handles # Unraid's 15-char NetBIOS hostname truncation vs. a longer name recorded in master.conf. # Only accept the match when exactly one peer could qualify; never guess between multiple # candidates that happen to share a prefix (e.g. server1/server10). local matches count matches=$(tailscale status 2>/dev/null | awk -v name="$hostname" ' { split(tolower($2), parts, "."); host = parts[1]; if (index(host, name) == 1 || index(name, host) == 1) print $1 }') count=$(echo "$matches" | grep -c .) [[ "$count" -eq 1 ]] && echo "$matches" } # Strips the "unraid-" prefix (case-insensitive) and title-cases what remains. # Usage: derive_short_name "unRAID-Gmer4Lfe" → "Gmer4lfe" derive_short_name() { local hostname="$1" local short="${hostname,,}" [[ "$short" == unraid-* ]] && short="${short:7}" echo "${short^}" } # True if the given host*.conf basename belongs to this host (case-insensitive). # Usage: is_own_conf_file "$(basename "$conf")" is_own_conf_file() { [[ "${1,,}" == "${MY_ID,,}.conf" ]] } # Sets key=value in a flat state file (setup.db style) — updates in place if the # key exists, appends if not. No indentation handling — for master.conf use # partnership_manager.sh's update_master_conf() instead. # Usage: set_state_var "$state_file" "$key" "$value" set_state_var() { local file="$1" key="$2" value="$3" if grep -q "^${key}=" "$file" 2>/dev/null; then sed -i "s|^${key}=.*|${key}=${value}|" "$file" else echo "${key}=${value}" >> "$file" fi } # ============================================================================================== # ── CONNECTIVITY CHECKS ─────────────────────────────────────────────────────────────────────── # ============================================================================================== # Fatal connectivity check — used by rsync and other scripts that must abort if unreachable. # For fallback use ping_remote() which returns status without exiting. check_connectivity() { log "Checking connectivity to $REMOTE_SERVER..." if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then error "$ICON_PING Remote $REMOTE_SERVER ($REMOTE_SERVER_NAME) is unreachable" info "Hint: tailscale status | grep $REMOTE_SERVER_NAME" exit 1 fi info "$ICON_PING $REMOTE_SERVER_NAME is reachable" } # Non-fatal ping — used by fallback.sh which handles its own state machine. # Returns 0 if reachable, 1 if not — does NOT exit. ping_remote() { ping -c2 -W3 "$REMOTE_SERVER" &>/dev/null } # Non-fatal external connectivity check — used by fallback.sh. # Returns 0 if internet reachable, 1 if not — does NOT exit. ping_internet() { ping -c2 -W3 "${EXTERNAL_IP:-8.8.8.8}" &>/dev/null } # ============================================================================================== # ── RSYNC GATE CHECK ────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Two-tier rsync enable/disable system. # Tier 1: RSYNC_ENABLED — global gate, overrides everything # Tier 2: orchestrator-specific flag — fine-grained control per window # # Usage: # check_rsync_enabled "CRITICAL" ← checks RSYNC_ENABLED + CRITICAL_RSYNC_ENABLED # check_rsync_enabled "INTERMEDIATE" ← checks RSYNC_ENABLED + INTERMEDIATE_RSYNC_ENABLED # check_rsync_enabled "DAILY" ← checks RSYNC_ENABLED + DAILY_RSYNC_ENABLED # check_rsync_enabled "WEEKLY" ← checks RSYNC_ENABLED + WEEKLY_RSYNC_ENABLED # check_rsync_enabled "FALLBACK" ← checks RSYNC_ENABLED + FALLBACK_RSYNC_ENABLED # check_rsync_enabled ← checks RSYNC_ENABLED only (direct rsync.sh call) # # Returns: 0 = enabled, proceed | 1 = disabled, skip cleanly check_rsync_enabled() { local orchestrator="${1:-}" # Tier 1 — global gate if [[ "${RSYNC_ENABLED:-true}" == false ]]; then warn "RSYNC_ENABLED=false — rsync globally disabled, skipping all syncs" return 1 fi # Tier 2 — per-orchestrator if [[ -n "$orchestrator" ]]; then local var_name="${orchestrator}_RSYNC_ENABLED" local var_value="${!var_name:-true}" if [[ "$var_value" == false ]]; then info "${var_name}=false — rsync disabled for $orchestrator orchestrator" info "All other $orchestrator jobs will still run normally" return 1 fi fi return 0 } # ============================================================================================== # ── PLATFORM SERVICE STATE ──────────────────────────────────────────────────────────────────── # ============================================================================================== # Check whether Docker and VM Manager are enabled on this host. # Delegates to platform_is_service_enabled() from the platform adapter. is_docker_enabled() { platform_is_service_enabled docker } is_vm_manager_enabled() { platform_is_service_enabled libvirt } # ============================================================================================== # ── LOCAL HEALTH CHECKS ─────────────────────────────────────────────────────────────────────── # ============================================================================================== # ============================================================================================== # ── REMOTE HEALTH CHECKS ───────────────────────────────────────────────────────────────────── # ============================================================================================== # Verifies remote storage is mounted via SSH. # Non-fatal — returns status. Used before handback rsync. # Syncing to remote with no array fills rootfs rapidly. check_remote_array() { log "Checking remote array on $REMOTE_SERVER_NAME..." local _spath result _spath="$REMOTE_STORAGE_PATH" result=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ "mountpoint -q '$_spath' && echo yes || echo no" 2>/dev/null) if [[ "$result" != "yes" ]]; then error "$ICON_DISK Remote array not started on $REMOTE_SERVER_NAME" return 1 fi log "Remote array is healthy" return 0 } # Aborts if remote rootfs (/) usage is at or above ROOTFS_WARN threshold. # Fatal — exits the calling script. # When remote array is down rsync writes land on rootfs and fill it rapidly. check_remote_rootfs() { log "Checking remote rootfs usage..." local remote_usage remote_usage=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null) if [[ -z "$remote_usage" ]]; then error "Could not retrieve rootfs usage from $REMOTE_SERVER_NAME" exit 1 fi if [[ "$remote_usage" -ge "${ROOTFS_WARN:-75}" ]]; then error "$ICON_HEALTH Remote rootfs ${remote_usage}% — threshold ${ROOTFS_WARN:-75}%" exit 1 fi info "$ICON_HEALTH Remote rootfs: ${remote_usage}% (threshold: ${ROOTFS_WARN:-75}%)" } # Verifies target directory exists and is not empty on remote. # Fatal — exits the calling script. # Usage: check_remote_share "/mnt/user/Movies" check_remote_share() { local dir="$1" log "Checking remote share: $dir..." local share_exists share_exists=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null) if [[ "$share_exists" != "yes" ]]; then error "$ICON_HEALTH Remote share does not exist: $dir" exit 1 fi local share_empty share_empty=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null) if [[ "$share_empty" == "yes" ]]; then warn "$ICON_HEALTH Remote share exists but is empty: $dir — aborting to protect data" exit 1 fi info "$ICON_HEALTH Remote share verified: $dir" } # ============================================================================================== # ── DISK TEMPERATURE CHECKS ─────────────────────────────────────────────────────────────────── # ============================================================================================== # Returns 0 (true) if device is non-rotational (SSD/NVMe), 1 (false) if HDD. # Checks /sys/block//queue/rotational — 0=SSD, 1=HDD. is_ssd() { local drive="$1" local base rotational base=$(basename "$drive") rotational="/sys/block/$base/queue/rotational" [[ -f "$rotational" ]] && [[ "$(cat "$rotational" 2>/dev/null)" == "0" ]] && return 0 # NVMe is always SSD — some NVMe controllers don't expose rotational correctly [[ "$drive" == *nvme* ]] && return 0 return 1 } # Sets globals: UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX # Falls back to master.conf values (SMART_TEMP_WARN/CRIT) if platform returns defaults. get_unraid_temp_thresholds() { read -r UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX \ <<< "$(platform_get_temp_thresholds)" UNRAID_DISK_HOT="${UNRAID_DISK_HOT:-${SMART_TEMP_WARN:-45}}" UNRAID_DISK_MAX="${UNRAID_DISK_MAX:-${SMART_TEMP_CRIT:-55}}" UNRAID_SSD_HOT="${UNRAID_SSD_HOT:-60}" UNRAID_SSD_MAX="${UNRAID_SSD_MAX:-70}" } # Checks local disk temperatures before rsync. # Fetches a domain's TLS cert expiry via openssl s_client + x509 and computes days # remaining. Pure data — no logging, no thresholds — callers classify warn/crit # themselves against their own CERT_WARN_DAYS/CERT_CRIT_DAYS and log however they like. # # Sets _CERT_DAYS, _CERT_EXPIRY (YYYY-MM-DD) on success. Sets _CERT_EXPIRY_RAW to the # unparsed openssl date string only when parsing failed (empty when unreachable) so a # caller that wants a more specific error message can distinguish the two failure modes. # # Usage: check_cert_expiry "$domain" [port=443] [timeout=$CERT_TIMEOUT or 10] # Returns: 0 = fetched and parsed | 1 = unreachable or unparseable check_cert_expiry() { local domain="$1" port="${2:-443}" cert_timeout="${3:-${CERT_TIMEOUT:-10}}" _CERT_DAYS="" _CERT_EXPIRY="" _CERT_EXPIRY_RAW="" local expiry_str expiry_str=$(echo | timeout "$cert_timeout" openssl s_client \ -connect "${domain}:${port}" -servername "$domain" \ 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) [[ -z "$expiry_str" ]] && return 1 local expiry_epoch expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null) if [[ -z "$expiry_epoch" ]]; then _CERT_EXPIRY_RAW="$expiry_str" return 1 fi _CERT_DAYS=$(( (expiry_epoch - $(date +%s)) / 86400 )) _CERT_EXPIRY=$(date -d "$expiry_str" '+%Y-%m-%d' 2>/dev/null) return 0 } # Reads temps and rotational flag from /var/local/emhttp/disks.ini. # Uses unRAID's own thresholds from dynamix.cfg via get_unraid_temp_thresholds(). # # Returns: # 0 = all temps OK — proceed # 1 = warn threshold exceeded — skip this profile only # 2 = critical threshold exceeded — abort all remaining profiles # # Sets TEMP_CHECK_RESULT with human readable summary for logging. check_local_disk_temps() { get_unraid_temp_thresholds local disks_ini_content if ! disks_ini_content=$(platform_get_disk_states); then warn "disk state unavailable — skipping temp check" TEMP_CHECK_RESULT="temp check skipped (disk state unavailable)" return 0 fi local worst_result=0 local hot_drives=() local crit_drives=() local current_name="" local current_device="" local current_rotational="" local current_temp="" check_drive() { [[ -z "$current_name" ]] || [[ -z "$current_temp" ]] && return [[ "$current_temp" -eq 0 ]] && return # spun down — skip local warn_thresh crit_thresh if [[ "$current_rotational" == "0" ]]; then warn_thresh="$UNRAID_SSD_HOT" crit_thresh="$UNRAID_SSD_MAX" else warn_thresh="$UNRAID_DISK_HOT" crit_thresh="$UNRAID_DISK_MAX" fi if [[ "$current_temp" -ge "$crit_thresh" ]]; then crit_drives+=("${current_name}(${current_device}):${current_temp}°C≥${crit_thresh}°C") [[ $worst_result -lt 2 ]] && worst_result=2 elif [[ "$current_temp" -ge "$warn_thresh" ]]; then hot_drives+=("${current_name}(${current_device}):${current_temp}°C≥${warn_thresh}°C") [[ $worst_result -lt 1 ]] && worst_result=1 fi } while IFS= read -r ini_line; do if echo "$ini_line" | grep -qE '^\["(disk[0-9]+|parity[0-9]?|cache[0-9]?)"\]'; then check_drive current_name=$(echo "$ini_line" | grep -o '"[^"]*"' | head -1 | tr -d '"') current_device="" current_rotational="1" # default HDD current_temp="" elif echo "$ini_line" | grep -q '^device='; then current_device=$(echo "$ini_line" | cut -d= -f2 | tr -d '"') elif echo "$ini_line" | grep -q '^rotational='; then current_rotational=$(echo "$ini_line" | cut -d= -f2 | tr -d '"') elif echo "$ini_line" | grep -q '^temp='; then current_temp=$(echo "$ini_line" | cut -d= -f2 | tr -d '"') current_temp="${current_temp//[^0-9]/}" current_temp="${current_temp:-0}" fi done <<< "$disks_ini_content" check_drive # process last drive if [[ ${#crit_drives[@]} -gt 0 ]]; then TEMP_CHECK_RESULT="CRITICAL temps: ${crit_drives[*]}" error "Drive temp CRITICAL — aborting all remaining syncs: ${crit_drives[*]}" notify "Rsync aborted on $(hostname) — drive temp CRITICAL: ${crit_drives[*]}" "Rsync Temp Check" "warning" return 2 elif [[ ${#hot_drives[@]} -gt 0 ]]; then TEMP_CHECK_RESULT="HOT drives: ${hot_drives[*]}" warn "Drive temp WARNING — skipping this profile: ${hot_drives[*]}" notify "Rsync profile skipped on $(hostname) — drive temp WARNING: ${hot_drives[*]}" "Rsync Temp Check" "normal" return 1 else TEMP_CHECK_RESULT="all normal (HDD warn:${UNRAID_DISK_HOT}°C crit:${UNRAID_DISK_MAX}°C SSD warn:${UNRAID_SSD_HOT}°C crit:${UNRAID_SSD_MAX}°C)" return 0 fi } # ============================================================================================== # ── REMOTE DISK VERIFICATION ────────────────────────────────────────────────────────────────── # ============================================================================================== # Verifies all disks backing a share are healthy on the remote server. # Auto-detects filesystem type from disks.ini — handles XFS, ZFS, and cache pools. # No configuration required — discovers disk layout at runtime. # # Three-tier detection handles all share locations: # Tier 1: Array disk paths — /mnt/disk*/sharename (XFS or ZFS per-disk) # Tier 2: ZFS standalone pools — finds nested paths up to 2 levels deep # catches /mnt/cache/appdata-Fallback/Critical-Data correctly # Tier 3: shfs fallback — if share exists under /mnt/user, resolve backing pool # or verify shfs itself is mounted if pool can't be determined # # Fatal — exits the calling script if any disk is offline. # Usage: check_remote_disks "/mnt/user/Movies" check_remote_disks() { local dir="$1" local share_name share_name=$(basename "$dir") info "$ICON_DISK Checking disks backing $share_name on $REMOTE_SERVER_NAME..." # Read remote disk state for fsType mapping local disks_ini_content _disk_states_path _disk_states_path=$(platform_disk_states_path) disks_ini_content=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "cat '$_disk_states_path' 2>/dev/null" 2>/dev/null) if [[ -z "$disks_ini_content" ]]; then error "$ICON_DISK Cannot read disk state from $REMOTE_SERVER_NAME" exit 1 fi # Tier 1 — array disk paths (/mnt/disk*/sharename) local backing_disks backing_disks=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "ls -d /mnt/disk*/$share_name 2>/dev/null | awk -F/ '{print \$3}'" 2>/dev/null) # Tier 2 — ZFS standalone pools, including nested paths local zfs_pool_paths zfs_pool_paths=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "zpool list -H -o name 2>/dev/null | while read pool; do [[ -d \"/mnt/\${pool}/$share_name\" ]] && echo \"\$pool\" && continue find \"/mnt/\${pool}\" -maxdepth 2 -name '$share_name' -type d 2>/dev/null | \ grep -q . && echo \"\$pool\" done | sort -u" 2>/dev/null) # Tier 3 — storage root fallback local _spath="$REMOTE_STORAGE_PATH" if [[ -z "$backing_disks" ]] && [[ -z "$zfs_pool_paths" ]]; then local on_user on_user=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "find '$_spath' -maxdepth 3 -name '$share_name' -type d 2>/dev/null | head -1" \ 2>/dev/null) if [[ -n "$on_user" ]]; then local actual_path actual_path=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "readlink -f '$on_user' 2>/dev/null || echo '$on_user'" 2>/dev/null) local pool_name pool_name=$(echo "$actual_path" | awk -F/ '{print $3}') if [[ -n "$pool_name" ]] && [[ "$pool_name" != "user" ]]; then zfs_pool_paths="$pool_name" info "$ICON_DISK $share_name found at $actual_path on $REMOTE_SERVER_NAME" else # Can't determine pool — verify storage root itself is mounted local shfs_ok shfs_ok=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "mountpoint -q '$_spath' && echo yes || echo no" 2>/dev/null) if [[ "$shfs_ok" == "yes" ]]; then success "All disks backing $share_name are online ✅ (via storage root)" return 0 fi fi fi fi if [[ -z "$backing_disks" ]] && [[ -z "$zfs_pool_paths" ]]; then error "$ICON_DISK No disks found backing $share_name on $REMOTE_SERVER_NAME" exit 1 fi local all_ok=true # Check array disks (XFS or ZFS single-disk-in-array) if [[ -n "$backing_disks" ]]; then while IFS= read -r disk_name; do [[ -z "$disk_name" ]] && continue local fs_type fs_type=$(echo "$disks_ini_content" | awk -F= -v disk="$disk_name" ' /^\["'"'"'?/ { current=substr($0,3,length($0)-4) } current==disk && /^fsType=/ { print $2; exit } ' | tr -d '"') fs_type="${fs_type:-xfs}" if [[ "$fs_type" == "zfs" ]]; then local pool_health pool_health=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "zpool list -H -o health '$disk_name' 2>/dev/null" 2>/dev/null) if [[ "$pool_health" == "ONLINE" ]]; then info "$ICON_DISK $disk_name (ZFS) $ICON_RUNNING — $share_name ONLINE" else error "$ICON_DISK $disk_name (ZFS) $ICON_STOPPED — pool ${pool_health:-offline}" all_ok=false fi else local mounted mounted=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "mountpoint -q '/mnt/$disk_name' && echo yes || echo no" 2>/dev/null) if [[ "$mounted" == "yes" ]]; then info "$ICON_DISK $disk_name (XFS) $ICON_RUNNING — $share_name present" else error "$ICON_DISK $disk_name (XFS) $ICON_STOPPED — not mounted" all_ok=false fi fi done <<< "$backing_disks" fi # Check ZFS standalone pools (cache, gaming, media-servers etc.) if [[ -n "$zfs_pool_paths" ]]; then while IFS= read -r pool_name; do [[ -z "$pool_name" ]] && continue local pool_health pool_health=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "zpool list -H -o health '$pool_name' 2>/dev/null" 2>/dev/null) if [[ "$pool_health" == "ONLINE" ]]; then info "$ICON_DISK $pool_name (ZFS pool) $ICON_RUNNING — $share_name ONLINE" else error "$ICON_DISK $pool_name (ZFS pool) $ICON_STOPPED — pool ${pool_health:-offline}" all_ok=false fi done <<< "$zfs_pool_paths" fi if [[ "$all_ok" == false ]]; then error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME" exit 1 fi success "All disks backing $share_name are online ✅" } # ============================================================================================== # ── CONTAINER MANAGEMENT ────────────────────────────────────────────────────────────────────── # ============================================================================================== # Four functions — stop/start on both local and remote. # State tracking: only containers that WERE running get restarted. # Was stopped → stays stopped. Was running → gets restarted. ✅ # Tracks which remote containers were running before stop — used by start_containers() RUNNING_CONTAINERS=() # ── Docker command helpers (used by docker_daily_restart, docker_weekly_restart, docker_update) ── # Default timeouts — scripts can override before calling. DOCKER_TIMEOUT=${DOCKER_TIMEOUT:-30} # seconds before docker command is killed RESTART_VERIFY_WAIT=${RESTART_VERIFY_WAIT:-5} # seconds to wait before checking state post-restart # Run a docker command with a hard timeout. Returns 124 on timeout, else docker's exit code. docker_cmd() { timeout "$DOCKER_TIMEOUT" "$@" local exit_code=$? if [[ "$exit_code" -eq 124 ]]; then error "Docker command timed out after ${DOCKER_TIMEOUT}s: $*" return 1 fi return "$exit_code" } # Verify a container is still running after a restart. # Waits RESTART_VERIFY_WAIT seconds to let it settle, then checks State.Running. verify_running() { local container="$1" sleep "$RESTART_VERIFY_WAIT" local state state=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) if [[ "$state" != "true" ]]; then error "$container failed to stay running after restart — may have crashed" return 1 fi return 0 } # Retry a docker command up to RETRY_COUNT times, sleeping SLEEP seconds between attempts. retry_docker() { local attempt=1 while [[ "$attempt" -le "$RETRY_COUNT" ]]; do log "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*" if docker_cmd "$@"; then log "Succeeded on attempt $attempt" return 0 else warn "Attempt $attempt failed" (( attempt++ )) [[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP" fi done error "Command failed after $RETRY_COUNT attempts: $*" return 1 } # ── Emby API helper ─────────────────────────────────────────────────────────── # General Emby REST call. Uses EMBY_URL and EMBY_API_KEY (aliased by detect_hosts). # Usage: emby_api [timeout_seconds] # Prints response body on 200; returns 1 on HTTP error or curl failure. emby_api() { local endpoint="$1" max_time="${2:-30}" local response http_code body response=$(curl -sf --max-time "$max_time" \ -H "X-Emby-Token: $EMBY_API_KEY" \ -w "\n%{http_code}" \ "${EMBY_URL}/${endpoint}" 2>/dev/null) http_code=$(echo "$response" | tail -1) body=$(echo "$response" | head -n -1) [[ "$http_code" != "200" ]] && { error "Emby API HTTP $http_code: $endpoint"; return 1; } echo "$body" } # ── Media Server Detection ──────────────────────────────────────────────────── # Picks whichever media server is actually running right now, so callers don't have to # hardcode Emby or Jellyfin. Emby is preferred when both are up (it's the primary player on # every host; Jellyfin is mainly kept around for its working ffprobe binary — see # HOST*_FFPROBE_CONTAINER). Either container being merely *configured* in host*.conf doesn't # count — this checks Docker's actual running state, so a host with Jellyfin installed but # stopped correctly falls through to Emby (or to nothing, if neither is up). # # Requires detect_hosts() to have already run (needs EMBY_*/JELLYFIN_* aliases). # Sets: MEDIA_SERVER ("Emby"|"Jellyfin"|""), MEDIA_SERVER_URL, MEDIA_SERVER_API_KEY. # Returns 1 and leaves MEDIA_SERVER="" if neither configured container is running. # Usage: detect_media_server [docker_timeout_seconds] detect_media_server() { local docker_timeout="${1:-10}" MEDIA_SERVER="" MEDIA_SERVER_URL="" MEDIA_SERVER_API_KEY="" local emby_up=false jellyfin_up=false if [[ -n "${EMBY_CONTAINER:-}" ]]; then [[ "$(timeout "$docker_timeout" docker inspect -f '{{.State.Running}}' \ "$EMBY_CONTAINER" 2>/dev/null)" == "true" ]] && emby_up=true fi if [[ -n "${JELLYFIN_CONTAINER:-}" ]]; then [[ "$(timeout "$docker_timeout" docker inspect -f '{{.State.Running}}' \ "$JELLYFIN_CONTAINER" 2>/dev/null)" == "true" ]] && jellyfin_up=true fi if [[ "$emby_up" == true && "$jellyfin_up" == true ]]; then log "Both Emby and Jellyfin are running — using Emby (primary)" elif [[ "$emby_up" == true ]]; then log "Emby is running, Jellyfin is not — using Emby" elif [[ "$jellyfin_up" == true ]]; then log "Jellyfin is running, Emby is not — using Jellyfin" else warn "Neither Emby nor Jellyfin is running — no media server available" return 1 fi if [[ "$emby_up" == true ]]; then MEDIA_SERVER="Emby" MEDIA_SERVER_URL="$EMBY_URL" MEDIA_SERVER_API_KEY="$EMBY_API_KEY" else MEDIA_SERVER="Jellyfin" MEDIA_SERVER_URL="$JELLYFIN_URL" MEDIA_SERVER_API_KEY="$JELLYFIN_API_KEY" fi } # General REST call against whichever server detect_media_server() picked. Jellyfin accepts # the same X-Emby-Token header Emby uses (inherited from its Emby-fork lineage — confirmed # live 2026-07-19), so one call shape covers both backends. # Usage: detect_media_server && media_server_api [timeout_seconds] # Prints response body on 200; returns 1 on HTTP error, curl failure, or no server detected. media_server_api() { local endpoint="$1" max_time="${2:-30}" if [[ -z "${MEDIA_SERVER:-}" ]]; then error "media_server_api called with no MEDIA_SERVER set — call detect_media_server first" return 1 fi local response http_code body response=$(curl -sf --max-time "$max_time" \ -H "X-Emby-Token: $MEDIA_SERVER_API_KEY" \ -w "\n%{http_code}" \ "${MEDIA_SERVER_URL}/${endpoint}" 2>/dev/null) http_code=$(echo "$response" | tail -1) body=$(echo "$response" | head -n -1) [[ "$http_code" != "200" ]] && { error "$MEDIA_SERVER API HTTP $http_code: $endpoint"; return 1; } echo "$body" } # Stop containers on the REMOTE server via SSH. # Reads CRITICAL_CONTAINER_NAMES — set from profile arrays by rsync.sh. # Tracks running state in RUNNING_CONTAINERS for restart after sync. stop_containers() { if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \ [[ "${CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then log "No remote containers configured for this profile, skipping remote stop." return fi echo "━━━ $ICON_STOP $ICON_CONTAINERS Containers ━━━" info "Stopping remote containers..." RUNNING_CONTAINERS=() for c in "${CRITICAL_CONTAINER_NAMES[@]}"; do [[ -z "$c" ]] && continue local status status=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "docker inspect -f '{{.State.Running}}' $c 2>/dev/null" 2>/dev/null || echo "unknown") if [[ "$status" == "true" ]]; then echo "$ICON_STOP Stopping remote $c..." RUNNING_CONTAINERS+=("$c") if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $c" >/dev/null; then echo "$ICON_STOPPED $c stopped" else error "Failed to stop remote $c" fi elif [[ "$status" == "false" ]]; then echo "$ICON_NOT_RUNNING $c is not running — skipping" else log "$c not found on remote — skipping" fi done } # Tracks which local containers were running before stop — used by start_local_containers() LOCAL_RUNNING_CONTAINERS=() # Stop containers on the LOCAL server. # Reads LOCAL_CRITICAL_CONTAINER_NAMES — set from profile arrays by rsync.sh. # Flushes local databases cleanly before pushing data to remote. stop_local_containers() { if [[ ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \ [[ "${LOCAL_CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then log "No local containers configured for this profile, skipping local stop." return fi info "Stopping local containers..." LOCAL_RUNNING_CONTAINERS=() for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]}"; do [[ -z "$c" ]] && continue local status status=$(docker inspect -f '{{.State.Running}}' "$c" 2>/dev/null || echo "unknown") if [[ "$status" == "true" ]]; then echo "$ICON_STOP Stopping local $c..." LOCAL_RUNNING_CONTAINERS+=("$c") if docker stop "$c" >/dev/null; then echo "$ICON_STOPPED $c stopped" else error "Failed to stop local $c" fi elif [[ "$status" == "false" ]]; then echo "$ICON_NOT_RUNNING $c is not running — skipping" else log "$c not found locally — skipping" fi done } # Restart containers on the REMOTE server that were running before sync. # Reads RUNNING_CONTAINERS set by stop_containers(). # Respects DELAYED_CONTAINERS — waits CONTAINER_DELAY seconds before starting them. start_containers() { if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then log "No remote containers to restart." return fi info "Starting remote containers..." for c in "${RUNNING_CONTAINERS[@]}"; do [[ -z "$c" ]] && continue local needs_delay=false for d in "${DELAYED_CONTAINERS[@]}"; do [[ "$c" == "$d" ]] && needs_delay=true && break done if [[ "$needs_delay" == true ]]; then info "Waiting ${CONTAINER_DELAY}s before starting $c..." sleep "$CONTAINER_DELAY" fi echo "$ICON_START Starting remote $c..." if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $c" >/dev/null 2>&1; then echo "$ICON_STARTED $c started" else error "Failed to start remote $c — start manually if needed" fi done } # Restart containers on the LOCAL server that were running before sync. # Reads LOCAL_RUNNING_CONTAINERS set by stop_local_containers(). # Respects DELAYED_CONTAINERS and CONTAINER_DELAY same as remote. start_local_containers() { if [[ ${#LOCAL_RUNNING_CONTAINERS[@]} -eq 0 ]]; then log "No local containers to restart." return fi info "Starting local containers..." for c in "${LOCAL_RUNNING_CONTAINERS[@]}"; do [[ -z "$c" ]] && continue local needs_delay=false for d in "${DELAYED_CONTAINERS[@]}"; do [[ "$c" == "$d" ]] && needs_delay=true && break done if [[ "$needs_delay" == true ]]; then info "Waiting ${CONTAINER_DELAY}s before starting local $c..." sleep "$CONTAINER_DELAY" fi echo "$ICON_START Starting local $c..." if docker start "$c" >/dev/null 2>&1; then echo "$ICON_STARTED $c started" else error "Failed to start local $c — start manually if needed" fi done } # Builds a dependency-safe restart order — dependency containers (anything named as a # value in WATCHDOG_DEPENDENCIES) restart first, then everything else in original order. # Sets ORDERED_RESTART. Usage: build_restart_order CONTAINERS_ARRAY_NAME (bare name, e.g. # build_restart_order DAILY_RESTART_CONTAINERS — not "${DAILY_RESTART_CONTAINERS[@]}"). build_restart_order() { local -n _bro_containers="$1" ORDERED_RESTART=() local remaining=("${_bro_containers[@]}") local placed=() # First pass — add dependency containers that appear in our list for container in "${remaining[@]}"; do [[ -z "$container" ]] && continue local is_dependency=false for dependent in "${!WATCHDOG_DEPENDENCIES[@]}"; do if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then is_dependency=true break fi done if [[ "$is_dependency" == true ]]; then local already=false for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done if [[ "$already" == false ]]; then ORDERED_RESTART+=("$container") placed+=("$container") fi fi done # Second pass — add remaining containers (dependents and independents) for container in "${remaining[@]}"; do [[ -z "$container" ]] && continue local already=false for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done if [[ "$already" == false ]]; then ORDERED_RESTART+=("$container") placed+=("$container") fi done log "Restart order: ${ORDERED_RESTART[*]}" } # Waits CONTAINER_DELAY if this container depends on the last restarted one. # Usage: check_dependency_delay "$container" "$last_restarted" check_dependency_delay() { local container="$1" last="$2" [[ -z "$last" ]] && return local deps="${WATCHDOG_DEPENDENCIES[$container]:-}" if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then log "Waiting ${CONTAINER_DELAY}s — $container depends on $last..." sleep "$CONTAINER_DELAY" fi } # ============================================================================================== # ── RSYNC OPTIONS ───────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Loads rsync options for the current profile. # Falls back to DEFAULT_RSYNC_OPTS from master.conf if no profile match found. # Profile opts do NOT inherit from defaults — list all desired flags explicitly in master.conf. get_rsync_opts() { if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then read -r -a RSYNC_OPTS <<< "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]}" log "Using profile rsync opts for $PROFILE_NAME: ${RSYNC_OPTS[*]}" else RSYNC_OPTS=("${DEFAULT_RSYNC_OPTS[@]}") log "Using default rsync opts: ${RSYNC_OPTS[*]}" fi } # ============================================================================================== # ── SCRIPT LOCKING ──────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Prevents multiple instances of the same script running simultaneously. # All lock files live in /tmp/unraid_locks/ — auto-cleared on reboot. # # Modes: # strict — exit immediately if already running (default) # wait — wait LOCK_WAIT_TIMEOUT seconds then exit if still locked # continuous — for long-running scripts: skip gracefully if healthy instance running # # Features: # Stale lock detection — if lock PID is dead, clears and proceeds # PID reuse protection — lock stores PID:scriptname, validates both # Age warning — warns if lock older than LOCK_WARN_AGE (skipped for continuous) # EXIT trap — lock always released on exit, crash, or kill signal LOCK_DIR="/tmp/unraid_locks" # Ensure DATA_DIR and STATE_DIR exist — created here so every script that sources common.sh # can safely write to either without checking first. if [[ -n "${DATA_DIR:-}" ]] && [[ ! -d "$DATA_DIR" ]]; then mkdir -p "$DATA_DIR" 2>/dev/null || true fi if [[ -n "${STATE_DIR:-}" ]] && [[ ! -d "$STATE_DIR" ]]; then mkdir -p "$STATE_DIR" 2>/dev/null || true fi RSYNC_COUNT_FILE="$LOCK_DIR/rsync_active_count" RSYNC_MAX_CONCURRENT=3 LOCK_WARN_AGE=300 # seconds — warn if lock older than this (5min default) LOCK_WAIT_TIMEOUT=30 # seconds — how long "wait" mode waits before giving up # Registry of all lock files acquired by this process — released together on exit. # Prevents the single-trap-per-acquire problem: each new acquire_lock/acquire_rsync_lock # call used to overwrite the EXIT trap, orphaning the previous lock file. declare -ga _LOCK_FILES=() _release_all_locks() { local lf for lf in "${_LOCK_FILES[@]}"; do [[ -f "$lf" ]] && rm -f "$lf" done } _register_lock() { _LOCK_FILES+=("$1") trap '_release_all_locks' EXIT } # Internal — script name used as lock identifier _lock_name() { basename "${BASH_SOURCE[1]:-$0}" .sh } # Internal — lock file path for this script _lock_file() { echo "$LOCK_DIR/${1:-$(_lock_name)}.lock" } # Acquire exclusive lock for this script. # Usage: acquire_lock [strict|wait|continuous] acquire_lock() { local mode="${1:-strict}" local script_name script_name=$(basename "${BASH_SOURCE[1]:-$0}" .sh) local lockfile lockfile="$(_lock_file "$script_name")" mkdir -p "$LOCK_DIR" if [[ -f "$lockfile" ]]; then local lock_content existing_pid locked_name lock_content=$(cat "$lockfile" 2>/dev/null) existing_pid="${lock_content%%:*}" locked_name="${lock_content##*:}" # Stale lock — PID dead if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then warn "Stale lock detected for $script_name (PID $existing_pid gone) — clearing" rm -f "$lockfile" # PID reuse — PID alive but belongs to different process elif [[ "$locked_name" != "$script_name" ]]; then warn "Lock PID $existing_pid reused by different process ($locked_name ≠ $script_name) — clearing" rm -f "$lockfile" else # Lock is genuinely active — check age local lock_age lock_age=$(( $(date +%s) - $(stat -c %Y "$lockfile" 2>/dev/null || echo 0) )) if [[ "$lock_age" -gt "$LOCK_WARN_AGE" ]] && [[ "$mode" != "continuous" ]]; then warn "$script_name has been running for ${lock_age}s — may be stuck (PID $existing_pid)" fi if [[ "$mode" == "continuous" ]]; then log "$ICON_SKIP $script_name already running healthy (PID $existing_pid) — skipping" exit 0 elif [[ "$mode" == "wait" ]]; then info "Another instance of $script_name is running — waiting up to ${LOCK_WAIT_TIMEOUT}s" local waited=0 while [[ -f "$lockfile" ]] && [[ "$waited" -lt "$LOCK_WAIT_TIMEOUT" ]]; do sleep 1 ((waited++)) lock_content=$(cat "$lockfile" 2>/dev/null) existing_pid="${lock_content%%:*}" locked_name="${lock_content##*:}" if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then warn "Lock became stale while waiting — clearing" rm -f "$lockfile" break elif [[ "$locked_name" != "$script_name" ]]; then warn "Lock PID reused while waiting — clearing" rm -f "$lockfile" break fi done if [[ -f "$lockfile" ]]; then error "$script_name still locked after ${LOCK_WAIT_TIMEOUT}s — exiting" exit 1 fi else error "Another instance of $script_name is already running (PID $existing_pid) — exiting" case "$script_name" in fallback|transcode_management|daily_sync_maintenance|stability_watchdog) notify "$script_name lock collision on $(hostname) — concurrent instance detected" "$script_name" "warning" ;; esac exit 1 fi fi fi # Acquire lock — store PID:scriptname to prevent PID reuse false positives echo "$$:$script_name" > "$lockfile" _register_lock "$lockfile" log "$ICON_LOCK Lock acquired: $script_name (PID $$)" } # Acquire per-profile rsync lock + enforce global concurrent limit. # Prevents same profile running twice and limits total concurrent rsync instances. # Usage: acquire_rsync_lock "$PROFILE_NAME" acquire_rsync_lock() { local profile="$1" local profile_lock profile_lock="$(_lock_file "rsync_${profile}")" mkdir -p "$LOCK_DIR" # Per-profile lock — same profile cannot run twice if [[ -f "$profile_lock" ]]; then local lock_content existing_pid locked_name lock_content=$(cat "$profile_lock" 2>/dev/null) existing_pid="${lock_content%%:*}" locked_name="${lock_content##*:}" if [[ -n "$existing_pid" ]] && kill -0 "$existing_pid" 2>/dev/null && \ [[ "$locked_name" == "rsync_${profile}" ]]; then error "rsync profile '$profile' is already running (PID $existing_pid) — exiting" exit 1 else warn "Stale rsync lock for profile '$profile' — clearing" rm -f "$profile_lock" fi fi # Global concurrent limit — validate count against actual live locks local current_count=0 if [[ -f "$RSYNC_COUNT_FILE" ]]; then current_count=$(cat "$RSYNC_COUNT_FILE" 2>/dev/null || echo 0) local actual_count=0 for lf in "$LOCK_DIR"/rsync_*.lock; do [[ -f "$lf" ]] || continue local lpid lf_content lf_content=$(cat "$lf" 2>/dev/null) lpid="${lf_content%%:*}" kill -0 "$lpid" 2>/dev/null && ((actual_count++)) done if [[ "$actual_count" -ne "$current_count" ]]; then log "rsync count corrected: $current_count → $actual_count" current_count=$actual_count echo "$current_count" > "$RSYNC_COUNT_FILE" fi fi if [[ "$current_count" -ge "$RSYNC_MAX_CONCURRENT" ]]; then error "Maximum concurrent rsync limit ($RSYNC_MAX_CONCURRENT) reached — exiting" info "Active rsync locks: $(ls "$LOCK_DIR"/rsync_*.lock 2>/dev/null | \ xargs -I{} basename {} .lock | tr '\n' ' ')" exit 1 fi # Acquire profile lock and increment counter echo "$$:rsync_${profile}" > "$profile_lock" echo $(( current_count + 1 )) > "$RSYNC_COUNT_FILE" _register_lock "$profile_lock" log "$ICON_LOCK rsync lock acquired: profile '$profile' (PID $$, active: $(( current_count + 1 ))/$RSYNC_MAX_CONCURRENT)" } # ============================================================================================== # ── FLAT STATE-FILE KEY/VALUE HELPERS ───────────────────────────────────────────────────────── # ============================================================================================== # Generic get/set for flat "keyvalue" state files (one entry per line) — the pattern # every watchdog's strike-tracking and state-file logic was independently reimplementing. # grep -v + tempfile-swap on write, not sed -i in place — avoids sed treating a key containing # regex metacharacters (container names, etc.) as part of the substitution pattern. # # Usage: wd_state_get "$key" "$file" [sep=:] # wd_state_set "$key" "$value" "$file" [sep=:] # # Callers with a fixed state file and/or separator (e.g. stability_watchdog.sh's # get_strikes/get_state_val, resource_watchdog.sh's rm_state_get/rm_state_get_eq) should # keep their own thin same-named wrapper around these rather than changing call sites. wd_state_get() { local key="$1" file="$2" sep="${3:-:}" grep -E "^${key}${sep}" "$file" 2>/dev/null | cut -d"$sep" -f2- } wd_state_set() { local key="$1" value="$2" file="$3" sep="${4:-:}" grep -vE "^${key}${sep}" "$file" 2>/dev/null > "${file}.tmp" echo "${key}${sep}${value}" >> "${file}.tmp" mv "${file}.tmp" "$file" } # ============================================================================================== # ── ORCHESTRATOR CHILD EXECUTION ────────────────────────────────────────────────────────────── # ============================================================================================== # Shared child-script runner for Orchestrators/*.sh. Replaces the run_job()/run_watchdog() # copies that used to be hand-duplicated (with small inconsistencies) into each orchestrator — # one implementation now, one place to fix a bug or extend behavior (e.g. per-child logging). # # Usage — caller declares its own tracking arrays before the loop: # JOB_PASS=() # JOB_FAIL=() # for entry in "${SOME_SCRIPTS[@]}"; do # run_orch_child "$entry" # done # # $entry is "relative/path.sh [extra args...]" — the same format every master.conf # script-list array already uses. Resolved against $ECOSYSTEM_ROOT, which the caller # must set before calling this (the absolute repo root — "$(cd "$SCRIPT_DIR/.." && pwd)"). # # --dry-run / --log are threaded down automatically from $DRY_RUN / $ENABLE_LOGGING — # never $VERBOSE, which nothing in this codebase ever assigns. run_orch_child() { local entry="$1" local script_args script_path script_name label extra_args run_args read -r -a script_args <<< "$entry" script_path="$ECOSYSTEM_ROOT/${script_args[0]}" script_name=$(basename "${script_args[0]}") extra_args=("${script_args[@]:1}") label="$script_name" [[ -n "${extra_args[*]}" ]] && label="$script_name ${extra_args[*]}" if [[ ! -f "$script_path" ]]; then error "$script_name — not found at $script_path" JOB_FAIL+=("$label") return 1 fi [[ ! -x "$script_path" ]] && chmod +x "$script_path" run_args=("${extra_args[@]}") [[ "$DRY_RUN" == true ]] && run_args+=("--dry-run") [[ "$ENABLE_LOGGING" == true ]] && run_args+=("--log") local _start _ec _start=$(date +%s) log "Running: $label" if bash "$script_path" "${run_args[@]}"; then log "$script_name — done in $(format_duration $(( $(date +%s) - _start )))" JOB_PASS+=("$label") return 0 else _ec=$? error "$label — failed (exit $_ec, $(format_duration $(( $(date +%s) - _start ))))" JOB_FAIL+=("$label") return 1 fi } # ============================================================================================== # ── PATH TRANSLATION ────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Translates container-internal file paths to host paths for arr cleanup scripts. # Arr APIs return file paths as seen inside the container — scripts need host paths to scan. # # Uses ARR_PATH_MAP associative array — must be declared before calling. # Longest-match wins — prevents partial path collisions. # # Usage: # declare -A ARR_PATH_MAP=(["/ext-music"]="/mnt/user/Music") # translate_path "/ext-music/Artist/Album/track.flac" # Returns: /mnt/user/Music/Artist/Album/track.flac # # If no match found — returns path unchanged (safe fallback). translate_path() { local api_path="$1" local best_match="" local best_len=0 for container_path in "${!ARR_PATH_MAP[@]}"; do if [[ "$api_path" == "$container_path"* ]]; then if [[ "${#container_path}" -gt "$best_len" ]]; then best_match="$container_path" best_len="${#container_path}" fi fi done if [[ -n "$best_match" ]]; then echo "${ARR_PATH_MAP[$best_match]}${api_path#$best_match}" else echo "$api_path" # no match — return unchanged fi } # ============================================================================================== # ── CONTAINER STOP/RESTART FOR MAINTENANCE ──────────────────────────────────────────────────── # ============================================================================================== # The "stop a container for a maintenance window, guarantee it comes back" shape shared by # container_data_export.sh and emby_database_repair.sh. Callers still register their own EXIT # trap (they may need extra cleanup, e.g. removing a partial archive) — the trap should call # container_force_restart_if_needed() for the actual "still down? bring it back" part. # Usage: container_stop_for_maintenance "$container" was_running_var_name \ # ["pre-stop reason message"] [log|warn] [post_stop_sleep=0] # The reason message (if given) prints via the chosen level (default: log) right before the # stop attempt, only when the container was actually running — callers use this for their # own context ("stopping for clean export" vs "active sessions will be interrupted"). # Sets the named var true/false. Returns 1 (caller should exit) if the container doesn't # exist or fails to stop; 0 otherwise (including the "wasn't running" case — caller logs # its own context-specific message for that, since wording/visibility differs per caller). container_stop_for_maintenance() { local container="$1" local -n _csm_was_running="$2" local pre_stop_msg="${3:-}" pre_stop_level="${4:-log}" post_stop_sleep="${5:-0}" _csm_was_running=false local status status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) case "$status" in true) _csm_was_running=true if [[ -n "$pre_stop_msg" ]]; then if [[ "$pre_stop_level" == "warn" ]]; then warn "$pre_stop_msg"; else log "$pre_stop_msg"; fi fi if [[ "$DRY_RUN" == false ]]; then if timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1; then echo "$container stopped ✅" [[ "$post_stop_sleep" -gt 0 ]] && sleep "$post_stop_sleep" else error "Failed to stop $container — aborting" return 1 fi else warn "DRY RUN — would stop $container" fi ;; false) : ;; # not running — caller logs its own context-specific message "") error "$container not found — check container name" return 1 ;; *) warn "$container status: $status — proceeding with caution" ;; esac return 0 } # Force-restart from inside a script's own EXIT trap if the container is still down after # a crash mid-maintenance. No-op if it wasn't running before, in dry-run, or already up. # Usage: container_force_restart_if_needed "$container" "$was_running" container_force_restart_if_needed() { local container="$1" was_running="$2" [[ "$was_running" == true && "$DRY_RUN" == false ]] || return 0 local status status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) if [[ "$status" != "true" ]]; then warn "Restarting $container (cleanup)..." timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1 || \ error "Failed to restart $container — start it manually" fi } # Normal-path restart after maintenance completes — verifies it stayed up, notifies on failure. # Sets RESTART_OK true/false. # Usage: container_restart_after_maintenance "$container" "$was_running" [settle_sleep=3] [notify_label] container_restart_after_maintenance() { local container="$1" was_running="$2" settle_sleep="${3:-3}" notify_label="${4:-Container Maintenance}" RESTART_OK=false if [[ "$was_running" != true ]]; then echo "$container was not running — leaving stopped (state respected) ✅" RESTART_OK=true return 0 fi if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would restart $container" RESTART_OK=true return 0 fi log "Restarting $container..." if timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1; then sleep "$settle_sleep" local post_status post_status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) if [[ "$post_status" == "true" ]]; then echo "$container restarted and running ✅" RESTART_OK=true else error "$container started but crashed — check container logs" notify "$container failed to stay running after maintenance on $(hostname)" \ "$notify_label" "warning" fi else error "Failed to restart $container — start it manually" notify "$container failed to restart after maintenance on $(hostname)" \ "$notify_label" "warning" fi } # ============================================================================================== # ── ARR CLEANUP SAFETY GATES ────────────────────────────────────────────────────────────────── # ============================================================================================== # Shared by lidarr/radarr/sonarr_cleanup.sh — these scripts DELETE files, so every function # here is a direct byte-for-byte port of what was independently duplicated three times, not # a rewrite. Callers should behave identically to before this consolidation. # Safety Layer 1 — container running, not starting/unhealthy. Exits 1 (with notify) on failure. # Usage: check_container_health "$LIDARR_CONTAINER" "$ARR_DOCKER_TIMEOUT" "Lidarr Cleanup" check_container_health() { local container="$1" docker_timeout="$2" notify_label="$3" local running running=$(timeout "$docker_timeout" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) if [[ "$running" != "true" ]]; then error "$container is not running — aborting" notify "$notify_label aborted on $(hostname) — container not running" "$notify_label" "warning" exit 1 fi local health health=$(timeout "$docker_timeout" docker inspect -f '{{.State.Health.Status}}' "$container" 2>/dev/null) case "$health" in healthy) info "$container is healthy" ;; "") info "$container has no health check — proceeding" ;; starting) error "$container is still starting — aborting" notify "$notify_label aborted on $(hostname) — container still starting" "$notify_label" "warning" exit 1 ;; unhealthy) error "$container is unhealthy — aborting" notify "$notify_label aborted on $(hostname) — container unhealthy" "$notify_label" "warning" exit 1 ;; *) warn "$container health: $health — proceeding with caution" ;; esac info "Safety layer 1 passed — container healthy" } # Safety Layer 6 — abort if tracked count dropped below min_pct of the last known count # (protects against a misconfigured path / partial API response silently wiping the library). # Writes the new baseline only if the check passes. Exits 1 (with notify) on failure. # Usage: check_tracked_count_floor "$TRACKED_COUNT" "$LIDARR_TRACKED_COUNT_FILE" "$LIDARR_MIN_TRACKED_PCT" "Lidarr Cleanup" check_tracked_count_floor() { local tracked_count="$1" baseline_file="$2" min_pct="$3" notify_label="$4" if [[ -f "$baseline_file" ]]; then local last_count pct last_count=$(cat "$baseline_file" 2>/dev/null || echo 0) if [[ "$last_count" -gt 0 ]]; then pct=$(awk "BEGIN {printf \"%d\", ($tracked_count / $last_count) * 100}") if [[ "$pct" -lt "$min_pct" ]]; then error "Tracked count dropped to ${pct}% of last run ($tracked_count vs $last_count)" error "Suggests API issue — aborting to prevent mass deletion" error "If expected (large library removal) delete: $baseline_file" notify "$notify_label aborted on $(hostname) — tracked count dropped to ${pct}%" \ "$notify_label" "warning" exit 1 fi info "Tracked count: ${pct}% of last run ($tracked_count vs $last_count) ✅" fi else info "No previous count on record — first run, saving baseline" fi echo "$tracked_count" > "$baseline_file" } # Filters --i-know-what-im-doing / --skip-age-check out of "$@" before parse_args sees them # (both are cleanup-script-specific, not part of the shared arg parser). # Sets I_KNOW, SKIP_AGE_CHECK, FILTERED_ARGS — call parse_args "${FILTERED_ARGS[@]}" after. parse_destructive_flags() { I_KNOW=false SKIP_AGE_CHECK=false FILTERED_ARGS=() local arg for arg in "$@"; do case "$arg" in --i-know-what-im-doing) I_KNOW=true ;; --skip-age-check) SKIP_AGE_CHECK=true ;; *) FILTERED_ARGS+=("$arg") ;; esac done } # Prints the nuclear-mode banner and sleeps 10s (Ctrl+C window) when both destructive flags # are set and this isn't a dry run. Call after parse_args. No-op otherwise. nuclear_mode_warning() { if [[ "$I_KNOW" == true ]] && [[ "$SKIP_AGE_CHECK" == true ]] && [[ "$DRY_RUN" != true ]]; then echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "⚠️ WARNING — NUCLEAR MODE ACTIVE" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo " Flags: --i-know-what-im-doing --skip-age-check" echo " Age check: BYPASSED — deletes on first pass" echo " Size threshold: BYPASSED — no GB limit" echo " Data recovery: NOT POSSIBLE after deletion" echo "" echo " Review --dry-run output before proceeding." echo " You have 10 seconds to cancel (Ctrl+C)..." echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" sleep 10 echo " Proceeding..." echo "" fi } # Safety Layer 7 — abort if total deletion size exceeds max_gb and --i-know-what-im-doing # wasn't passed. Sets TOTAL_HUMAN. Exits 1 (with notify) on failure; warns and continues # if I_KNOW is true (set by parse_destructive_flags). # Usage: check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$LIDARR_MAX_DELETE_GB" "Lidarr Cleanup" check_delete_size_threshold() { local total_bytes="$1" max_gb="$2" notify_label="$3" local max_bytes max_bytes=$(awk "BEGIN {printf \"%d\", $max_gb * 1073741824}") if [[ "$total_bytes" -gt "$max_bytes" ]]; then TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $total_bytes / 1073741824}") if [[ "$I_KNOW" != true ]]; then echo "" error "Deletion would exceed ${max_gb}GB — $TOTAL_HUMAN would be deleted" error "Review ORPHAN lines above carefully before proceeding" error "Rerun with: --i-know-what-im-doing" error "To also bypass age check: add --skip-age-check" notify "$notify_label halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \ "$notify_label" "warning" exit 1 else warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing" fi fi } # True if filepath's extension (case-insensitive) matches one of the given extensions. # Usage: has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}" has_extension() { local filepath="$1"; shift local ext="${filepath##*.}" ext="${ext,,}" local valid_ext for valid_ext in "$@"; do [[ "$ext" == "$valid_ext" ]] && return 0 done return 1 } # True if filepath's basename matches one of the given glob patterns. # Usage: matches_pattern_list "$filepath" "${LIDARR_PROTECTED_PATTERNS[@]}" # # Parameter expansion instead of external basename (2026-07-17) — called once per # non-tracked file in lidarr_cleanup.sh/sonarr_cleanup.sh/radarr_cleanup.sh's classification # loop, which is most files in a media library (every protected sidecar: nfo/jpg/srt/etc). # Measured: this was the actual dominant cost left in those scripts even after fixing the # per-file stat fork and dirname elsewhere — 47.3s vs 1.65s for 10,000 calls (~28.6x), # confirmed identical results across a spot check first. matches_pattern_list() { local filepath="$1"; shift local filename="${filepath##*/}" local pattern for pattern in "$@"; do # shellcheck disable=SC2254 case "$filename" in $pattern) return 0 ;; esac done return 1 } # Builds the global ARR_PATH_MAP associative array (container path → host path) from # ${MY_ID}__PATH_MAP, for translate_path() to use. Usage: build_arr_path_map "LIDARR" build_arr_path_map() { local arr_type="$1" declare -gA ARR_PATH_MAP=() local _bapm_var="${MY_ID}_${arr_type}_PATH_MAP" eval "for key in \"\${!${_bapm_var}[@]}\"; do ARR_PATH_MAP[\"\$key\"]=\"\${${_bapm_var}[\$key]}\" done" } # Generic arr API GET wrapper with HTTP status check. # Usage: arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "movie" "Radarr" arr_api() { local base_url="$1" api_key="$2" api_version="$3" endpoint="$4" label="${5:-Arr}" local response http_code body response=$(curl -sf \ --max-time 30 \ -H "X-Api-Key: $api_key" \ -w "\n%{http_code}" \ "${base_url}/api/${api_version}/${endpoint}" 2>/dev/null) http_code=$(echo "$response" | tail -1) body=$(echo "$response" | head -n -1) if [[ "$http_code" != "200" ]]; then error "$label API HTTP $http_code for: $endpoint" return 1 fi echo "$body" } # Triggers an arr "command" endpoint with the given JSON payload, then polls every 10s until # completed/failed/timeout, logging progress every 60s. Best-effort pre-flight — never treats # an unreachable/timed-out scan as fatal, matching original per-script behavior. # # On a genuine "completed" observation, records the actual elapsed duration keyed by the # command's own name via arr_record_rescan_duration() (needs arr_type to know which arr's # duration DB to write to) — this is the one place in the codebase that reliably watches a # command from trigger to completion, so it's the natural spot to build up real historical # duration data for the wait-calibration logic in arr_get_tracked_data(). A timeout doesn't # record anything — we only know a lower bound, not the true duration, and recording that # would corrupt future wait calculations downward. arr_type is optional — omit it (or pass # empty) to skip duration recording entirely, e.g. for one-off commands that aren't one of # the three tracked arrs. # # Usage: trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" \ # '{"name": "DownloadedAlbumsScan"}' "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}" "lidarr" trigger_and_await_command() { local base_url="$1" api_key="$2" api_version="$3" payload="$4" poll_timeout="${5:-600}" arr_type="${6:-}" local cmd_name cmd_name=$(echo "$payload" | jq -r '.name // empty' 2>/dev/null) local scan_response scan_cmd_id scan_response=$(curl -sf --max-time 30 -X POST \ -H "X-Api-Key: $api_key" \ -H "Content-Type: application/json" \ -d "$payload" \ "${base_url}/api/${api_version}/command" 2>/dev/null) scan_cmd_id=$(echo "$scan_response" | jq -r '.id // empty' 2>/dev/null) if [[ -z "$scan_cmd_id" ]]; then warn "Could not trigger import scan — proceeding without pre-flight" return 0 fi info "Import scan queued (command ID: $scan_cmd_id) — waiting for completion..." local start_epoch=$(date +%s) local polled=0 scan_status while [[ "$polled" -lt "$poll_timeout" ]]; do scan_status=$(curl -sf --max-time 10 \ -H "X-Api-Key: $api_key" \ "${base_url}/api/${api_version}/command/${scan_cmd_id}" 2>/dev/null | \ jq -r '.status // empty' 2>/dev/null) case "$scan_status" in completed) info "Import scan complete ✅" [[ -n "$cmd_name" && -n "$arr_type" ]] && \ arr_record_rescan_duration "$arr_type" "$cmd_name" "$(( $(date +%s) - start_epoch ))" return 0 ;; failed) warn "Import scan reported failed — proceeding anyway" return 0 ;; esac sleep 10 (( polled += 10 )) [[ $(( polled % 60 )) -eq 0 ]] && log " Still scanning... (${polled}s elapsed)" done warn "Import scan timed out after ${poll_timeout}s — proceeding anyway" return 0 } # ============================================================================================== # ── ARR TRACKED-DATA CACHE (Lidarr, Sonarr, Radarr) ────────────────────────────────────────── # ============================================================================================== # Shared cache for each arr's tracked-library data (Lidarr artists, Sonarr series, Radarr # movies). Every script across all three arrs that needs "what does this arr think it has # tracked" goes through this — 15+ consumers as of 2026-07-17 (all three cleanup scripts, the # missing-art/release-fixer/duplicate-cleanup scripts, all three playback-aware discovery # scripts, the TVDB/TMDb-removed checks, arr_sync.sh's local side, and the emby_to_*_sync # bootstrap tools). Hitting the live API fresh every time is wasteful, and during an active # rescan the live number is actively misleading — tracked counts dip and recover as files are # detached/re-verified one by one (confirmed 2026-07-16 on Lidarr: a whole-library # RescanFolders made trackFileCount read 22% of normal mid-scan, which is exactly the # false-alarm lidarr_cleanup.sh's count-drop guard is meant to catch, but a genuine rescan # isn't the "something's actually wrong" case that guard exists for — the same risk applies to # Sonarr's RescanSeries and Radarr's RescanMovie). # # Built Lidarr-only first (2026-07-16), generalized the same day to cover all three arrs — # identical mechanism, keyed by arr_type ("lidarr"/"sonarr"/"radarr") so each arr's cache and # duration history stay separate. # # Storage (2026-07-17): primary copy lives on tmpfs (ARR_CACHE_DIR) — reads/writes never touch # the array disk, and losing it on reboot costs nothing since a full rebuild for all three arrs # measures ~12s live. A persistent backup on $DATA_DIR is kept in sync by every write and gets # transparently restored into tmpfs by arr_cache_age_seconds() the moment it notices tmpfs is # missing — so a cache that was fresh before a reboot reads as fresh after too. # # Write-through: any script that already does a live library-list fetch for its own purposes # writes the result here as a side effect via arr_cache_write() — no dedicated polling timer # needed. arr_cache_write() itself refuses to write while a rescan-type command is active for # that arr (2026-07-17) — protects every caller uniformly, not just arr_get_tracked_data(). # Arrs_Stack/arr_cache_prefill.sh keeps the cache warm two ways: once at array start (10min # wait ceiling, closes the cold-boot gap) and again every 30min via CRITICAL_MAINTENANCE_SCRIPTS # (1min wait ceiling — a live fetch takes seconds, not the boot-time wait). # # Consumers should call arr_get_tracked_data() — never read a cache file directly. It handles # the fresh/stale-no-rescan/stale-rescan-active branching so no script reimplements it. # # This is the top-level LIBRARY cache only. A separate, shorter-lived cache exists for the # much more expensive per-item track/episode data — see "PER-ITEM TRACKED-FILE CACHE" below. # ============================================================================================== # Rescan-type command names per arr — operations long/heavy enough that overlapping with one # should trigger a wait rather than a live fetch. Verified live against each arr's API # 2026-07-16 (not assumed from memory — RescanSeries/RescanMovie confirmed to scan the whole # library when given no id, matching Lidarr's RescanFolders). declare -gA ARR_RESCAN_COMMANDS=( [lidarr]="RescanFolders DownloadedAlbumsScan RefreshArtist" [sonarr]="RescanSeries DownloadedEpisodesScan RefreshSeries" [radarr]="RescanMovie DownloadedMoviesScan RefreshMovie" ) # jq expression computing each arr's "total tracked" number from its library-list response — # schemas differ: Lidarr/Sonarr have a per-item file count to sum, Radarr is a binary hasFile # per movie (one file at most), so the aggregate has to be a count of true values instead. declare -gA ARR_TRACKED_COUNT_EXPR=( [lidarr]='[.[].statistics.trackFileCount] | add' [sonarr]='[.[].statistics.episodeFileCount] | add' [radarr]='[.[] | select(.hasFile==true)] | length' ) # Library-list endpoint name per arr, for arr_api(). declare -gA ARR_LIBRARY_ENDPOINT=( [lidarr]="artist" [sonarr]="series" [radarr]="movie" ) # API version per arr — Lidarr is still v1, Sonarr/Radarr are v3. declare -gA ARR_API_VERSION=( [lidarr]="v1" [sonarr]="v3" [radarr]="v3" ) # tmpfs — primary read/write location. Cleared every reboot; that's fine, a full rebuild for # all three arrs measures ~12s live, and arr_cache_age_seconds() transparently restores from # the persistent backup below the moment it notices this is missing. arr_cache_file() { echo "${ARR_CACHE_DIR}/${1}_tracked_cache.json"; } # Persistent backup — survives reboot. Kept in sync by arr_cache_write() dual-writing here on # every write, so it's never more than one write-cycle stale (2026-07-17). Exists purely so a # reboot doesn't leave the tmpfs cache genuinely empty until the next live fetch completes — # not a source of truth in its own right, just what tmpfs gets restored from when missing. arr_cache_backup_file() { echo "${DATA_DIR}/${1}_tracked_cache.json"; } arr_rescan_duration_db() { echo "${DATA_DIR}/${1}_rescan_duration.db"; } # Writes the current library-list JSON + computed total tracked count to arr_type's cache. # Radarr's movie list alone runs ~16MB — passing that through jq's --argjson as a literal # command-line argument blows past the OS's ARG_MAX ("Argument list too long"), the exact # same class of bug the queue-pagination fix (2026-07-16, arrs_failed_stalled_recovery.sh) # hit before. Fixed the same way: write the payload to a temp file and use --slurpfile, # which reads from disk instead of argv. # # Refuses to write while a rescan-type command is active for arr_type (2026-07-17). Every # direct caller here fetches the library list for its own purposes and writes it through as # a side effect — none of them go through arr_get_tracked_data()'s fresh/stale/active-rescan # branching, so none of them knew to check scan state first. Confirmed live: a caller wrote # Lidarr's cache mid-RescanFolders at 22% of the real total, and that partial number then # looked exactly like real data loss to every consumer (check_tracked_count_floor et al.) # until the scan finished. The guard lives here instead of in each caller so it applies # uniformly — last-known-good stays in place until something writes through the real # post-scan number (arr_full_rescan.sh does this itself on completion; for a rescan someone # else triggered, see arr_rescan_monitor.sh in Tools/). # Args: arr_type, items_json (the full library-list response body) arr_cache_write() { local arr_type="$1" items_json="$2" local expr="${ARR_TRACKED_COUNT_EXPR[$arr_type]:-}" [[ -z "$expr" ]] && return 1 local url_var="${arr_type^^}_URL" key_var="${arr_type^^}_API_KEY" local url="${!url_var:-}" api_key="${!key_var:-}" api_version="${ARR_API_VERSION[$arr_type]:-}" if [[ -n "$url" && -n "$api_key" && -n "$api_version" ]]; then local active_cmd active_cmd=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version") [[ -n "$active_cmd" ]] && return 1 fi local total total=$(echo "$items_json" | jq "$expr" 2>/dev/null) [[ -z "$total" || "$total" == "null" ]] && return 1 local cache_file backup_file items_tmp cache_file=$(arr_cache_file "$arr_type") backup_file=$(arr_cache_backup_file "$arr_type") items_tmp=$(mktemp) echo "$items_json" > "$items_tmp" mkdir -p "$(dirname "$cache_file")" "$(dirname "$backup_file")" 2>/dev/null jq -c -n --slurpfile items "$items_tmp" --argjson total "$total" --argjson ts "$(date +%s)" \ '{ts:$ts, totalTracked:$total, items:$items[0]}' > "${cache_file}.tmp" 2>/dev/null \ && mv "${cache_file}.tmp" "$cache_file" \ && cp "$cache_file" "$backup_file" 2>/dev/null rm -f "$items_tmp" } # Echoes the cached library-list JSON array for arr_type if a cache file exists, regardless # of age — staleness is arr_get_tracked_data()'s decision, not this raw reader's. arr_cache_read_raw() { local arr_type="$1" local cache_file cache_file=$(arr_cache_file "$arr_type") [[ -f "$cache_file" ]] || return 1 jq -c '.items // empty' "$cache_file" 2>/dev/null } # Echoes arr_type's cache age in seconds, or a very large number if no cache and no backup # exist (so age comparisons naturally treat that as "very stale"). Transparently restores the # tmpfs cache from its persistent backup first if tmpfs is missing (e.g. right after a reboot) # — a cache that was fresh before reboot reads as fresh here too, not as if it never existed. arr_cache_age_seconds() { local arr_type="$1" local cache_file cache_file=$(arr_cache_file "$arr_type") if [[ ! -f "$cache_file" ]]; then local backup_file backup_file=$(arr_cache_backup_file "$arr_type") if [[ -f "$backup_file" ]]; then mkdir -p "$(dirname "$cache_file")" 2>/dev/null cp "$backup_file" "$cache_file" 2>/dev/null fi fi [[ -f "$cache_file" ]] || { echo 999999999; return; } local ts ts=$(jq -r '.ts // 0' "$cache_file" 2>/dev/null) echo $(( $(date +%s) - ${ts:-0} )) } # ============================================================================================== # ── PER-ITEM TRACKED-FILE CACHE (Lidarr trackFile, Sonarr episodefile) ─────────────────────── # ============================================================================================== # Different in kind from the tracked-LIBRARY cache above. That one caches the top-level # artist/series/movie list (one call). This caches the much more expensive per-item walk — # one live API call per artist/series — that lidarr_cleanup.sh/sonarr_cleanup.sh already have # to do for their own cleanup decisions regardless. Write-through only, from a walk that's # already happening: no independent live-fetch-and-refresh path here, since the field set # each consumer needs differs (lidarr_cleanup.sh only needs .path; lidarr_missing_art.sh also # needs .albumId) and there's no single generic "fetch everything" call worth centralizing — # each script keeps its own live per-item fallback, this is purely a fast path in front of it. # # Freshness is short (default 4h, not the library cache's 1 day) because this is meant for a # script running shortly after in the same maintenance window (e.g. lidarr_missing_art.sh # right after lidarr_cleanup.sh), not held across a whole day. tmpfs only, no persistent # backup — unlike the library cache, nothing unique lives only here (every consumer already # has its own live fallback), and it's short-lived by design, so surviving a reboot doesn't # matter the way it did for the always-wanted library cache. (2026-07-17) # # Future scripts needing this data: call arr_get_cached_items() first, fall back to your own # live per-item fetch on a miss, and write through via arr_item_cache_write() if you're the # one doing that fetch anyway — same pattern as lidarr_cleanup.sh/sonarr_cleanup.sh below. # ============================================================================================== arr_item_cache_file() { echo "${ARR_CACHE_DIR}/${1}_items_cache.json"; } # Args: arr_type, items_json (array of raw per-item track/episode-file objects) arr_item_cache_write() { local arr_type="$1" items_json="$2" local url_var="${arr_type^^}_URL" key_var="${arr_type^^}_API_KEY" local url="${!url_var:-}" api_key="${!key_var:-}" api_version="${ARR_API_VERSION[$arr_type]:-}" if [[ -n "$url" && -n "$api_key" && -n "$api_version" ]]; then local active_cmd active_cmd=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version") [[ -n "$active_cmd" ]] && return 1 fi local cache_file items_tmp cache_file=$(arr_item_cache_file "$arr_type") items_tmp=$(mktemp) echo "$items_json" > "$items_tmp" mkdir -p "$(dirname "$cache_file")" 2>/dev/null jq -c -n --slurpfile items "$items_tmp" --argjson ts "$(date +%s)" \ '{ts:$ts, items:$items[0]}' > "${cache_file}.tmp" 2>/dev/null \ && mv "${cache_file}.tmp" "$cache_file" rm -f "$items_tmp" } # Echoes the cached per-item array if present and fresher than max_age_seconds, empty/1 on a # miss or stale cache — caller falls back to its own live per-item fetch exactly as it already # does today. # Args: arr_type, max_age_seconds (default 14400 = 4h) arr_get_cached_items() { local arr_type="$1" max_age="${2:-14400}" local cache_file cache_file=$(arr_item_cache_file "$arr_type") [[ -f "$cache_file" ]] || return 1 local ts age ts=$(jq -r '.ts // 0' "$cache_file" 2>/dev/null) age=$(( $(date +%s) - ${ts:-0} )) [[ "$age" -ge "$max_age" ]] && return 1 jq -c '.items // empty' "$cache_file" 2>/dev/null } # Records how long a rescan-type command actually took for arr_type, keyed by command name, # so future waits can be calibrated per command type instead of guessed or blended across # very different operations — a whole-library RescanFolders/RescanSeries/RescanMovie takes # vastly longer than a targeted DownloadedXScan, and averaging them would miscalibrate the # wait for both. # Args: arr_type, command_name, duration_seconds arr_record_rescan_duration() { local arr_type="$1" cmd_name="$2" duration="$3" [[ -z "$cmd_name" || -z "$duration" ]] && return 1 local db db=$(arr_rescan_duration_db "$arr_type") mkdir -p "$(dirname "$db")" 2>/dev/null local tmp tmp=$(mktemp) [[ -f "$db" ]] && grep -v "^${cmd_name}|" "$db" > "$tmp" 2>/dev/null echo "${cmd_name}|${duration}" >> "$tmp" mv "$tmp" "$db" } # Echoes the last recorded duration (seconds) for arr_type + command name, or a fallback # default if none has ever been recorded. # Args: arr_type, command_name, fallback_default_seconds arr_get_rescan_duration() { local arr_type="$1" cmd_name="$2" fallback="${3:-300}" local db db=$(arr_rescan_duration_db "$arr_type") [[ -f "$db" ]] || { echo "$fallback"; return; } local val val=$(grep "^${cmd_name}|" "$db" 2>/dev/null | tail -1 | cut -d'|' -f2) echo "${val:-$fallback}" } # Checks arr_type's command queue for any currently-active (started or queued) rescan-type # command (per ARR_RESCAN_COMMANDS). Echoes the command name of the first one found (so the # caller can look up its specific historical duration), empty if none active. # Args: arr_type, url, api_key, api_version arr_active_rescan_command() { local arr_type="$1" url="$2" api_key="$3" api_version="$4" local names="${ARR_RESCAN_COMMANDS[$arr_type]:-}" [[ -z "$names" ]] && return 1 local jq_names jq_names=$(printf '%s\n' $names | jq -R . | jq -sc .) curl -sf --max-time 15 -H "X-Api-Key: $api_key" \ "${url}/api/${api_version}/command" 2>/dev/null | \ jq -r --argjson names "$jq_names" '[.[] | select( (.status=="started" or .status=="queued") and (.name as $n | $names | index($n) != null) )] | .[0].name // empty' 2>/dev/null } # Waits for an already-active rescan-type command for arr_type to finish — unlike # trigger_and_await_command(), this never triggers anything, it only watches. For a rescan # someone/something else started (manual intervention, another script) where the caller just # needs to know when it's safe to fetch+cache the real post-scan numbers. Returns 0 once no # active rescan-type command remains (including immediately if none was active to begin # with), 1 on timeout. # Args: arr_type, url, api_key, api_version, poll_timeout (default 7200s), poll_interval (default 15s) arr_wait_for_active_rescan() { local arr_type="$1" url="$2" api_key="$3" api_version="$4" local poll_timeout="${5:-7200}" poll_interval="${6:-15}" local polled=0 active active=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version") [[ -z "$active" ]] && return 0 while [[ "$polled" -lt "$poll_timeout" ]]; do sleep "$poll_interval" (( polled += poll_interval )) active=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version") [[ -z "$active" ]] && return 0 [[ $(( polled % 300 )) -lt "$poll_interval" ]] && log " Still waiting on ${arr_type}'s ${active} (${polled}s elapsed)" done return 1 } # Main entry point — the only function consuming scripts should call for tracked library # data. Handles fresh/stale-no-rescan/stale-rescan-active branching and always writes through # on any live fetch it performs. Echoes the library-list JSON array on success, returns 1 if # no usable data (no cache and live fetch impossible) could be obtained. # Args: arr_type, url, api_key, api_version, max_age_days (default 1), max_wait_strikes (default 3) arr_get_tracked_data() { local arr_type="$1" url="$2" api_key="$3" api_version="$4" local max_age_days="${5:-${LIDARR_CACHE_MAX_AGE_DAYS:-1}}" max_strikes="${6:-3}" local max_age_seconds=$(( max_age_days * 86400 )) local age age=$(arr_cache_age_seconds "$arr_type") if [[ "$age" -lt "$max_age_seconds" ]]; then arr_cache_read_raw "$arr_type" && return 0 fi # Cache missing or stale — check for an active rescan before deciding how to proceed. local active_cmd active_cmd=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version") if [[ -n "$active_cmd" ]]; then local wait_duration strike arr_label wait_duration=$(( $(arr_get_rescan_duration "$arr_type" "$active_cmd" 300) / 2 )) [[ "$wait_duration" -lt 30 ]] && wait_duration=30 arr_label="$(tr '[:lower:]' '[:upper:]' <<< "${arr_type:0:1}")${arr_type:1}" for (( strike=1; strike<=max_strikes; strike++ )); do # Redirected to stderr — this function's stdout is a data channel (callers # capture it via command substitution), never mix log/warn output into it. warn " $arr_label busy ($active_cmd) — waiting ${wait_duration}s (strike ${strike}/${max_strikes})" >&2 sleep "$wait_duration" active_cmd=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version") [[ -z "$active_cmd" ]] && break done if [[ -n "$active_cmd" ]]; then warn " $arr_label still busy ($active_cmd) after ${max_strikes} strikes — using cache if present, else skipping" >&2 arr_cache_read_raw "$arr_type" && return 0 return 1 fi fi # No active rescan (or it just finished mid-wait) — safe to fetch live and refresh cache. local endpoint fresh arr_label endpoint="${ARR_LIBRARY_ENDPOINT[$arr_type]:-}" [[ -z "$endpoint" ]] && return 1 arr_label="$(tr '[:lower:]' '[:upper:]' <<< "${arr_type:0:1}")${arr_type:1}" fresh=$(arr_api "$url" "$api_key" "$api_version" "$endpoint" "$arr_label") || { arr_cache_read_raw "$arr_type" && return 0 return 1 } arr_cache_write "$arr_type" "$fresh" echo "$fresh" } # ============================================================================================== # ── ARR VERSION CHECK ───────────────────────────────────────────────────────────────────────── # ============================================================================================== # Verifies arr major version matches tested version in master.conf. # Exits if version doesn't match — prevents running against untested API structure. # Warns and proceeds if version endpoint is unreachable. # # Usage: # check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" "Radarr" # # Arguments: # $1 = arr base URL # $2 = API key # $3 = API path prefix (v3 or v1) # $4 = expected major version number from master.conf # $5 = arr name for error messages check_arr_version() { local url="$1" local api_key="$2" local api_prefix="$3" local expected_major="$4" local arr_name="${5:-Arr}" local status_response version major_version status_response=$(curl -sf --max-time 10 \ -H "X-Api-Key: $api_key" \ "${url}/api/${api_prefix}/system/status" 2>/dev/null) if [[ -z "$status_response" ]]; then warn "$arr_name version check failed — could not reach system/status endpoint" warn "Proceeding without version verification — monitor for API errors" return 0 fi version=$(echo "$status_response" | \ grep -o '"version": *"[^"]*"' | \ grep -o '[0-9][^"]*' | head -1) if [[ -z "$version" ]]; then warn "$arr_name version check failed — could not parse version from response" warn "Proceeding without version verification — monitor for API errors" return 0 fi major_version="${version%%.*}" if [[ "$major_version" == "$expected_major" ]]; then success "$arr_name version: $version (major $major_version — tested ✅)" return 0 else error "$arr_name version mismatch — running v${major_version}, tested against v${expected_major}" error "The API endpoint structure may have changed — exiting to protect your library" error "Update ${arr_name^^}_VERSION_MAJOR in master.conf after verifying the script works with v${major_version}" notify "$arr_name version mismatch on $(hostname) — running v${major_version}, script tested against v${expected_major}" "$arr_name Cleanup" "warning" exit 1 fi } # ============================================================================================== # ── TMDB DISCOVERY SCORING ──────────────────────────────────────────────────────────────────── # ============================================================================================== # Shared by playback_aware_radarr_discovery.sh and playback_aware_sonarr_discovery.sh — stage 2 # candidate scoring against TMDB rating/vote/seed-breadth data. Byte-for-byte identical between # the two before this consolidation. # Formats a TMDB vote_average×10 integer back to one decimal place (e.g. 78 → 7.8). _fmt_rating() { local v="${1:-0}"; echo "${v::-1}.${v: -1}" 2>/dev/null || echo "$v"; } # Stage 2: TMDB vote_average × 10 as integer (0-40) _rating_score_s2() { local v="$1" if (( v >= 80 )); then echo 40 elif (( v >= 75 )); then echo 32 elif (( v >= 70 )); then echo 25 elif (( v >= 65 )); then echo 18 elif (( v >= 60 )); then echo 12 else echo 5 fi } # Stage 2: vote count (0-20) _votes_score() { local c="$1" if (( c >= 10000 )); then echo 20 elif (( c >= 5000 )); then echo 15 elif (( c >= 1000 )); then echo 10 elif (( c >= 200 )); then echo 5 else echo 2 fi } # Stage 2: seed breadth (0-40) _breadth_score() { local seeds="$1" if (( seeds >= 3 )); then echo 40 elif (( seeds == 2 )); then echo 25 else echo 10 fi } # ============================================================================================== # ── EMBY LIBRARY SCAN ───────────────────────────────────────────────────────────────────────── # ============================================================================================== # Triggers Emby "Clean Missing Files" scheduled task after arr cleanup scripts delete files. # Without this Emby keeps showing deleted files as ghost entries until its next scheduled scan. # User clicks a ghost entry → file not found error. This prevents that entirely. # # Uses EMBY_URL and EMBY_API_KEY — aliased by detect_hosts() from HOST*_EMBY_* vars. # Skips cleanly if Emby is not configured on this host (empty URL or API key). # # Emby task discovery: # GET /ScheduledTasks → find task where Name contains "Clean Missing" # POST /ScheduledTasks/Running/{taskId} → trigger it # # Non-fatal — logs warning if task cannot be triggered but does not exit. # Clean Missing Files runs across all libraries at once — one call handles # music, TV, and movies together regardless of which arr triggered the cleanup. # # Usage: notify_emby_scan (no args — always triggers Clean Missing Files) notify_emby_scan() { # Skip if Emby not configured on this host if [[ -z "${EMBY_URL:-}" ]] || [[ -z "${EMBY_API_KEY:-}" ]]; then log "Emby not configured on $MY_ID — skipping library scan notification" return 0 fi log "Notifying Emby to clean missing files..." # Get all scheduled tasks local tasks_response tasks_response=$(curl -sf --max-time 15 \ -H "X-Emby-Token: $EMBY_API_KEY" \ "${EMBY_URL}/ScheduledTasks" 2>/dev/null) if [[ -z "$tasks_response" ]]; then warn "Could not reach Emby scheduled tasks API at $EMBY_URL — skipping scan" return 0 fi # Find the "Clean Missing Files" task ID local task_id task_id=$(echo "$tasks_response" | \ grep -o '"Id":"[^"]*","Name":"[^"]*Clean Missing[^"]*"' | \ grep -o '"Id":"[^"]*"' | \ sed 's/"Id":"//;s/"//' | head -1) # Fallback — try "Scan Media Library" if Clean Missing not found if [[ -z "$task_id" ]]; then task_id=$(echo "$tasks_response" | \ grep -o '"Id":"[^"]*","Name":"[^"]*Scan Media Library[^"]*"' | \ grep -o '"Id":"[^"]*"' | \ sed 's/"Id":"//;s/"//' | head -1) [[ -n "$task_id" ]] && log "Clean Missing Files not found — using Scan Media Library" fi if [[ -z "$task_id" ]]; then warn "Could not find Emby Clean Missing Files or Scan Media Library task" warn "Check Emby scheduled tasks — ghost entries will persist until next Emby scan" return 0 fi log "Found Emby task ID: $task_id" # Trigger the task local http_code http_code=$(curl -sf --max-time 15 \ -X POST \ -H "X-Emby-Token: $EMBY_API_KEY" \ -w "%{http_code}" \ -o /dev/null \ "${EMBY_URL}/ScheduledTasks/Running/${task_id}" 2>/dev/null) if [[ "$http_code" == "204" ]] || [[ "$http_code" == "200" ]]; then warn "$ICON_EMBY Emby Clean Missing Files triggered — ghost entries will be removed" else warn "Emby task trigger returned HTTP $http_code — ghost entries may persist" warn "Check Emby dashboard — manually run Clean Missing Files if needed" fi } # ============================================================================================== # ── API REACHABILITY CHECK ──────────────────────────────────────────────────────────────────── # ============================================================================================== # Pre-flight check — verifies an API endpoint is reachable before attempting operations. # Non-fatal — returns 0 if reachable, 1 if not. Caller decides whether to exit. # # Usage: check_api "http://localhost:8989" "Sonarr" || exit 1 check_api() { local url="$1" local service="${2:-API}" local timeout="${3:-10}" if curl -sf --max-time "$timeout" "$url" >/dev/null 2>&1; then log "$service API reachable: $url" return 0 else error "$service API not reachable: $url" return 1 fi } # ============================================================================================== # ── UNRAID VERSION PARITY CHECK ─────────────────────────────────────────────────────────────── # ============================================================================================== # Verifies local and remote servers are running compatible unRAID versions before any # remote operation. Version mismatches can mean changed APIs, commands, or behaviours # that silently break remote container operations, rsync, or fallback logic. # # Reads platform OS version on both sides via adapter. # # Mismatch behaviour (UNRAID_VERSION_MISMATCH_ACTION in master.conf): # "warn" — log warning and continue (default for minor/patch differences) # "abort" — exit the calling script (default for major version differences) # Major version mismatch always aborts regardless of setting # # Usage: check_os_version_parity || exit 1 check_os_version_parity() { local local_version remote_version # Read local version local_version=$(platform_get_os_version 2>/dev/null) if [[ -z "$local_version" ]]; then warn "Cannot read local OS version — skipping version parity check" return 0 fi # Read remote version via SSH local _probe_cmd _probe_cmd=$(platform_os_version_probe_cmd) remote_version=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ "$_probe_cmd" 2>/dev/null) if [[ -z "$remote_version" ]]; then warn "Cannot read remote OS version from $REMOTE_SERVER_NAME — skipping parity check" return 0 fi log "Version parity: local=$local_version remote=$remote_version" if [[ "$local_version" == "$remote_version" ]]; then log "OS versions match: $local_version ✅" return 0 fi # Parse major versions local local_major remote_major local_major="${local_version%%.*}" remote_major="${remote_version%%.*}" if [[ "$local_major" != "$remote_major" ]]; then error "OS MAJOR version mismatch — local: $local_version remote: $remote_version" error "Major version differences may break remote APIs, commands, and behaviours" error "Update both servers to the same major version before running remote operations" notify "OS major version mismatch on $(hostname) — local: $local_version remote: $remote_version — remote operations aborted" \ "Version Parity" "warning" return 1 fi # Minor/patch mismatch — action depends on config local action="${UNRAID_VERSION_MISMATCH_ACTION:-warn}" warn "OS version mismatch — local: $local_version remote: $remote_version" if [[ "$action" == "abort" ]]; then error "UNRAID_VERSION_MISMATCH_ACTION=abort — refusing to continue" notify "OS version mismatch on $(hostname) — local: $local_version remote: $remote_version — aborted" \ "Version Parity" "warning" return 1 fi warn "UNRAID_VERSION_MISMATCH_ACTION=warn — continuing despite mismatch (monitor for issues)" return 0 } # ============================================================================================== # ── REMOTE DOCKER DAEMON CHECK ──────────────────────────────────────────────────────────────── # ============================================================================================== # Verifies the remote Docker daemon is responsive before issuing any remote container commands. # A hung remote daemon causes docker commands to hang or silently fail — checking first # prevents cascading failures and gives clear diagnostic output. # # Strike system: # Each consecutive failed check adds a strike to REMOTE_DOCKER_STRIKES # Below REMOTE_DOCKER_STRIKE_LIMIT → warn and return 1 (caller skips operation) # At limit → notify critical and return 1 (caller should exit or escalate) # Strikes reset when daemon recovers # # Usage: check_remote_docker_daemon || return 1 REMOTE_DOCKER_STRIKES=0 check_remote_docker_daemon() { local timeout="${1:-10}" if ssh -i "$SSH_KEY" -o ConnectTimeout="$timeout" root@"$REMOTE_SERVER" \ "timeout $timeout docker info" >/dev/null 2>&1; then # Daemon healthy — clear strikes if we had issues if [[ "$REMOTE_DOCKER_STRIKES" -gt 0 ]]; then info "Remote Docker daemon on $REMOTE_SERVER_NAME recovered — clearing strikes" REMOTE_DOCKER_STRIKES=0 fi log "Remote Docker daemon healthy on $REMOTE_SERVER_NAME" return 0 fi REMOTE_DOCKER_STRIKES=$(( REMOTE_DOCKER_STRIKES + 1 )) local limit="${REMOTE_DOCKER_STRIKE_LIMIT:-3}" warn "Remote Docker daemon not responding on $REMOTE_SERVER_NAME (strike $REMOTE_DOCKER_STRIKES/$limit)" if [[ "$REMOTE_DOCKER_STRIKES" -ge "$limit" ]]; then error "Remote Docker daemon unresponsive on $REMOTE_SERVER_NAME after $limit consecutive checks" error "Skipping all remote container operations — check Docker on $REMOTE_SERVER_NAME" notify "Remote Docker daemon unresponsive on $REMOTE_SERVER_NAME — remote operations skipped on $(hostname)" "Remote Docker Daemon" "warning" fi return 1 } # ============================================================================================== # ── STATUS DISPLAY ──────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Prints current runtime configuration — triggered by --status flag in any script. # Useful for verifying detect_hosts() resolved correctly and profile loaded as expected. show_status() { local local_ver local_ver=$(platform_get_os_version 2>/dev/null || echo "unknown") echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" echo "Local: $LOCAL_SERVER_NAME" echo "Remote: $REMOTE_SERVER_NAME" echo "IP: ${REMOTE_SERVER:-not resolved}" echo "My ID: ${MY_ID:-not set}" echo "Remote ID: ${REMOTE_ID:-not set}" echo "OS ver: $local_ver" echo "Profile: ${PROFILE_NAME:-n/a}" echo "DryRun: ${DRY_RUN:-false}" echo "Logging: ${ENABLE_LOGGING:-false}" echo "SSH Key: ${SSH_KEY:-not set}" echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]:-n/a}" echo "Delayed: ${DELAYED_CONTAINERS[*]:-n/a}" echo "Excludes: ${EXCLUDE_DIRS[*]:-n/a}" echo "━━━━━━━━━━━━━━━━━━━━━━━" } require_partnership() { [[ "${PARTNERSHIP_ENABLED:-false}" == "true" ]] && return 0 log "PARTNERSHIP_ENABLED=false — skipping cross-server operation" exit 0 }