Gmer4Lfe
c377ddfcca
Finishes the pass: every script now documents its safeguards, and the deliberate absences in the sourced libraries are recorded so they are not "corrected" later.
3135 lines
147 KiB
Bash
Executable File
3135 lines
147 KiB
Bash
Executable File
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ================================= COMMON LIBRARY =============================================
|
||
# ==============================================================================================
|
||
#
|
||
# PURPOSE
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Shared functions used by every script in the ecosystem.
|
||
# Sourced automatically by load_config.sh — do not source directly.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL MODEL
|
||
# ==============================================================================================
|
||
#
|
||
# A pure function library. It defines and returns; it starts nothing on its own.
|
||
#
|
||
# Load order matters and is owned by load_config.sh:
|
||
# varaverk.cfg → master.conf → host*.conf → common.sh → Plugin/$PLATFORM/adapter.sh
|
||
#
|
||
# common.sh is sourced AFTER the conf files because detect_hosts() and the health checks
|
||
# need HOST* and the thresholds to already exist. It is sourced BEFORE the adapter so the
|
||
# adapter can rely on log()/warn()/error() being defined.
|
||
#
|
||
# Note it defines detect_hosts() but never calls it. MY_ID and REMOTE_ID stay unset until a
|
||
# script calls it itself — anything building HOST*-prefixed variable names must do so after
|
||
# that call, not before.
|
||
#
|
||
# ==============================================================================================
|
||
# DESIGN PRINCIPLES
|
||
# ==============================================================================================
|
||
#
|
||
# Define, Never Act
|
||
# Sourcing this file changes no system state: no containers touched, no files written, no
|
||
# locks taken, no host detected. Every script in the ecosystem sources it, including
|
||
# read-only reporting ones, so anything that acted here would act everywhere.
|
||
#
|
||
# One Implementation of Each Shared Behaviour
|
||
# Locking, host detection, notification, retry and timeout wrapping live here once. When
|
||
# a safeguard needs strengthening it is strengthened for every caller at the same moment —
|
||
# which is why individual scripts call acquire_lock() rather than rolling their own flock.
|
||
#
|
||
# Platform-Agnostic
|
||
# No OS-specific paths or commands. Anything Unraid-specific belongs in the adapter, which
|
||
# is why this file calls platform_*() rather than touching /etc/rc.d or /boot directly.
|
||
#
|
||
# Callers Own Policy
|
||
# Helpers return status; they do not decide what failure means. retry_docker() reports
|
||
# that a command failed after N attempts — whether that aborts a run, skips an item, or
|
||
# raises a notification is the caller's call, and only the caller has the context.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL SAFEGUARDS
|
||
# ==============================================================================================
|
||
#
|
||
# No Root, No Lock, No detect_hosts on Load — Deliberate
|
||
# This is a sourced library and must stay inert. A root check here would fire for every
|
||
# script including ones that legitimately do not need it; a lock would be taken on every
|
||
# source; calling detect_hosts() automatically would exit the caller on an unknown
|
||
# hostname before it had a chance to handle that itself. Do not add them.
|
||
#
|
||
# Safe Defaults on Every Threshold
|
||
# Helpers apply :- defaults (DOCKER_TIMEOUT, RESTART_VERIFY_WAIT, RETRY_COUNT and friends)
|
||
# so a conf missing a key degrades to a sane value rather than an empty string that would
|
||
# silently disable a timeout or a retry ceiling.
|
||
#
|
||
# Timeout Wrapping Provided Centrally
|
||
# docker_cmd() and the remote helpers wrap calls in timeout, so no caller has to remember
|
||
# to. A hung daemon or unreachable partner cannot stall a scheduled window.
|
||
#
|
||
# Locking Provided Centrally
|
||
# acquire_lock() implements strict, wait and continuous modes in one place, so every
|
||
# script gets identical semantics and a fix applies everywhere at once.
|
||
#
|
||
# Host Identity Is Exact-Match Only
|
||
# detect_hosts() matches hostname exactly, with a single explicit NetBIOS-truncation
|
||
# fallback that requires an unambiguous candidate. No fuzzy or similarity matching — a
|
||
# wrong host identity would alias the wrong credentials and paths into every script.
|
||
#
|
||
# ==============================================================================================
|
||
# CONFIGURATION
|
||
# ==============================================================================================
|
||
#
|
||
# Defines no config of its own. It consumes what load_config.sh has already sourced from
|
||
# master.conf and host*.conf, and applies defaults where a key may be absent.
|
||
#
|
||
# Consumed broadly: DOCKER_TIMEOUT, RESTART_VERIFY_WAIT, RETRY_COUNT, SLEEP, SSH_KEY,
|
||
# SSH_TIMEOUT, STATE_DIR, DATA_DIR, LOCK_DIR, NOTIFY_UNRAID, HOST*, and the RSYNC_*/
|
||
# PARTNERSHIP_* gates.
|
||
#
|
||
# ==============================================================================================
|
||
# RUNTIME MODES
|
||
# ==============================================================================================
|
||
#
|
||
# None — sourced, never executed:
|
||
#
|
||
# source "$SCRIPT_DIR/../load_config.sh" # which sources this file
|
||
#
|
||
# It parses no arguments of its own. parse_args() is defined here for callers to invoke,
|
||
# and --dry-run/--status/--log are honoured by the calling script, not by this file.
|
||
#
|
||
# ==============================================================================================
|
||
# ── WHAT THIS FILE PROVIDES ───────────────────────────────────────────────────────────────────
|
||
# Icons — consistent visual language across all script output
|
||
# Output helpers — info, warn, error, success, log, notify
|
||
# Arg parser — unified --dry-run, --log, --status, KEY=VALUE handling
|
||
# Validation — require_var, validate_int
|
||
# Host detection — detect_hosts() identifies local/remote server, sets all aliases
|
||
# Connectivity — resolve_remote_ip, check_connectivity, ping_remote, ping_internet
|
||
# Health checks — local/remote array, docker daemon, rootfs, share, disk temps
|
||
# Disk verification — check_remote_disks() — XFS, ZFS, cache pools, nested paths
|
||
# Container mgmt — stop/start local and remote containers with state tracking
|
||
# Rsync helpers — get_rsync_opts, check_rsync_enabled
|
||
# Locking — acquire_lock, acquire_rsync_lock — concurrent execution protection
|
||
# Path translation — translate_path() — container → host path for arr cleanup scripts
|
||
# Arr utilities — check_arr_version() — API version safety gate
|
||
# Status display — show_status() — runtime config dump on --status flag
|
||
#
|
||
# ── CHANGELOG ─────────────────────────────────────────────────────────────────────────────────
|
||
#
|
||
# Version What changed
|
||
# ───────────────────────────────────────────────────────────────────────────────────────────
|
||
# v1.0 Initial stable framework — output helpers, arg parser, host detection
|
||
#
|
||
# v1.1 format_duration() moved here from daily_sync_maintenance.sh for shared use
|
||
# SSH_KEY naming collision resolved — gitea key renamed GITEA_SSH_KEY in master.conf
|
||
#
|
||
# v1.2 check_connectivity() added — fatal ping with Tailscale hint on failure
|
||
# Function header comment blocks standardised across all functions
|
||
#
|
||
# v1.3 check_remote_rootfs() — aborts rsync if remote rootfs exceeds ROOTFS_WARN
|
||
# check_remote_share() — aborts if target directory missing or empty on remote
|
||
# Both protect against rsync running when remote array is down
|
||
#
|
||
# v1.4 check_remote_disks() — verifies all physical disks backing a share are mounted
|
||
# Auto-discovers disk layout at runtime from disks.ini — no config required
|
||
# Aborts if any disk backing the share is offline or unmounted
|
||
#
|
||
# v1.5-2.9 Icon set expanded progressively — each operation has its own distinct icon
|
||
# All function output updated to use correct icon per context
|
||
# notify() added — unRAID native + Discord webhook notifications
|
||
# Script locking system added — acquire_lock(), acquire_rsync_lock()
|
||
# translate_path() added — container → host path for arr cleanup scripts
|
||
# check_arr_version() added — API version safety gate before arr operations
|
||
# check_api() added — pre-flight API reachability check
|
||
# ping_remote(), ping_internet() — non-fatal ping for fallback use
|
||
# check_remote_array() — array health check
|
||
# get_unraid_temp_thresholds() — reads thresholds from dynamix.cfg
|
||
# check_local_disk_temps() — pre-rsync temp check with exit codes 0/1/2
|
||
# stop/start local containers added alongside existing remote variants
|
||
#
|
||
# v3.1 Three-file config split — master.conf + host1.conf + host2.conf
|
||
# load_config.sh introduced — auto-discovers all host*.conf files
|
||
# detect_hosts() rewritten — sets MY_ID/REMOTE_ID and aliases all HOST* vars
|
||
# All scripts now source load_config.sh instead of conf files directly
|
||
# Adding a new server = add host*.conf, zero script changes required
|
||
#
|
||
# v3.2 check_remote_disks() rewritten — three-tier detection:
|
||
# Tier 1: array disk paths (/mnt/disk*/sharename)
|
||
# Tier 2: ZFS pools with find -maxdepth 2 (catches nested cache paths)
|
||
# Tier 3: shfs fallback — verifies /mnt/user is mounted
|
||
# Fixes: nested paths like /mnt/cache/appdata-Fallback/Critical-Data
|
||
# Fixes: ZFS cache pools not detected when share is not at pool root
|
||
#
|
||
# v3.3 PROFILE_REMOTE_RESTART_CONTAINERS support added
|
||
# Dirty sync profiles (critical-fallback) restart remote
|
||
# containers after sync if they were running before — picks up config changes
|
||
# Was stopped → stays stopped. Was running → gets restarted. ✅
|
||
#
|
||
# v3.5 notify_emby_scan() added — triggers Emby "Clean Missing Files" task
|
||
# after arr cleanup scripts delete orphaned files
|
||
# Emby immediately removes ghost entries — no user-facing file-not-found errors
|
||
# Called by lidarr/sonarr/radarr_cleanup.sh when files are deleted
|
||
#
|
||
#
|
||
# Two new safety functions added:
|
||
# check_os_version_parity() — refuses remote ops on version mismatch
|
||
# reads OS version via platform_get_os_version() / platform_os_version_probe_cmd()
|
||
# major mismatch → abort | minor mismatch → configurable warn/abort
|
||
# check_remote_docker_daemon() — verifies remote Docker daemon before
|
||
# issuing any remote container commands — strike system → skip/retry/exit
|
||
#
|
||
# ── VERSION ───────────────────────────────────────────────────────────────────────────────────
|
||
# Current: v3.5
|
||
# ==============================================================================================
|
||
|
||
# Cron runs with a minimal PATH (/usr/bin:/bin) that omits /usr/local/sbin where tailscale,
|
||
# and other Unraid tools live. Prepend it here so every sourcing script finds them.
|
||
export PATH="/usr/local/sbin:/usr/local/bin:$PATH"
|
||
|
||
# ==============================================================================================
|
||
# ── ICONS ─────────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Each icon has one job — do not reuse across different contexts.
|
||
# Adding a new icon: add to the appropriate group below with a comment describing its job.
|
||
|
||
# System / Host
|
||
ICON_HOST="🖥️" # host detection and identity
|
||
ICON_NET="🌐" # network / IP resolution
|
||
ICON_PING="📡" # connectivity check
|
||
ICON_GEAR="⚙️" # setup section header / profile load
|
||
|
||
# Health Checks
|
||
ICON_DISK="💾" # disk checks
|
||
ICON_HEALTH="🩺" # rootfs / share health checks
|
||
ICON_SHIELD="🛡️" # pre-flight section header
|
||
|
||
# Containers
|
||
ICON_CONTAINERS="📦" # container section anchor
|
||
ICON_STOP="⛔" # stop command being issued
|
||
ICON_STOPPED="🔴" # container confirmed stopped
|
||
ICON_START="▶️" # start command being issued
|
||
ICON_STARTED="💚" # container confirmed started
|
||
ICON_RUNNING="🟢" # container already running when checked
|
||
ICON_SKIP="⏭️" # skipping — already running healthy instance
|
||
ICON_PROTECTED="🔰" # file/item is protected — never delete
|
||
ICON_NOT_RUNNING="⭕" # container already stopped when checked
|
||
|
||
# Transfer
|
||
ICON_SYNC="🔄" # transfer section header
|
||
ICON_RUN="🚀" # sync starting / rsync attempt
|
||
ICON_RETRY="🔁" # retry attempt
|
||
ICON_DONE="🏁" # transfer complete
|
||
ICON_GIT="🌱" # git pull/push section header
|
||
|
||
# Summary
|
||
ICON_SUMMARY="📋" # summary section header
|
||
ICON_TIME="⏱️" # duration line
|
||
|
||
# System Operations
|
||
ICON_MOVER="🔃" # mover operations
|
||
ICON_REBOOT="⚡" # scheduled server reboot
|
||
ICON_REBOOT_SMART="🚨" # smart conditional reboot triggered
|
||
ICON_PLUGIN="🧩" # user scripts plugin operations
|
||
ICON_PHP="👥" # PHP-FPM operations
|
||
ICON_WEBGUI="💻" # WebGUI / nginx / emhttp operations
|
||
|
||
# Diagnostics
|
||
ICON_ZFS="📊" # ZFS ARC statistics
|
||
ICON_MEM="🧠" # memory status
|
||
ICON_WATCHDOG="🐾" # docker watchdog monitoring operations
|
||
|
||
# Media Operations
|
||
ICON_CLEAN="🧹" # media cleaner operations
|
||
ICON_TRASH="🗑️" # files being deleted
|
||
ICON_PERMS="🔐" # permissions operation / section header
|
||
ICON_UNLOCKED="🔓" # permissions successfully applied to a share
|
||
ICON_LOCK="🔏" # script instance lock — acquired/released
|
||
|
||
# Transcode Operations
|
||
ICON_RAM="💨" # ramdisk operations — fast ephemeral storage
|
||
ICON_LINK="🔗" # symlink state and management
|
||
|
||
# fallback Operations
|
||
ICON_FALLBACK="🔀" # fallback state changes and operations
|
||
|
||
# Docker Network Operations
|
||
ICON_DOCKER_NET="🔌" # Docker network connect operations
|
||
|
||
# Security / Certificate Operations
|
||
ICON_CERT="🔒" # SSL certificate monitoring
|
||
|
||
# Monitor Operations
|
||
ICON_MONITOR="📈" # monitoring section headers and general monitoring
|
||
ICON_SMART="🔧" # drive SMART health attribute monitoring
|
||
ICON_BANDWIDTH="📶" # bandwidth usage tracking and reporting
|
||
ICON_DIGEST="📰" # health digest — aggregated system summary
|
||
ICON_EMBY="🎬" # Emby media server session reporting
|
||
ICON_VERIFY="✔️" # backup verification — checksum comparison
|
||
|
||
# Notifications
|
||
ICON_NOTIFY="🔔" # notification operations
|
||
|
||
# Output
|
||
ICON_INFO="ℹ️"
|
||
ICON_WARN="⚠️"
|
||
ICON_ERROR="❌"
|
||
ICON_SUCCESS="✅"
|
||
|
||
# ==============================================================================================
|
||
# ── OUTPUT HELPERS ────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Standardised output functions used across all scripts.
|
||
#
|
||
# Output tiers:
|
||
# echo — always visible — summaries, status conclusions, section headers
|
||
# info() — always visible — status confirmations, reachability, check pass
|
||
# warn() — always visible — state transitions, warnings, important events
|
||
# error() — always visible — something broke
|
||
# success() — always visible — operation completed successfully
|
||
# log() — only with --log flag — per-item detail, internal checks
|
||
#
|
||
# All output goes to stdout — callers can redirect as needed.
|
||
|
||
info() { echo "$ICON_INFO [INFO] $*"; }
|
||
warn() { echo "$ICON_WARN [WARN] $*"; }
|
||
error() { echo "$ICON_ERROR [ERROR] $*"; }
|
||
success() { echo "$ICON_DONE [OK] $*"; }
|
||
|
||
log() {
|
||
[[ "${ENABLE_LOGGING:-false}" == true ]] && echo "[LOG] $*"
|
||
return 0
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── NOTIFICATION ──────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Sends a notification via unRAID native system and/or Discord webhook.
|
||
# Both channels are optional and independently controlled:
|
||
# NOTIFY_UNRAID — shared toggle in master.conf
|
||
# MY_DISCORD_WEBHOOK — per-host in host*.conf, set by detect_hosts()
|
||
#
|
||
# Severity levels: normal, warning, alert
|
||
# Usage: notify "message" "subject" "severity"
|
||
|
||
notify() {
|
||
local message="$1"
|
||
local subject="${2:-unRAID Notification}"
|
||
local severity="${3:-normal}"
|
||
|
||
log "$ICON_NOTIFY Sending notification: $subject — $message"
|
||
|
||
if [[ "${NOTIFY_UNRAID:-false}" == true ]]; then
|
||
if platform_send_os_notification "$message" "$subject" "$severity"; then
|
||
log "$ICON_NOTIFY OS notification sent"
|
||
else
|
||
log "$ICON_NOTIFY OS notify not available — skipping"
|
||
fi
|
||
fi
|
||
|
||
# Uses MY_DISCORD_WEBHOOK — set by detect_hosts() from HOST*_DISCORD_WEBHOOK
|
||
if [[ -n "${MY_DISCORD_WEBHOOK:-}" ]]; then
|
||
local payload
|
||
payload=$(printf '{"content": "%s — **%s**\\n%s"}' \
|
||
"$ICON_NOTIFY" "$subject" "$message")
|
||
if curl -s -H "Content-Type: application/json" \
|
||
-d "$payload" "$MY_DISCORD_WEBHOOK" >/dev/null 2>&1; then
|
||
log "$ICON_NOTIFY Discord notification sent"
|
||
else
|
||
warn "Discord notification failed — check HOST*_DISCORD_WEBHOOK in host*.conf"
|
||
fi
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── DURATION FORMATTER ────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Converts raw seconds into a human readable string.
|
||
# Examples: 47 → "47s" | 653 → "10m53s"
|
||
# Usage: format_duration $SECONDS
|
||
|
||
format_duration() {
|
||
local secs=$1
|
||
local days=$(( secs / 86400 ))
|
||
local hrs=$(( (secs % 86400) / 3600 ))
|
||
local mins=$(( (secs % 3600) / 60 ))
|
||
local rem=$(( secs % 60 ))
|
||
if [[ $days -gt 0 ]]; then echo "${days}d ${hrs}h ${mins}m"
|
||
elif [[ $hrs -gt 0 ]]; then echo "${hrs}h ${mins}m"
|
||
elif [[ $mins -gt 0 ]]; then echo "${mins}m${rem}s"
|
||
else echo "${rem}s"; fi
|
||
}
|
||
|
||
# Tiered human-readable size — picks B/MB/GB based on magnitude.
|
||
# Usage: format_bytes 5368709120 → 5.0GB
|
||
format_bytes() {
|
||
local bytes=${1:-0}
|
||
if (( bytes > 1073741824 )); then
|
||
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
|
||
elif (( bytes > 1048576 )); then
|
||
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
|
||
else
|
||
echo "${bytes}B"
|
||
fi
|
||
}
|
||
|
||
# Fixed-precision bytes → GB (no tiering) — for values always compared against a GB threshold.
|
||
# Usage: bytes_to_gb 5368709120 [decimals=2] → 5.00
|
||
bytes_to_gb() {
|
||
local bytes=${1:-0} decimals=${2:-2}
|
||
awk "BEGIN {printf \"%.${decimals}f\", $bytes / 1073741824}"
|
||
}
|
||
|
||
# Fixed-precision KB → GB (no tiering) — df --output=used/avail reports in KB.
|
||
# Usage: kb_to_gb 5242880 [decimals=2] → 5.00
|
||
kb_to_gb() {
|
||
local kb=${1:-0} decimals=${2:-2}
|
||
awk "BEGIN {printf \"%.${decimals}f\", $kb / 1048576}"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── ARG PARSER ────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Processes all flags and key=value pairs passed to any script.
|
||
# Call early in every script: parse_args "$@"
|
||
#
|
||
# Supported flags:
|
||
# --dry-run | -n → DRY_RUN=true
|
||
# --log → ENABLE_LOGGING=true
|
||
# --no-log → ENABLE_LOGGING=false
|
||
# --status → SHOW_STATUS=true
|
||
# --help | -h → print usage and exit
|
||
#
|
||
# Supported key=value:
|
||
# LOG=true/false → toggle logging
|
||
# BW_LIMIT=5000 → override any declared master.conf variable for this run
|
||
#
|
||
# Unparsed positional args returned in PARSED_ARGS array.
|
||
|
||
parse_args() {
|
||
ENABLE_LOGGING=${ENABLE_LOGGING:-false}
|
||
DRY_RUN=${DRY_RUN:-false}
|
||
SHOW_STATUS=${SHOW_STATUS:-false}
|
||
CLEAN_ARGS=()
|
||
|
||
for ARG in "$@"; do
|
||
if [[ "$ARG" == *=* ]]; then
|
||
VAR="${ARG%%=*}"
|
||
VAL="${ARG#*=}"
|
||
case "$VAR" in
|
||
LOG)
|
||
[[ "$VAL" == "true" ]] && ENABLE_LOGGING=true
|
||
[[ "$VAL" == "false" ]] && ENABLE_LOGGING=false
|
||
;;
|
||
*)
|
||
if declare -p "$VAR" &>/dev/null; then
|
||
printf -v "$VAR" '%s' "$VAL"
|
||
log "Set $VAR=$VAL"
|
||
else
|
||
warn "Unknown variable: $VAR"
|
||
fi
|
||
;;
|
||
esac
|
||
else
|
||
case "$ARG" in
|
||
--dry-run|-n) DRY_RUN=true ;;
|
||
--log) ENABLE_LOGGING=true ;;
|
||
--no-log) ENABLE_LOGGING=false ;;
|
||
--status|--summary) SHOW_STATUS=true ;;
|
||
--help|-h)
|
||
echo "Usage: script [--dry-run] [--log] [--status]"
|
||
exit 0
|
||
;;
|
||
*) CLEAN_ARGS+=("$ARG") ;;
|
||
esac
|
||
fi
|
||
done
|
||
|
||
PARSED_ARGS=("${CLEAN_ARGS[@]}")
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── VALIDATION HELPERS ────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# Exits with error if a required variable is empty or unset.
|
||
# Usage: require_var VAR_NAME
|
||
require_var() {
|
||
[[ -z "${!1:-}" ]] && error "Missing required: $1" && exit 1
|
||
}
|
||
|
||
# Exits with error if a variable is not a valid positive integer.
|
||
# Usage: validate_int VAR_NAME "$VAR_VALUE"
|
||
validate_int() {
|
||
local name="$1" value="$2"
|
||
if [[ -z "$value" ]]; then
|
||
error "$name is not set — check master.conf"
|
||
exit 1
|
||
fi
|
||
if ! [[ "$value" =~ ^[0-9]+$ ]]; then
|
||
error "$name must be a positive integer — got: '$value'"
|
||
exit 1
|
||
fi
|
||
log "$name validated: $value"
|
||
}
|
||
|
||
# Membership check against a list of values (e.g. an ignore list).
|
||
# Usage: is_in_list "$needle" "${HAYSTACK_ARRAY[@]}"
|
||
is_in_list() {
|
||
local needle="$1"; shift
|
||
local item
|
||
for item in "$@"; do
|
||
[[ "$needle" == "$item" ]] && return 0
|
||
done
|
||
return 1
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── HOST DETECTION ────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Determines which server is local and which is remote by comparing hostname against
|
||
# all HOST* values discovered from host*.conf files.
|
||
#
|
||
# Sets:
|
||
# MY_ID — "HOST1" or "HOST2" (the role key, not the hostname)
|
||
# REMOTE_ID — the peer's role key
|
||
# LOCAL_SERVER_NAME — local hostname string (e.g. "unRAID-Gmer4Lfe")
|
||
# REMOTE_SERVER_NAME — remote hostname string
|
||
# SSH_KEY — SSH key for server-to-server operations
|
||
# REMOTE_STORAGE_PATH — remote's storage root from HOST*_STORAGE_PATH in host*.conf;
|
||
# falls back to platform_storage_path() for unconfigured pairs
|
||
#
|
||
# Also sets aliases for all host-specific arrays so scripts use unprefixed names:
|
||
# DAILY_SYNC_SHARES ← HOST*_DAILY_SYNC_SHARES
|
||
# INTERMEDIATE_SYNC_SHARES ← HOST*_INTERMEDIATE_SYNC_SHARES
|
||
# WEEKLY_SYNC_SHARES ← HOST*_WEEKLY_SYNC_SHARES
|
||
# CRITICAL_SYNC_SHARES ← HOST*_CRITICAL_SYNC_SHARES
|
||
# DAILY_RESTART_CONTAINERS ← HOST*_DAILY_RESTART_CONTAINERS
|
||
# WEEKLY_RESTART_CONTAINERS ← HOST*_WEEKLY_RESTART_CONTAINERS
|
||
# WATCHDOG_CONTAINERS ← HOST*_WATCHDOG_CONTAINERS (associative)
|
||
# WATCHDOG_CONTAINER_URLS ← HOST*_WATCHDOG_CONTAINER_URLS (associative)
|
||
# WATCHDOG_REQUIRED_CONTAINERS ← HOST*_WATCHDOG_REQUIRED_CONTAINERS
|
||
# WATCHDOG_SCAN_IGNORE ← HOST*_WATCHDOG_SCAN_IGNORE
|
||
# WATCHDOG_DEPENDENCIES ← HOST*_WATCHDOG_DEPENDENCIES (associative)
|
||
# WATCHDOG_APPDATA_SIZES ← HOST*_WATCHDOG_APPDATA_SIZES (associative)
|
||
# NETWORK_CONNECT_CONTAINERS ← HOST*_NETWORK_CONNECT_CONTAINERS
|
||
# NETWORK_CONNECT_NETWORKS ← HOST*_NETWORK_CONNECT_NETWORKS
|
||
# MEDIA_PERMISSION_SHARES ← HOST*_MEDIA_PERMISSION_SHARES
|
||
# ANIME_CLEAN_FOLDERS ← HOST*_ANIME_CLEAN_FOLDERS
|
||
# MEDIA_CLEAN_FOLDERS ← HOST*_MEDIA_CLEAN_FOLDERS
|
||
# CERT_MONITOR_DOMAINS ← HOST*_CERT_MONITOR_DOMAINS
|
||
# SMART_IGNORE_DRIVES ← HOST*_SMART_IGNORE_DRIVES
|
||
# ZFS_REPORT_IGNORE_POOLS ← HOST*_ZFS_REPORT_IGNORE_POOLS
|
||
# TRANSCODE_SSD ← HOST*_TRANSCODE_SSD
|
||
# TRANSCODE_SERVERS ← HOST*_TRANSCODE_SERVERS
|
||
# RAMDISK_SIZE ← HOST*_RAMDISK_SIZE
|
||
# RAMDISK_WARN_GB ← HOST*_RAMDISK_WARN_GB
|
||
# RAMDISK_LOW_GB ← HOST*_RAMDISK_LOW_GB
|
||
# BACKUP_VERIFY_SHARES ← HOST*_BACKUP_VERIFY_SHARES
|
||
# DDNS_CONTAINERS ← HOST*_DDNS_CONTAINERS
|
||
# PARTNERSHIP_AUTH_WEBUIS ← HOST*_PARTNERSHIP_AUTH_WEBUIS
|
||
# PARTNERSHIP_MIRROR_BACKUPS ← HOST*_PARTNERSHIP_MIRROR_BACKUPS
|
||
# MY_DISCORD_WEBHOOK ← HOST*_DISCORD_WEBHOOK
|
||
# EMBY_CONTAINER ← HOST*_EMBY_CONTAINER
|
||
# EMBY_URL ← HOST*_EMBY_URL
|
||
# EMBY_API_KEY ← HOST*_EMBY_API_KEY
|
||
# JELLYFIN_CONTAINER ← HOST*_JELLYFIN_CONTAINER
|
||
# JELLYFIN_URL ← HOST*_JELLYFIN_URL
|
||
# JELLYFIN_API_KEY ← HOST*_JELLYFIN_API_KEY
|
||
# GITEA_API_TOKEN ← HOST*_GITEA_API_TOKEN
|
||
#
|
||
# Arr-specific vars set by detect_hosts() when needed by arr scripts:
|
||
# LIDARR_URL / LIDARR_API_KEY / LIDARR_MUSIC_ROOT (MY_ID only — Lidarr is HOST1 only)
|
||
# SONARR_URL / SONARR_API_KEY / SONARR_TV_ROOT
|
||
# RADARR_URL / RADARR_API_KEY / RADARR_MOVIES_ROOT
|
||
# SLSKD_URL / SLSKD_API_KEY / SLSKD_FAILED_IMPORTS_DIR (MY_ID only)
|
||
# SABNZBD_URL / SABNZBD_API_KEY (MY_ID only)
|
||
# QBIT_URL / QBIT_USERNAME / QBIT_PASSWORD (MY_ID only)
|
||
|
||
detect_hosts() {
|
||
local local_hostname
|
||
local_hostname="$(hostname)"
|
||
|
||
# ── Find MY_ID by matching hostname against all HOST* vars ────────────────
|
||
MY_ID=""
|
||
local host_var host_val
|
||
|
||
# Scan all HOST* vars that hold a hostname value
|
||
# HOST1, HOST2, HOST3 etc. — all defined in host*.conf files
|
||
for host_var in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
||
host_val="${!host_var:-}"
|
||
[[ -z "$host_val" ]] && continue
|
||
if [[ "${local_hostname,,}" == "${host_val,,}" ]]; then
|
||
MY_ID="$host_var"
|
||
break
|
||
fi
|
||
done
|
||
|
||
# Unraid truncates the Server Name to 15 chars (NetBIOS limit). If the live hostname
|
||
# is at that exact limit, a configured HOST* value may be a longer, untruncated version
|
||
# (matching what Tailscale shows for this peer — resolve_tailscale_ip() keys off the same
|
||
# value). Only accept the match if it's unambiguous — exactly one HOST* may qualify.
|
||
if [[ -z "$MY_ID" && ${#local_hostname} -eq 15 ]]; then
|
||
local candidate="" candidate_count=0
|
||
for host_var in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
||
host_val="${!host_var:-}"
|
||
[[ -z "$host_val" ]] && continue
|
||
if [[ ${#host_val} -gt 15 && "${host_val,,}" == "${local_hostname,,}"* ]]; then
|
||
candidate="$host_var"
|
||
(( candidate_count++ ))
|
||
fi
|
||
done
|
||
[[ "$candidate_count" -eq 1 ]] && MY_ID="$candidate"
|
||
fi
|
||
|
||
if [[ -z "$MY_ID" ]]; then
|
||
error "Unknown host: $local_hostname"
|
||
error "Hostname must match a HOST* value in host*.conf"
|
||
error "Available: $(for h in HOST1 HOST2 HOST3 HOST4; do
|
||
[[ -n "${!h:-}" ]] && echo -n "${!h} "; done)"
|
||
exit 1
|
||
fi
|
||
|
||
# ── Find REMOTE_ID — first HOST* that isn't MY_ID ────────────────────────
|
||
REMOTE_ID=""
|
||
for host_var in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
||
[[ "$host_var" == "$MY_ID" ]] && continue
|
||
host_val="${!host_var:-}"
|
||
[[ -z "$host_val" ]] && continue
|
||
REMOTE_ID="$host_var"
|
||
break
|
||
done
|
||
|
||
# ── Set host name strings ─────────────────────────────────────────────────
|
||
LOCAL_SERVER_NAME="${!MY_ID}"
|
||
[[ -n "$REMOTE_ID" ]] && REMOTE_SERVER_NAME="${!REMOTE_ID}" || REMOTE_SERVER_NAME="unknown"
|
||
|
||
# ── Set SSH key ───────────────────────────────────────────────────────────
|
||
local ssh_key_var="${MY_ID}_SSH_KEY"
|
||
SSH_KEY="${!ssh_key_var:-}"
|
||
if [[ -z "$SSH_KEY" ]]; then
|
||
error "Missing SSH key: ${ssh_key_var} not set in host*.conf"
|
||
exit 1
|
||
fi
|
||
|
||
# ── Set scalar aliases ────────────────────────────────────────────────────
|
||
MY_DISCORD_WEBHOOK_VAR="${MY_ID}_DISCORD_WEBHOOK"
|
||
MY_DISCORD_WEBHOOK="${!MY_DISCORD_WEBHOOK_VAR:-}"
|
||
|
||
OWNER_NAME_VAR="${MY_ID}_OWNER"
|
||
OWNER_NAME="${!OWNER_NAME_VAR:-}"
|
||
OWNER_EMAIL_VAR="${MY_ID}_OWNER_EMAIL"
|
||
OWNER_EMAIL="${!OWNER_EMAIL_VAR:-}"
|
||
|
||
EMBY_CONTAINER_VAR="${MY_ID}_EMBY_CONTAINER"
|
||
EMBY_CONTAINER="${!EMBY_CONTAINER_VAR:-}"
|
||
EMBY_URL_VAR="${MY_ID}_EMBY_URL"
|
||
EMBY_URL="${!EMBY_URL_VAR:-}"
|
||
EMBY_API_KEY_VAR="${MY_ID}_EMBY_API_KEY"
|
||
EMBY_API_KEY="${!EMBY_API_KEY_VAR:-}"
|
||
|
||
JELLYFIN_CONTAINER_VAR="${MY_ID}_JELLYFIN_CONTAINER"
|
||
JELLYFIN_CONTAINER="${!JELLYFIN_CONTAINER_VAR:-}"
|
||
JELLYFIN_URL_VAR="${MY_ID}_JELLYFIN_URL"
|
||
JELLYFIN_URL="${!JELLYFIN_URL_VAR:-}"
|
||
JELLYFIN_API_KEY_VAR="${MY_ID}_JELLYFIN_API_KEY"
|
||
JELLYFIN_API_KEY="${!JELLYFIN_API_KEY_VAR:-}"
|
||
|
||
GITEA_API_TOKEN_VAR="${MY_ID}_GITEA_API_TOKEN"
|
||
GITEA_API_TOKEN="${!GITEA_API_TOKEN_VAR:-}"
|
||
|
||
local _prov_var
|
||
for _prov_var in \
|
||
PARTNERSHIP_PROVISION_EMBY_ADMIN \
|
||
PARTNERSHIP_EMBY_ADMIN_USER \
|
||
PARTNERSHIP_EMBY_ADMIN_PASS \
|
||
PARTNERSHIP_EMBY_PORT; do
|
||
local _src="${MY_ID}_${_prov_var}"
|
||
printf -v "$_prov_var" '%s' "${!_src:-}"
|
||
done
|
||
|
||
# ── System watchdog check toggles — per-host ─────────────────────────────
|
||
# Aliased as unprefixed SYS_WATCHDOG_CHECK_* for use in system_watchdog.sh
|
||
local _wd_checks=(
|
||
SYS_WATCHDOG_NIC
|
||
SYS_WATCHDOG_CHECK_DOCKER_DAEMON SYS_WATCHDOG_CHECK_ROOTFS
|
||
SYS_WATCHDOG_CHECK_KERNEL_OOPS SYS_WATCHDOG_CHECK_FD
|
||
SYS_WATCHDOG_CHECK_BOOT SYS_WATCHDOG_CHECK_OOM
|
||
SYS_WATCHDOG_CHECK_RAM SYS_WATCHDOG_CHECK_LOG
|
||
SYS_WATCHDOG_CHECK_ARC SYS_WATCHDOG_CHECK_CPU_TEMP
|
||
SYS_WATCHDOG_CHECK_LOAD SYS_WATCHDOG_CHECK_ZOMBIES
|
||
SYS_WATCHDOG_CHECK_CONTAINERS SYS_WATCHDOG_CHECK_TMP
|
||
SYS_WATCHDOG_CHECK_MDSTAT SYS_WATCHDOG_CHECK_NETWORK
|
||
SYS_WATCHDOG_CHECK_SSHD SYS_WATCHDOG_CHECK_RUNAWAY
|
||
)
|
||
local _wd_var
|
||
for _wd_var in "${_wd_checks[@]}"; do
|
||
local _wd_src="${MY_ID}_${_wd_var}"
|
||
# Only alias if the host-specific var is set — preserves master.conf defaults
|
||
[[ -n "${!_wd_src+x}" ]] && eval "${_wd_var}=\"\${${_wd_src}}\""
|
||
done
|
||
|
||
TRANSCODE_SSD_VAR="${MY_ID}_TRANSCODE_SSD"
|
||
TRANSCODE_SSD="${!TRANSCODE_SSD_VAR:-}"
|
||
RAMDISK_SIZE_VAR="${MY_ID}_RAMDISK_SIZE"
|
||
RAMDISK_SIZE="${!RAMDISK_SIZE_VAR:-8G}"
|
||
RAMDISK_WARN_GB_VAR="${MY_ID}_RAMDISK_WARN_GB"
|
||
RAMDISK_WARN_GB="${!RAMDISK_WARN_GB_VAR:-6.8}"
|
||
RAMDISK_LOW_GB_VAR="${MY_ID}_RAMDISK_LOW_GB"
|
||
RAMDISK_LOW_GB="${!RAMDISK_LOW_GB_VAR:-5.5}"
|
||
|
||
# Arr credentials — MY_ID only (remote arr accessed via its own host)
|
||
LIDARR_URL_VAR="${MY_ID}_LIDARR_URL"; LIDARR_URL="${!LIDARR_URL_VAR:-}"
|
||
LIDARR_API_KEY_VAR="${MY_ID}_LIDARR_API_KEY"; LIDARR_API_KEY="${!LIDARR_API_KEY_VAR:-}"
|
||
LIDARR_MUSIC_ROOT_VAR="${MY_ID}_LIDARR_MUSIC_ROOT"; LIDARR_MUSIC_ROOT="${!LIDARR_MUSIC_ROOT_VAR:-}"
|
||
LIDARR_RECOVERY_VAR="${MY_ID}_LIDARR_RECOVERY"; LIDARR_RECOVERY="${!LIDARR_RECOVERY_VAR:-false}"
|
||
FANART_API_KEY_VAR="${MY_ID}_FANART_API_KEY"; FANART_API_KEY="${!FANART_API_KEY_VAR:-}"
|
||
LASTFM_API_KEY_VAR="${MY_ID}_LASTFM_API_KEY"; LASTFM_API_KEY="${!LASTFM_API_KEY_VAR:-}"
|
||
TMDB_API_KEY_VAR="${MY_ID}_TMDB_API_KEY"; TMDB_API_KEY="${!TMDB_API_KEY_VAR:-}"
|
||
|
||
SONARR_URL_VAR="${MY_ID}_SONARR_URL"; SONARR_URL="${!SONARR_URL_VAR:-}"
|
||
SONARR_API_KEY_VAR="${MY_ID}_SONARR_API_KEY"; SONARR_API_KEY="${!SONARR_API_KEY_VAR:-}"
|
||
SONARR_TV_ROOT_VAR="${MY_ID}_SONARR_TV_ROOT"; SONARR_TV_ROOT="${!SONARR_TV_ROOT_VAR:-}"
|
||
SONARR_GENERAL_ROOT_VAR="${MY_ID}_SONARR_GENERAL_ROOT"; SONARR_GENERAL_ROOT="${!SONARR_GENERAL_ROOT_VAR:-}"
|
||
SONARR_KIDS_ROOT_VAR="${MY_ID}_SONARR_KIDS_ROOT"; SONARR_KIDS_ROOT="${!SONARR_KIDS_ROOT_VAR:-}"
|
||
SONARR_ANIME_ROOT_VAR="${MY_ID}_SONARR_ANIME_ROOT"; SONARR_ANIME_ROOT="${!SONARR_ANIME_ROOT_VAR:-}"
|
||
SONARR_RECOVERY_VAR="${MY_ID}_SONARR_RECOVERY"; SONARR_RECOVERY="${!SONARR_RECOVERY_VAR:-true}"
|
||
|
||
RADARR_URL_VAR="${MY_ID}_RADARR_URL"; RADARR_URL="${!RADARR_URL_VAR:-}"
|
||
RADARR_API_KEY_VAR="${MY_ID}_RADARR_API_KEY"; RADARR_API_KEY="${!RADARR_API_KEY_VAR:-}"
|
||
RADARR_MOVIES_ROOT_VAR="${MY_ID}_RADARR_MOVIES_ROOT"; RADARR_MOVIES_ROOT="${!RADARR_MOVIES_ROOT_VAR:-}"
|
||
RADARR_GENERAL_ROOT_VAR="${MY_ID}_RADARR_GENERAL_ROOT"; RADARR_GENERAL_ROOT="${!RADARR_GENERAL_ROOT_VAR:-}"
|
||
RADARR_KIDS_ROOT_VAR="${MY_ID}_RADARR_KIDS_ROOT"; RADARR_KIDS_ROOT="${!RADARR_KIDS_ROOT_VAR:-}"
|
||
RADARR_ANIME_ROOT_VAR="${MY_ID}_RADARR_ANIME_ROOT"; RADARR_ANIME_ROOT="${!RADARR_ANIME_ROOT_VAR:-}"
|
||
RADARR_RECOVERY_VAR="${MY_ID}_RADARR_RECOVERY"; RADARR_RECOVERY="${!RADARR_RECOVERY_VAR:-true}"
|
||
|
||
SLSKD_URL_VAR="${MY_ID}_SLSKD_URL"; SLSKD_URL="${!SLSKD_URL_VAR:-}"
|
||
SLSKD_API_KEY_VAR="${MY_ID}_SLSKD_API_KEY"; SLSKD_API_KEY="${!SLSKD_API_KEY_VAR:-}"
|
||
SLSKD_FAILED_IMPORTS_DIR_VAR="${MY_ID}_SLSKD_FAILED_IMPORTS_DIR"
|
||
SLSKD_FAILED_IMPORTS_DIR="${!SLSKD_FAILED_IMPORTS_DIR_VAR:-}"
|
||
|
||
SABNZBD_URL_VAR="${MY_ID}_SABNZBD_URL"; SABNZBD_URL="${!SABNZBD_URL_VAR:-}"
|
||
SABNZBD_API_KEY_VAR="${MY_ID}_SABNZBD_API_KEY"; SABNZBD_API_KEY="${!SABNZBD_API_KEY_VAR:-}"
|
||
|
||
QBIT_URL_VAR="${MY_ID}_QBIT_URL"; QBIT_URL="${!QBIT_URL_VAR:-}"
|
||
QBIT_USERNAME_VAR="${MY_ID}_QBIT_USERNAME"; QBIT_USERNAME="${!QBIT_USERNAME_VAR:-}"
|
||
QBIT_PASSWORD_VAR="${MY_ID}_QBIT_PASSWORD"; QBIT_PASSWORD="${!QBIT_PASSWORD_VAR:-}"
|
||
|
||
# ── Set array aliases — indexed arrays ────────────────────────────────────
|
||
# Each eval copies the host-specific array into the unprefixed name scripts use
|
||
|
||
_alias_array() {
|
||
local alias_name="$1"
|
||
local source_var="${MY_ID}_${alias_name}"
|
||
eval "${alias_name}=(\"\${${source_var}[@]}\")"
|
||
}
|
||
|
||
_alias_array "DAILY_SYNC_SHARES"
|
||
_alias_array "PERSONAL_SHARES"
|
||
_alias_array "INTERMEDIATE_SYNC_SHARES"
|
||
_alias_array "WEEKLY_SYNC_SHARES"
|
||
_alias_array "CRITICAL_SYNC_SHARES"
|
||
_alias_array "BACKUP_VERIFY_SHARES"
|
||
_alias_array "DAILY_RESTART_CONTAINERS"
|
||
_alias_array "WEEKLY_RESTART_CONTAINERS"
|
||
_alias_array "WATCHDOG_REQUIRED_CONTAINERS"
|
||
_alias_array "WATCHDOG_SCAN_IGNORE"
|
||
_alias_array "NETWORK_CONNECT_CONTAINERS"
|
||
_alias_array "NETWORK_CONNECT_NETWORKS"
|
||
_alias_array "MEDIA_PERMISSION_SHARES"
|
||
_alias_array "ANIME_CLEAN_FOLDERS"
|
||
_alias_array "MEDIA_CLEAN_FOLDERS"
|
||
_alias_array "CERT_MONITOR_DOMAINS"
|
||
_alias_array "SMART_IGNORE_DRIVES"
|
||
_alias_array "ZFS_REPORT_IGNORE_POOLS"
|
||
_alias_array "TRANSCODE_SERVERS"
|
||
_alias_array "DDNS_CONTAINERS"
|
||
_alias_array "PARTNERSHIP_AUTH_WEBUIS"
|
||
_alias_array "PARTNERSHIP_MIRROR_BACKUPS"
|
||
_alias_array "PARTNERSHIP_OWN_CONTAINERS"
|
||
_alias_array "PARTNERSHIP_AUTH_STACK"
|
||
_alias_array "PARTNERSHIP_REPLACE_CONTAINERS"
|
||
_alias_array "PARTNERSHIP_ARR_STACK"
|
||
_alias_array "PARTNERSHIP_ARR_REPLACE_CONTAINERS"
|
||
_alias_array "PARTNERSHIP_SERVICES_STACK"
|
||
_alias_array "PARTNERSHIP_SERVICES_REPLACE_CONTAINERS"
|
||
_alias_array "RW_PAUSE_CONTAINERS"
|
||
_alias_array "RW_STOP_CONTAINERS"
|
||
|
||
# ── Set array aliases — associative arrays ────────────────────────────────
|
||
# Associative arrays cannot be copied with eval — must be rebuilt key by key
|
||
|
||
_alias_assoc() {
|
||
local alias_name="$1"
|
||
local source_name="${MY_ID}_${alias_name}"
|
||
eval "declare -gA ${alias_name}"
|
||
local -n _assoc_src="${source_name}" 2>/dev/null || return 0
|
||
local -n _assoc_dst="${alias_name}"
|
||
local _assoc_key
|
||
for _assoc_key in "${!_assoc_src[@]}"; do
|
||
_assoc_dst["$_assoc_key"]="${_assoc_src[$_assoc_key]}"
|
||
done
|
||
}
|
||
|
||
_alias_assoc "WATCHDOG_CONTAINERS"
|
||
_alias_assoc "WATCHDOG_CONTAINER_URLS"
|
||
_alias_assoc "WATCHDOG_CONTAINER_API_CHECKS"
|
||
_alias_assoc "WATCHDOG_DEPENDENCIES"
|
||
_alias_assoc "WATCHDOG_APPDATA_SIZES"
|
||
|
||
# ── Remote storage path ───────────────────────────────────────────────────
|
||
# Resolved from REMOTE_ID's host*.conf so SSH commands use the correct path
|
||
# even when the remote is a different platform. Falls back to the local
|
||
# platform_storage_path() for unconfigured same-platform pairs.
|
||
local _rsp_var="${REMOTE_ID}_STORAGE_PATH"
|
||
REMOTE_STORAGE_PATH="${!_rsp_var:-$(platform_storage_path)}"
|
||
|
||
# ── Output ────────────────────────────────────────────────────────────────
|
||
log "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME"
|
||
}
|
||
|
||
# Derives owner/mirror roles from PARTNERSHIP_OWNER_HOST — used by every Partnership/*.sh
|
||
# script. Requires detect_hosts() to have already run (needs MY_ID, SSH_KEY).
|
||
# Sets: OWNER_ID MIRROR_ID OWNER MIRROR MIRROR_SSH_KEY OWNER_SSH_KEY AM_OWNER AM_MIRROR
|
||
# Does NOT set any *_STATE_FILE vars — callers that need those set them after calling this,
|
||
# since not every Partnership script needs the same set (transfer/onboard need fewer than
|
||
# manager/offboard).
|
||
partnership_resolve_roles() {
|
||
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
|
||
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
|
||
OWNER="${!OWNER_ID}"
|
||
MIRROR="${!MIRROR_ID}"
|
||
# SSH_KEY (set by detect_hosts) is this server's own private key. The remote accepts it
|
||
# because this server's PUBLIC key was installed there via ssh_setup.sh. With sparse
|
||
# checkout, each server only has its own host{N}.conf — the other server's key path is
|
||
# never available here. Use SSH_KEY for all outbound SSH regardless of mode.
|
||
MIRROR_SSH_KEY="$SSH_KEY"
|
||
OWNER_SSH_KEY="$SSH_KEY"
|
||
|
||
AM_OWNER=false
|
||
AM_MIRROR=false
|
||
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
|
||
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
|
||
}
|
||
|
||
# Read a scalar/array var from a partner's own config via SSH — the partner sources its
|
||
# own load_config.sh + detect_hosts() remotely so the value reflects ITS host*.conf, not
|
||
# ours. Requires $SCRIPT_DIR to be set by the caller (one level above the repo root, same
|
||
# convention every Partnership/*.sh script already uses for its own SCRIPT_DIR).
|
||
# Usage: read_remote_conf_var "$mirror_ip" "VAR_NAME"
|
||
# read_remote_conf_array "$mirror_ip" "ARRAY_NAME" # one element per line
|
||
read_remote_conf_var() {
|
||
local mirror_ip="$1" var_name="$2"
|
||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
|
||
detect_hosts 2>/dev/null
|
||
printf '%s' \"\${${var_name}:-}\"" 2>/dev/null
|
||
}
|
||
|
||
read_remote_conf_array() {
|
||
local mirror_ip="$1" var_name="$2"
|
||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
|
||
detect_hosts 2>/dev/null
|
||
printf '%s\n' \"\${${var_name}[@]:-}\"" 2>/dev/null
|
||
}
|
||
|
||
# Probes a remote host's SCRIPTS_DIR (in case it differs from ours — e.g. one host runs
|
||
# internal storage mode, the other flash), falling back to our own $SCRIPTS_DIR if the
|
||
# probe fails or the remote isn't reachable. strict_host_key defaults to "yes" (matches
|
||
# existing behavior everywhere except partnership_transfer.sh, which passes "no" since
|
||
# it may be contacting a mirror for the first time during an ownership transfer).
|
||
# Usage: resolve_remote_scripts_dir "$ip" ["$ssh_key"] ["no"|"yes"]
|
||
resolve_remote_scripts_dir() {
|
||
local ip="$1" ssh_key="${2:-$SSH_KEY}" strict_host_key="${3:-yes}"
|
||
local probe_cmd result
|
||
local -a opts=(-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes)
|
||
[[ "$strict_host_key" == "no" ]] && opts+=(-o StrictHostKeyChecking=no)
|
||
probe_cmd=$(platform_scripts_dir_probe_cmd)
|
||
result=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" "${opts[@]}" root@"$ip" "$probe_cmd" \
|
||
2>/dev/null | tr -d '[:space:]')
|
||
echo "${result:-$SCRIPTS_DIR}"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── MULTI-NODE DISCOVERY ──────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Scans HOST1..HOST8 for every configured node other than this one. Requires detect_hosts()
|
||
# to have already run (needs MY_ID). Sets the global REMOTE_NODES array — empty if none found.
|
||
discover_remote_nodes() {
|
||
REMOTE_NODES=()
|
||
local _hv
|
||
for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
||
[[ "$_hv" == "$MY_ID" ]] && continue
|
||
[[ -z "${!_hv:-}" ]] && continue
|
||
REMOTE_NODES+=("$_hv")
|
||
done
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── REMOTE IP RESOLUTION ──────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Resolves the Tailscale IPv4 address of the remote server.
|
||
# Sets REMOTE_SERVER used by all subsequent SSH and rsync calls.
|
||
# Exits if resolution fails — Tailscale may be down or peer offline.
|
||
|
||
resolve_remote_ip() {
|
||
log "Resolving remote IP for $REMOTE_SERVER_NAME..."
|
||
local ts_name="${REMOTE_SERVER_NAME,,}"
|
||
local attempts=3
|
||
|
||
for ((i=1; i<=attempts; i++)); do
|
||
# Direct lookup — works when MagicDNS short-name resolution is active
|
||
REMOTE_SERVER=$(tailscale ip -4 "$ts_name" 2>/dev/null)
|
||
|
||
# Fallback — parse tailscale status for FQDN entries (e.g. hostname.tailXXXX.ts.net)
|
||
if [[ -z "$REMOTE_SERVER" ]]; then
|
||
REMOTE_SERVER=$(tailscale status 2>/dev/null | awk -v name="$ts_name" '$2 ~ "^" name { print $1; exit }')
|
||
fi
|
||
|
||
[[ -n "$REMOTE_SERVER" ]] && break
|
||
[[ $i -lt $attempts ]] && { warn "Tailscale resolution attempt $i/$attempts failed — retrying in 5s..."; sleep 5; }
|
||
done
|
||
|
||
if [[ -z "$REMOTE_SERVER" ]]; then
|
||
error "Failed to resolve Tailscale IP for $REMOTE_SERVER_NAME after $attempts attempts"
|
||
error "Check: tailscale status | grep $ts_name"
|
||
exit 1
|
||
fi
|
||
info "$ICON_NET Remote IP: $REMOTE_SERVER"
|
||
}
|
||
|
||
# Resolve any hostname to a Tailscale IPv4 — tries direct lookup, falls back to status parse.
|
||
# Usage: ip=$(resolve_tailscale_ip "hostname") — returns empty string on failure.
|
||
resolve_tailscale_ip() {
|
||
local hostname="${1,,}"
|
||
local ip
|
||
ip=$(tailscale ip -4 "$hostname" 2>/dev/null)
|
||
if [[ -n "$ip" ]]; then
|
||
echo "$ip"
|
||
return
|
||
fi
|
||
# Fallback: unambiguous prefix match against tailscale status (either direction) — handles
|
||
# Unraid's 15-char NetBIOS hostname truncation vs. a longer name recorded in master.conf.
|
||
# Only accept the match when exactly one peer could qualify; never guess between multiple
|
||
# candidates that happen to share a prefix (e.g. server1/server10).
|
||
local matches count
|
||
matches=$(tailscale status 2>/dev/null | awk -v name="$hostname" '
|
||
{ split(tolower($2), parts, "."); host = parts[1];
|
||
if (index(host, name) == 1 || index(name, host) == 1) print $1 }')
|
||
count=$(echo "$matches" | grep -c .)
|
||
[[ "$count" -eq 1 ]] && echo "$matches"
|
||
}
|
||
|
||
# Strips the "unraid-" prefix (case-insensitive) and title-cases what remains.
|
||
# Usage: derive_short_name "unRAID-Gmer4Lfe" → "Gmer4lfe"
|
||
derive_short_name() {
|
||
local hostname="$1"
|
||
local short="${hostname,,}"
|
||
[[ "$short" == unraid-* ]] && short="${short:7}"
|
||
echo "${short^}"
|
||
}
|
||
|
||
# True if the given host*.conf basename belongs to this host (case-insensitive).
|
||
# Usage: is_own_conf_file "$(basename "$conf")"
|
||
is_own_conf_file() {
|
||
[[ "${1,,}" == "${MY_ID,,}.conf" ]]
|
||
}
|
||
|
||
# Sets key=value in a flat state file (setup.db style) — updates in place if the
|
||
# key exists, appends if not. No indentation handling — for master.conf use
|
||
# partnership_manager.sh's update_master_conf() instead.
|
||
# Usage: set_state_var "$state_file" "$key" "$value"
|
||
set_state_var() {
|
||
local file="$1" key="$2" value="$3"
|
||
if grep -q "^${key}=" "$file" 2>/dev/null; then
|
||
sed -i "s|^${key}=.*|${key}=${value}|" "$file"
|
||
else
|
||
echo "${key}=${value}" >> "$file"
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── CONNECTIVITY CHECKS ───────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# Fatal connectivity check — used by rsync and other scripts that must abort if unreachable.
|
||
# For fallback use ping_remote() which returns status without exiting.
|
||
check_connectivity() {
|
||
log "Checking connectivity to $REMOTE_SERVER..."
|
||
if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then
|
||
error "$ICON_PING Remote $REMOTE_SERVER ($REMOTE_SERVER_NAME) is unreachable"
|
||
info "Hint: tailscale status | grep $REMOTE_SERVER_NAME"
|
||
exit 1
|
||
fi
|
||
info "$ICON_PING $REMOTE_SERVER_NAME is reachable"
|
||
}
|
||
|
||
# Non-fatal ping — used by fallback.sh which handles its own state machine.
|
||
# Returns 0 if reachable, 1 if not — does NOT exit.
|
||
ping_remote() {
|
||
ping -c2 -W3 "$REMOTE_SERVER" &>/dev/null
|
||
}
|
||
|
||
# Non-fatal external connectivity check — used by fallback.sh.
|
||
# Returns 0 if internet reachable, 1 if not — does NOT exit.
|
||
ping_internet() {
|
||
ping -c2 -W3 "${EXTERNAL_IP:-8.8.8.8}" &>/dev/null
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── RSYNC GATE CHECK ──────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Two-tier rsync enable/disable system.
|
||
# Tier 1: RSYNC_ENABLED — global gate, overrides everything
|
||
# Tier 2: orchestrator-specific flag — fine-grained control per window
|
||
#
|
||
# Usage:
|
||
# check_rsync_enabled "CRITICAL" ← checks RSYNC_ENABLED + CRITICAL_RSYNC_ENABLED
|
||
# check_rsync_enabled "INTERMEDIATE" ← checks RSYNC_ENABLED + INTERMEDIATE_RSYNC_ENABLED
|
||
# check_rsync_enabled "DAILY" ← checks RSYNC_ENABLED + DAILY_RSYNC_ENABLED
|
||
# check_rsync_enabled "WEEKLY" ← checks RSYNC_ENABLED + WEEKLY_RSYNC_ENABLED
|
||
# check_rsync_enabled "FALLBACK" ← checks RSYNC_ENABLED + FALLBACK_RSYNC_ENABLED
|
||
# check_rsync_enabled ← checks RSYNC_ENABLED only (direct rsync.sh call)
|
||
#
|
||
# Returns: 0 = enabled, proceed | 1 = disabled, skip cleanly
|
||
|
||
check_rsync_enabled() {
|
||
local orchestrator="${1:-}"
|
||
|
||
# Tier 1 — global gate
|
||
if [[ "${RSYNC_ENABLED:-true}" == false ]]; then
|
||
warn "RSYNC_ENABLED=false — rsync globally disabled, skipping all syncs"
|
||
return 1
|
||
fi
|
||
|
||
# Tier 2 — per-orchestrator
|
||
if [[ -n "$orchestrator" ]]; then
|
||
local var_name="${orchestrator}_RSYNC_ENABLED"
|
||
local var_value="${!var_name:-true}"
|
||
if [[ "$var_value" == false ]]; then
|
||
info "${var_name}=false — rsync disabled for $orchestrator orchestrator"
|
||
info "All other $orchestrator jobs will still run normally"
|
||
return 1
|
||
fi
|
||
fi
|
||
|
||
return 0
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── PLATFORM SERVICE STATE ────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Check whether Docker and VM Manager are enabled on this host.
|
||
# Delegates to platform_is_service_enabled() from the platform adapter.
|
||
|
||
is_docker_enabled() {
|
||
platform_is_service_enabled docker
|
||
}
|
||
|
||
is_vm_manager_enabled() {
|
||
platform_is_service_enabled libvirt
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── LOCAL HEALTH CHECKS ───────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# ==============================================================================================
|
||
# ── REMOTE HEALTH CHECKS ─────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# Verifies remote storage is mounted via SSH.
|
||
# Non-fatal — returns status. Used before handback rsync.
|
||
# Syncing to remote with no array fills rootfs rapidly.
|
||
check_remote_array() {
|
||
log "Checking remote array on $REMOTE_SERVER_NAME..."
|
||
local _spath result
|
||
_spath="$REMOTE_STORAGE_PATH"
|
||
result=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
||
"mountpoint -q '$_spath' && echo yes || echo no" 2>/dev/null)
|
||
if [[ "$result" != "yes" ]]; then
|
||
error "$ICON_DISK Remote array not started on $REMOTE_SERVER_NAME"
|
||
return 1
|
||
fi
|
||
log "Remote array is healthy"
|
||
return 0
|
||
}
|
||
|
||
# Aborts if remote rootfs (/) usage is at or above ROOTFS_WARN threshold.
|
||
# Fatal — exits the calling script.
|
||
# When remote array is down rsync writes land on rootfs and fill it rapidly.
|
||
check_remote_rootfs() {
|
||
log "Checking remote rootfs usage..."
|
||
local remote_usage
|
||
remote_usage=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null)
|
||
if [[ -z "$remote_usage" ]]; then
|
||
error "Could not retrieve rootfs usage from $REMOTE_SERVER_NAME"
|
||
exit 1
|
||
fi
|
||
if [[ "$remote_usage" -ge "${ROOTFS_WARN:-75}" ]]; then
|
||
error "$ICON_HEALTH Remote rootfs ${remote_usage}% — threshold ${ROOTFS_WARN:-75}%"
|
||
exit 1
|
||
fi
|
||
info "$ICON_HEALTH Remote rootfs: ${remote_usage}% (threshold: ${ROOTFS_WARN:-75}%)"
|
||
}
|
||
|
||
# Verifies target directory exists and is not empty on remote.
|
||
# Fatal — exits the calling script.
|
||
# Usage: check_remote_share "/mnt/user/Movies"
|
||
check_remote_share() {
|
||
local dir="$1"
|
||
log "Checking remote share: $dir..."
|
||
|
||
local share_exists
|
||
share_exists=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null)
|
||
if [[ "$share_exists" != "yes" ]]; then
|
||
error "$ICON_HEALTH Remote share does not exist: $dir"
|
||
exit 1
|
||
fi
|
||
|
||
local share_empty
|
||
share_empty=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null)
|
||
if [[ "$share_empty" == "yes" ]]; then
|
||
warn "$ICON_HEALTH Remote share exists but is empty: $dir — aborting to protect data"
|
||
exit 1
|
||
fi
|
||
|
||
info "$ICON_HEALTH Remote share verified: $dir"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── DISK TEMPERATURE CHECKS ───────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# Returns 0 (true) if device is non-rotational (SSD/NVMe), 1 (false) if HDD.
|
||
# Checks /sys/block/<device>/queue/rotational — 0=SSD, 1=HDD.
|
||
is_ssd() {
|
||
local drive="$1"
|
||
local base rotational
|
||
base=$(basename "$drive")
|
||
rotational="/sys/block/$base/queue/rotational"
|
||
[[ -f "$rotational" ]] && [[ "$(cat "$rotational" 2>/dev/null)" == "0" ]] && return 0
|
||
# NVMe is always SSD — some NVMe controllers don't expose rotational correctly
|
||
[[ "$drive" == *nvme* ]] && return 0
|
||
return 1
|
||
}
|
||
|
||
# Sets globals: UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX
|
||
# Falls back to master.conf values (SMART_TEMP_WARN/CRIT) if platform returns defaults.
|
||
get_unraid_temp_thresholds() {
|
||
read -r UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX \
|
||
<<< "$(platform_get_temp_thresholds)"
|
||
UNRAID_DISK_HOT="${UNRAID_DISK_HOT:-${SMART_TEMP_WARN:-45}}"
|
||
UNRAID_DISK_MAX="${UNRAID_DISK_MAX:-${SMART_TEMP_CRIT:-55}}"
|
||
UNRAID_SSD_HOT="${UNRAID_SSD_HOT:-60}"
|
||
UNRAID_SSD_MAX="${UNRAID_SSD_MAX:-70}"
|
||
}
|
||
|
||
# Checks local disk temperatures before rsync.
|
||
# Fetches a domain's TLS cert expiry via openssl s_client + x509 and computes days
|
||
# remaining. Pure data — no logging, no thresholds — callers classify warn/crit
|
||
# themselves against their own CERT_WARN_DAYS/CERT_CRIT_DAYS and log however they like.
|
||
#
|
||
# Sets _CERT_DAYS, _CERT_EXPIRY (YYYY-MM-DD) on success. Sets _CERT_EXPIRY_RAW to the
|
||
# unparsed openssl date string only when parsing failed (empty when unreachable) so a
|
||
# caller that wants a more specific error message can distinguish the two failure modes.
|
||
#
|
||
# Usage: check_cert_expiry "$domain" [port=443] [timeout=$CERT_TIMEOUT or 10]
|
||
# Returns: 0 = fetched and parsed | 1 = unreachable or unparseable
|
||
check_cert_expiry() {
|
||
local domain="$1" port="${2:-443}" cert_timeout="${3:-${CERT_TIMEOUT:-10}}"
|
||
_CERT_DAYS=""
|
||
_CERT_EXPIRY=""
|
||
_CERT_EXPIRY_RAW=""
|
||
local expiry_str
|
||
expiry_str=$(echo | timeout "$cert_timeout" openssl s_client \
|
||
-connect "${domain}:${port}" -servername "$domain" \
|
||
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
||
[[ -z "$expiry_str" ]] && return 1
|
||
local expiry_epoch
|
||
expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
|
||
if [[ -z "$expiry_epoch" ]]; then
|
||
_CERT_EXPIRY_RAW="$expiry_str"
|
||
return 1
|
||
fi
|
||
_CERT_DAYS=$(( (expiry_epoch - $(date +%s)) / 86400 ))
|
||
_CERT_EXPIRY=$(date -d "$expiry_str" '+%Y-%m-%d' 2>/dev/null)
|
||
return 0
|
||
}
|
||
|
||
# Reads temps and rotational flag from /var/local/emhttp/disks.ini.
|
||
# Uses unRAID's own thresholds from dynamix.cfg via get_unraid_temp_thresholds().
|
||
#
|
||
# Returns:
|
||
# 0 = all temps OK — proceed
|
||
# 1 = warn threshold exceeded — skip this profile only
|
||
# 2 = critical threshold exceeded — abort all remaining profiles
|
||
#
|
||
# Sets TEMP_CHECK_RESULT with human readable summary for logging.
|
||
check_local_disk_temps() {
|
||
get_unraid_temp_thresholds
|
||
|
||
local disks_ini_content
|
||
if ! disks_ini_content=$(platform_get_disk_states); then
|
||
warn "disk state unavailable — skipping temp check"
|
||
TEMP_CHECK_RESULT="temp check skipped (disk state unavailable)"
|
||
return 0
|
||
fi
|
||
|
||
local worst_result=0
|
||
local hot_drives=()
|
||
local crit_drives=()
|
||
local current_name=""
|
||
local current_device=""
|
||
local current_rotational=""
|
||
local current_temp=""
|
||
|
||
check_drive() {
|
||
[[ -z "$current_name" ]] || [[ -z "$current_temp" ]] && return
|
||
[[ "$current_temp" -eq 0 ]] && return # spun down — skip
|
||
|
||
local warn_thresh crit_thresh
|
||
if [[ "$current_rotational" == "0" ]]; then
|
||
warn_thresh="$UNRAID_SSD_HOT"
|
||
crit_thresh="$UNRAID_SSD_MAX"
|
||
else
|
||
warn_thresh="$UNRAID_DISK_HOT"
|
||
crit_thresh="$UNRAID_DISK_MAX"
|
||
fi
|
||
|
||
if [[ "$current_temp" -ge "$crit_thresh" ]]; then
|
||
crit_drives+=("${current_name}(${current_device}):${current_temp}°C≥${crit_thresh}°C")
|
||
[[ $worst_result -lt 2 ]] && worst_result=2
|
||
elif [[ "$current_temp" -ge "$warn_thresh" ]]; then
|
||
hot_drives+=("${current_name}(${current_device}):${current_temp}°C≥${warn_thresh}°C")
|
||
[[ $worst_result -lt 1 ]] && worst_result=1
|
||
fi
|
||
}
|
||
|
||
while IFS= read -r ini_line; do
|
||
if echo "$ini_line" | grep -qE '^\["(disk[0-9]+|parity[0-9]?|cache[0-9]?)"\]'; then
|
||
check_drive
|
||
current_name=$(echo "$ini_line" | grep -o '"[^"]*"' | head -1 | tr -d '"')
|
||
current_device=""
|
||
current_rotational="1" # default HDD
|
||
current_temp=""
|
||
elif echo "$ini_line" | grep -q '^device='; then
|
||
current_device=$(echo "$ini_line" | cut -d= -f2 | tr -d '"')
|
||
elif echo "$ini_line" | grep -q '^rotational='; then
|
||
current_rotational=$(echo "$ini_line" | cut -d= -f2 | tr -d '"')
|
||
elif echo "$ini_line" | grep -q '^temp='; then
|
||
current_temp=$(echo "$ini_line" | cut -d= -f2 | tr -d '"')
|
||
current_temp="${current_temp//[^0-9]/}"
|
||
current_temp="${current_temp:-0}"
|
||
fi
|
||
done <<< "$disks_ini_content"
|
||
check_drive # process last drive
|
||
|
||
if [[ ${#crit_drives[@]} -gt 0 ]]; then
|
||
TEMP_CHECK_RESULT="CRITICAL temps: ${crit_drives[*]}"
|
||
error "Drive temp CRITICAL — aborting all remaining syncs: ${crit_drives[*]}"
|
||
notify "Rsync aborted on $(hostname) — drive temp CRITICAL: ${crit_drives[*]}" "Rsync Temp Check" "warning"
|
||
return 2
|
||
elif [[ ${#hot_drives[@]} -gt 0 ]]; then
|
||
TEMP_CHECK_RESULT="HOT drives: ${hot_drives[*]}"
|
||
warn "Drive temp WARNING — skipping this profile: ${hot_drives[*]}"
|
||
notify "Rsync profile skipped on $(hostname) — drive temp WARNING: ${hot_drives[*]}" "Rsync Temp Check" "normal"
|
||
return 1
|
||
else
|
||
TEMP_CHECK_RESULT="all normal (HDD warn:${UNRAID_DISK_HOT}°C crit:${UNRAID_DISK_MAX}°C SSD warn:${UNRAID_SSD_HOT}°C crit:${UNRAID_SSD_MAX}°C)"
|
||
return 0
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── REMOTE DISK VERIFICATION ──────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Verifies all disks backing a share are healthy on the remote server.
|
||
# Auto-detects filesystem type from disks.ini — handles XFS, ZFS, and cache pools.
|
||
# No configuration required — discovers disk layout at runtime.
|
||
#
|
||
# Three-tier detection handles all share locations:
|
||
# Tier 1: Array disk paths — /mnt/disk*/sharename (XFS or ZFS per-disk)
|
||
# Tier 2: ZFS standalone pools — finds nested paths up to 2 levels deep
|
||
# catches /mnt/cache/appdata-Fallback/Critical-Data correctly
|
||
# Tier 3: shfs fallback — if share exists under /mnt/user, resolve backing pool
|
||
# or verify shfs itself is mounted if pool can't be determined
|
||
#
|
||
# Fatal — exits the calling script if any disk is offline.
|
||
# Usage: check_remote_disks "/mnt/user/Movies"
|
||
|
||
check_remote_disks() {
|
||
local dir="$1"
|
||
local share_name
|
||
share_name=$(basename "$dir")
|
||
|
||
info "$ICON_DISK Checking disks backing $share_name on $REMOTE_SERVER_NAME..."
|
||
|
||
# Read remote disk state for fsType mapping
|
||
local disks_ini_content _disk_states_path
|
||
_disk_states_path=$(platform_disk_states_path)
|
||
disks_ini_content=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"cat '$_disk_states_path' 2>/dev/null" 2>/dev/null)
|
||
|
||
if [[ -z "$disks_ini_content" ]]; then
|
||
error "$ICON_DISK Cannot read disk state from $REMOTE_SERVER_NAME"
|
||
exit 1
|
||
fi
|
||
|
||
# Tier 1 — array disk paths (/mnt/disk*/sharename)
|
||
local backing_disks
|
||
backing_disks=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"ls -d /mnt/disk*/$share_name 2>/dev/null | awk -F/ '{print \$3}'" 2>/dev/null)
|
||
|
||
# Tier 2 — ZFS standalone pools, including nested paths
|
||
local zfs_pool_paths
|
||
zfs_pool_paths=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"zpool list -H -o name 2>/dev/null | while read pool; do
|
||
[[ -d \"/mnt/\${pool}/$share_name\" ]] && echo \"\$pool\" && continue
|
||
find \"/mnt/\${pool}\" -maxdepth 2 -name '$share_name' -type d 2>/dev/null | \
|
||
grep -q . && echo \"\$pool\"
|
||
done | sort -u" 2>/dev/null)
|
||
|
||
# Tier 3 — storage root fallback
|
||
local _spath="$REMOTE_STORAGE_PATH"
|
||
if [[ -z "$backing_disks" ]] && [[ -z "$zfs_pool_paths" ]]; then
|
||
local on_user
|
||
on_user=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"find '$_spath' -maxdepth 3 -name '$share_name' -type d 2>/dev/null | head -1" \
|
||
2>/dev/null)
|
||
if [[ -n "$on_user" ]]; then
|
||
local actual_path
|
||
actual_path=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"readlink -f '$on_user' 2>/dev/null || echo '$on_user'" 2>/dev/null)
|
||
local pool_name
|
||
pool_name=$(echo "$actual_path" | awk -F/ '{print $3}')
|
||
if [[ -n "$pool_name" ]] && [[ "$pool_name" != "user" ]]; then
|
||
zfs_pool_paths="$pool_name"
|
||
info "$ICON_DISK $share_name found at $actual_path on $REMOTE_SERVER_NAME"
|
||
else
|
||
# Can't determine pool — verify storage root itself is mounted
|
||
local shfs_ok
|
||
shfs_ok=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"mountpoint -q '$_spath' && echo yes || echo no" 2>/dev/null)
|
||
if [[ "$shfs_ok" == "yes" ]]; then
|
||
success "All disks backing $share_name are online ✅ (via storage root)"
|
||
return 0
|
||
fi
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
if [[ -z "$backing_disks" ]] && [[ -z "$zfs_pool_paths" ]]; then
|
||
error "$ICON_DISK No disks found backing $share_name on $REMOTE_SERVER_NAME"
|
||
exit 1
|
||
fi
|
||
|
||
local all_ok=true
|
||
|
||
# Check array disks (XFS or ZFS single-disk-in-array)
|
||
if [[ -n "$backing_disks" ]]; then
|
||
while IFS= read -r disk_name; do
|
||
[[ -z "$disk_name" ]] && continue
|
||
|
||
local fs_type
|
||
fs_type=$(echo "$disks_ini_content" | awk -F= -v disk="$disk_name" '
|
||
/^\["'"'"'?/ { current=substr($0,3,length($0)-4) }
|
||
current==disk && /^fsType=/ { print $2; exit }
|
||
' | tr -d '"')
|
||
fs_type="${fs_type:-xfs}"
|
||
|
||
if [[ "$fs_type" == "zfs" ]]; then
|
||
local pool_health
|
||
pool_health=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"zpool list -H -o health '$disk_name' 2>/dev/null" 2>/dev/null)
|
||
if [[ "$pool_health" == "ONLINE" ]]; then
|
||
info "$ICON_DISK $disk_name (ZFS) $ICON_RUNNING — $share_name ONLINE"
|
||
else
|
||
error "$ICON_DISK $disk_name (ZFS) $ICON_STOPPED — pool ${pool_health:-offline}"
|
||
all_ok=false
|
||
fi
|
||
else
|
||
local mounted
|
||
mounted=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"mountpoint -q '/mnt/$disk_name' && echo yes || echo no" 2>/dev/null)
|
||
if [[ "$mounted" == "yes" ]]; then
|
||
info "$ICON_DISK $disk_name (XFS) $ICON_RUNNING — $share_name present"
|
||
else
|
||
error "$ICON_DISK $disk_name (XFS) $ICON_STOPPED — not mounted"
|
||
all_ok=false
|
||
fi
|
||
fi
|
||
done <<< "$backing_disks"
|
||
fi
|
||
|
||
# Check ZFS standalone pools (cache, gaming, media-servers etc.)
|
||
if [[ -n "$zfs_pool_paths" ]]; then
|
||
while IFS= read -r pool_name; do
|
||
[[ -z "$pool_name" ]] && continue
|
||
local pool_health
|
||
pool_health=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"zpool list -H -o health '$pool_name' 2>/dev/null" 2>/dev/null)
|
||
if [[ "$pool_health" == "ONLINE" ]]; then
|
||
info "$ICON_DISK $pool_name (ZFS pool) $ICON_RUNNING — $share_name ONLINE"
|
||
else
|
||
error "$ICON_DISK $pool_name (ZFS pool) $ICON_STOPPED — pool ${pool_health:-offline}"
|
||
all_ok=false
|
||
fi
|
||
done <<< "$zfs_pool_paths"
|
||
fi
|
||
|
||
if [[ "$all_ok" == false ]]; then
|
||
error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME"
|
||
exit 1
|
||
fi
|
||
success "All disks backing $share_name are online ✅"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── CONTAINER MANAGEMENT ──────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Four functions — stop/start on both local and remote.
|
||
# State tracking: only containers that WERE running get restarted.
|
||
# Was stopped → stays stopped. Was running → gets restarted. ✅
|
||
|
||
# Tracks which remote containers were running before stop — used by start_containers()
|
||
RUNNING_CONTAINERS=()
|
||
|
||
# ── Docker command helpers (used by docker_daily_restart, docker_weekly_restart, docker_update) ──
|
||
|
||
# Default timeouts — scripts can override before calling.
|
||
DOCKER_TIMEOUT=${DOCKER_TIMEOUT:-30} # seconds before docker command is killed
|
||
RESTART_VERIFY_WAIT=${RESTART_VERIFY_WAIT:-5} # seconds to wait before checking state post-restart
|
||
|
||
# Run a docker command with a hard timeout. Returns 124 on timeout, else docker's exit code.
|
||
docker_cmd() {
|
||
timeout "$DOCKER_TIMEOUT" "$@"
|
||
local exit_code=$?
|
||
if [[ "$exit_code" -eq 124 ]]; then
|
||
error "Docker command timed out after ${DOCKER_TIMEOUT}s: $*"
|
||
return 1
|
||
fi
|
||
return "$exit_code"
|
||
}
|
||
|
||
# Verify a container is still running after a restart.
|
||
# Waits RESTART_VERIFY_WAIT seconds to let it settle, then checks State.Running.
|
||
verify_running() {
|
||
local container="$1"
|
||
sleep "$RESTART_VERIFY_WAIT"
|
||
local state
|
||
state=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||
if [[ "$state" != "true" ]]; then
|
||
error "$container failed to stay running after restart — may have crashed"
|
||
return 1
|
||
fi
|
||
return 0
|
||
}
|
||
|
||
# Retry a docker command up to RETRY_COUNT times, sleeping SLEEP seconds between attempts.
|
||
retry_docker() {
|
||
local attempt=1
|
||
while [[ "$attempt" -le "$RETRY_COUNT" ]]; do
|
||
log "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
|
||
if docker_cmd "$@"; then
|
||
log "Succeeded on attempt $attempt"
|
||
return 0
|
||
else
|
||
warn "Attempt $attempt failed"
|
||
(( attempt++ ))
|
||
[[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP"
|
||
fi
|
||
done
|
||
error "Command failed after $RETRY_COUNT attempts: $*"
|
||
return 1
|
||
}
|
||
|
||
# ── Emby API helper ───────────────────────────────────────────────────────────
|
||
# General Emby REST call. Uses EMBY_URL and EMBY_API_KEY (aliased by detect_hosts).
|
||
# Usage: emby_api <endpoint> [timeout_seconds]
|
||
# Prints response body on 200; returns 1 on HTTP error or curl failure.
|
||
emby_api() {
|
||
local endpoint="$1" max_time="${2:-30}"
|
||
local response http_code body
|
||
response=$(curl -sf --max-time "$max_time" \
|
||
-H "X-Emby-Token: $EMBY_API_KEY" \
|
||
-w "\n%{http_code}" \
|
||
"${EMBY_URL}/${endpoint}" 2>/dev/null)
|
||
http_code=$(echo "$response" | tail -1)
|
||
body=$(echo "$response" | head -n -1)
|
||
[[ "$http_code" != "200" ]] && { error "Emby API HTTP $http_code: $endpoint"; return 1; }
|
||
echo "$body"
|
||
}
|
||
|
||
# ── Media Server Detection ────────────────────────────────────────────────────
|
||
# Picks whichever media server is actually running right now, so callers don't have to
|
||
# hardcode Emby or Jellyfin. Emby is preferred when both are up (it's the primary player on
|
||
# every host; Jellyfin is mainly kept around for its working ffprobe binary — see
|
||
# HOST*_FFPROBE_CONTAINER). Either container being merely *configured* in host*.conf doesn't
|
||
# count — this checks Docker's actual running state, so a host with Jellyfin installed but
|
||
# stopped correctly falls through to Emby (or to nothing, if neither is up).
|
||
#
|
||
# Requires detect_hosts() to have already run (needs EMBY_*/JELLYFIN_* aliases).
|
||
# Sets: MEDIA_SERVER ("Emby"|"Jellyfin"|""), MEDIA_SERVER_URL, MEDIA_SERVER_API_KEY.
|
||
# Returns 1 and leaves MEDIA_SERVER="" if neither configured container is running.
|
||
# Usage: detect_media_server [docker_timeout_seconds]
|
||
detect_media_server() {
|
||
local docker_timeout="${1:-10}"
|
||
MEDIA_SERVER="" MEDIA_SERVER_URL="" MEDIA_SERVER_API_KEY=""
|
||
|
||
local emby_up=false jellyfin_up=false
|
||
|
||
if [[ -n "${EMBY_CONTAINER:-}" ]]; then
|
||
[[ "$(timeout "$docker_timeout" docker inspect -f '{{.State.Running}}' \
|
||
"$EMBY_CONTAINER" 2>/dev/null)" == "true" ]] && emby_up=true
|
||
fi
|
||
if [[ -n "${JELLYFIN_CONTAINER:-}" ]]; then
|
||
[[ "$(timeout "$docker_timeout" docker inspect -f '{{.State.Running}}' \
|
||
"$JELLYFIN_CONTAINER" 2>/dev/null)" == "true" ]] && jellyfin_up=true
|
||
fi
|
||
|
||
if [[ "$emby_up" == true && "$jellyfin_up" == true ]]; then
|
||
log "Both Emby and Jellyfin are running — using Emby (primary)"
|
||
elif [[ "$emby_up" == true ]]; then
|
||
log "Emby is running, Jellyfin is not — using Emby"
|
||
elif [[ "$jellyfin_up" == true ]]; then
|
||
log "Jellyfin is running, Emby is not — using Jellyfin"
|
||
else
|
||
warn "Neither Emby nor Jellyfin is running — no media server available"
|
||
return 1
|
||
fi
|
||
|
||
if [[ "$emby_up" == true ]]; then
|
||
MEDIA_SERVER="Emby"
|
||
MEDIA_SERVER_URL="$EMBY_URL"
|
||
MEDIA_SERVER_API_KEY="$EMBY_API_KEY"
|
||
else
|
||
MEDIA_SERVER="Jellyfin"
|
||
MEDIA_SERVER_URL="$JELLYFIN_URL"
|
||
MEDIA_SERVER_API_KEY="$JELLYFIN_API_KEY"
|
||
fi
|
||
}
|
||
|
||
# General REST call against whichever server detect_media_server() picked. Jellyfin accepts
|
||
# the same X-Emby-Token header Emby uses (inherited from its Emby-fork lineage — confirmed
|
||
# live 2026-07-19), so one call shape covers both backends.
|
||
# Usage: detect_media_server && media_server_api <endpoint> [timeout_seconds]
|
||
# Prints response body on 200; returns 1 on HTTP error, curl failure, or no server detected.
|
||
media_server_api() {
|
||
local endpoint="$1" max_time="${2:-30}"
|
||
if [[ -z "${MEDIA_SERVER:-}" ]]; then
|
||
error "media_server_api called with no MEDIA_SERVER set — call detect_media_server first"
|
||
return 1
|
||
fi
|
||
local response http_code body
|
||
response=$(curl -sf --max-time "$max_time" \
|
||
-H "X-Emby-Token: $MEDIA_SERVER_API_KEY" \
|
||
-w "\n%{http_code}" \
|
||
"${MEDIA_SERVER_URL}/${endpoint}" 2>/dev/null)
|
||
http_code=$(echo "$response" | tail -1)
|
||
body=$(echo "$response" | head -n -1)
|
||
[[ "$http_code" != "200" ]] && { error "$MEDIA_SERVER API HTTP $http_code: $endpoint"; return 1; }
|
||
echo "$body"
|
||
}
|
||
|
||
# Stop containers on the REMOTE server via SSH.
|
||
# Reads CRITICAL_CONTAINER_NAMES — set from profile arrays by rsync.sh.
|
||
# Tracks running state in RUNNING_CONTAINERS for restart after sync.
|
||
stop_containers() {
|
||
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \
|
||
[[ "${CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then
|
||
log "No remote containers configured for this profile, skipping remote stop."
|
||
return
|
||
fi
|
||
echo "━━━ $ICON_STOP $ICON_CONTAINERS Containers ━━━"
|
||
info "Stopping remote containers..."
|
||
RUNNING_CONTAINERS=()
|
||
for c in "${CRITICAL_CONTAINER_NAMES[@]}"; do
|
||
[[ -z "$c" ]] && continue
|
||
local status
|
||
status=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||
"docker inspect -f '{{.State.Running}}' $c 2>/dev/null" 2>/dev/null || echo "unknown")
|
||
if [[ "$status" == "true" ]]; then
|
||
echo "$ICON_STOP Stopping remote $c..."
|
||
RUNNING_CONTAINERS+=("$c")
|
||
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $c" >/dev/null; then
|
||
echo "$ICON_STOPPED $c stopped"
|
||
else
|
||
error "Failed to stop remote $c"
|
||
fi
|
||
elif [[ "$status" == "false" ]]; then
|
||
echo "$ICON_NOT_RUNNING $c is not running — skipping"
|
||
else
|
||
log "$c not found on remote — skipping"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# Tracks which local containers were running before stop — used by start_local_containers()
|
||
LOCAL_RUNNING_CONTAINERS=()
|
||
|
||
# Stop containers on the LOCAL server.
|
||
# Reads LOCAL_CRITICAL_CONTAINER_NAMES — set from profile arrays by rsync.sh.
|
||
# Flushes local databases cleanly before pushing data to remote.
|
||
stop_local_containers() {
|
||
if [[ ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \
|
||
[[ "${LOCAL_CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then
|
||
log "No local containers configured for this profile, skipping local stop."
|
||
return
|
||
fi
|
||
info "Stopping local containers..."
|
||
LOCAL_RUNNING_CONTAINERS=()
|
||
for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]}"; do
|
||
[[ -z "$c" ]] && continue
|
||
local status
|
||
status=$(docker inspect -f '{{.State.Running}}' "$c" 2>/dev/null || echo "unknown")
|
||
if [[ "$status" == "true" ]]; then
|
||
echo "$ICON_STOP Stopping local $c..."
|
||
LOCAL_RUNNING_CONTAINERS+=("$c")
|
||
if docker stop "$c" >/dev/null; then
|
||
echo "$ICON_STOPPED $c stopped"
|
||
else
|
||
error "Failed to stop local $c"
|
||
fi
|
||
elif [[ "$status" == "false" ]]; then
|
||
echo "$ICON_NOT_RUNNING $c is not running — skipping"
|
||
else
|
||
log "$c not found locally — skipping"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# Restart containers on the REMOTE server that were running before sync.
|
||
# Reads RUNNING_CONTAINERS set by stop_containers().
|
||
# Respects DELAYED_CONTAINERS — waits CONTAINER_DELAY seconds before starting them.
|
||
start_containers() {
|
||
if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then
|
||
log "No remote containers to restart."
|
||
return
|
||
fi
|
||
info "Starting remote containers..."
|
||
for c in "${RUNNING_CONTAINERS[@]}"; do
|
||
[[ -z "$c" ]] && continue
|
||
local needs_delay=false
|
||
for d in "${DELAYED_CONTAINERS[@]}"; do
|
||
[[ "$c" == "$d" ]] && needs_delay=true && break
|
||
done
|
||
if [[ "$needs_delay" == true ]]; then
|
||
info "Waiting ${CONTAINER_DELAY}s before starting $c..."
|
||
sleep "$CONTAINER_DELAY"
|
||
fi
|
||
echo "$ICON_START Starting remote $c..."
|
||
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $c" >/dev/null 2>&1; then
|
||
echo "$ICON_STARTED $c started"
|
||
else
|
||
error "Failed to start remote $c — start manually if needed"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# Restart containers on the LOCAL server that were running before sync.
|
||
# Reads LOCAL_RUNNING_CONTAINERS set by stop_local_containers().
|
||
# Respects DELAYED_CONTAINERS and CONTAINER_DELAY same as remote.
|
||
start_local_containers() {
|
||
if [[ ${#LOCAL_RUNNING_CONTAINERS[@]} -eq 0 ]]; then
|
||
log "No local containers to restart."
|
||
return
|
||
fi
|
||
info "Starting local containers..."
|
||
for c in "${LOCAL_RUNNING_CONTAINERS[@]}"; do
|
||
[[ -z "$c" ]] && continue
|
||
local needs_delay=false
|
||
for d in "${DELAYED_CONTAINERS[@]}"; do
|
||
[[ "$c" == "$d" ]] && needs_delay=true && break
|
||
done
|
||
if [[ "$needs_delay" == true ]]; then
|
||
info "Waiting ${CONTAINER_DELAY}s before starting local $c..."
|
||
sleep "$CONTAINER_DELAY"
|
||
fi
|
||
echo "$ICON_START Starting local $c..."
|
||
if docker start "$c" >/dev/null 2>&1; then
|
||
echo "$ICON_STARTED $c started"
|
||
else
|
||
error "Failed to start local $c — start manually if needed"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# Builds a dependency-safe restart order — dependency containers (anything named as a
|
||
# value in WATCHDOG_DEPENDENCIES) restart first, then everything else in original order.
|
||
# Sets ORDERED_RESTART. Usage: build_restart_order CONTAINERS_ARRAY_NAME (bare name, e.g.
|
||
# build_restart_order DAILY_RESTART_CONTAINERS — not "${DAILY_RESTART_CONTAINERS[@]}").
|
||
build_restart_order() {
|
||
local -n _bro_containers="$1"
|
||
ORDERED_RESTART=()
|
||
local remaining=("${_bro_containers[@]}")
|
||
local placed=()
|
||
|
||
# First pass — add dependency containers that appear in our list
|
||
for container in "${remaining[@]}"; do
|
||
[[ -z "$container" ]] && continue
|
||
local is_dependency=false
|
||
for dependent in "${!WATCHDOG_DEPENDENCIES[@]}"; do
|
||
if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then
|
||
is_dependency=true
|
||
break
|
||
fi
|
||
done
|
||
if [[ "$is_dependency" == true ]]; then
|
||
local already=false
|
||
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
|
||
if [[ "$already" == false ]]; then
|
||
ORDERED_RESTART+=("$container")
|
||
placed+=("$container")
|
||
fi
|
||
fi
|
||
done
|
||
|
||
# Second pass — add remaining containers (dependents and independents)
|
||
for container in "${remaining[@]}"; do
|
||
[[ -z "$container" ]] && continue
|
||
local already=false
|
||
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
|
||
if [[ "$already" == false ]]; then
|
||
ORDERED_RESTART+=("$container")
|
||
placed+=("$container")
|
||
fi
|
||
done
|
||
|
||
log "Restart order: ${ORDERED_RESTART[*]}"
|
||
}
|
||
|
||
# Waits CONTAINER_DELAY if this container depends on the last restarted one.
|
||
# Usage: check_dependency_delay "$container" "$last_restarted"
|
||
check_dependency_delay() {
|
||
local container="$1" last="$2"
|
||
[[ -z "$last" ]] && return
|
||
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
|
||
if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then
|
||
log "Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
|
||
sleep "$CONTAINER_DELAY"
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── RSYNC OPTIONS ─────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Loads rsync options for the current profile.
|
||
# Falls back to DEFAULT_RSYNC_OPTS from master.conf if no profile match found.
|
||
# Profile opts do NOT inherit from defaults — list all desired flags explicitly in master.conf.
|
||
|
||
get_rsync_opts() {
|
||
if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then
|
||
read -r -a RSYNC_OPTS <<< "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]}"
|
||
log "Using profile rsync opts for $PROFILE_NAME: ${RSYNC_OPTS[*]}"
|
||
else
|
||
RSYNC_OPTS=("${DEFAULT_RSYNC_OPTS[@]}")
|
||
log "Using default rsync opts: ${RSYNC_OPTS[*]}"
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── SCRIPT LOCKING ────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Prevents multiple instances of the same script running simultaneously.
|
||
# All lock files live in /tmp/unraid_locks/ — auto-cleared on reboot.
|
||
#
|
||
# Modes:
|
||
# strict — exit immediately if already running (default)
|
||
# wait — wait LOCK_WAIT_TIMEOUT seconds then exit if still locked
|
||
# continuous — for long-running scripts: skip gracefully if healthy instance running
|
||
#
|
||
# Features:
|
||
# Stale lock detection — if lock PID is dead, clears and proceeds
|
||
# PID reuse protection — lock stores PID:scriptname, validates both
|
||
# Age warning — warns if lock older than LOCK_WARN_AGE (skipped for continuous)
|
||
# EXIT trap — lock always released on exit, crash, or kill signal
|
||
|
||
LOCK_DIR="/tmp/unraid_locks"
|
||
|
||
# Ensure DATA_DIR and STATE_DIR exist — created here so every script that sources common.sh
|
||
# can safely write to either without checking first.
|
||
if [[ -n "${DATA_DIR:-}" ]] && [[ ! -d "$DATA_DIR" ]]; then
|
||
mkdir -p "$DATA_DIR" 2>/dev/null || true
|
||
fi
|
||
if [[ -n "${STATE_DIR:-}" ]] && [[ ! -d "$STATE_DIR" ]]; then
|
||
mkdir -p "$STATE_DIR" 2>/dev/null || true
|
||
fi
|
||
|
||
RSYNC_COUNT_FILE="$LOCK_DIR/rsync_active_count"
|
||
RSYNC_MAX_CONCURRENT=3
|
||
LOCK_WARN_AGE=300 # seconds — warn if lock older than this (5min default)
|
||
LOCK_WAIT_TIMEOUT=30 # seconds — how long "wait" mode waits before giving up
|
||
|
||
# Registry of all lock files acquired by this process — released together on exit.
|
||
# Prevents the single-trap-per-acquire problem: each new acquire_lock/acquire_rsync_lock
|
||
# call used to overwrite the EXIT trap, orphaning the previous lock file.
|
||
declare -ga _LOCK_FILES=()
|
||
_release_all_locks() {
|
||
local lf
|
||
for lf in "${_LOCK_FILES[@]}"; do
|
||
[[ -f "$lf" ]] && rm -f "$lf"
|
||
done
|
||
}
|
||
_register_lock() {
|
||
_LOCK_FILES+=("$1")
|
||
trap '_release_all_locks' EXIT
|
||
}
|
||
|
||
# Internal — script name used as lock identifier
|
||
_lock_name() {
|
||
basename "${BASH_SOURCE[1]:-$0}" .sh
|
||
}
|
||
|
||
# Internal — lock file path for this script
|
||
_lock_file() {
|
||
echo "$LOCK_DIR/${1:-$(_lock_name)}.lock"
|
||
}
|
||
|
||
# Acquire exclusive lock for this script.
|
||
# Usage: acquire_lock [strict|wait|continuous]
|
||
acquire_lock() {
|
||
local mode="${1:-strict}"
|
||
local script_name
|
||
script_name=$(basename "${BASH_SOURCE[1]:-$0}" .sh)
|
||
local lockfile
|
||
lockfile="$(_lock_file "$script_name")"
|
||
|
||
mkdir -p "$LOCK_DIR"
|
||
|
||
if [[ -f "$lockfile" ]]; then
|
||
local lock_content existing_pid locked_name
|
||
lock_content=$(cat "$lockfile" 2>/dev/null)
|
||
existing_pid="${lock_content%%:*}"
|
||
locked_name="${lock_content##*:}"
|
||
|
||
# Stale lock — PID dead
|
||
if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then
|
||
warn "Stale lock detected for $script_name (PID $existing_pid gone) — clearing"
|
||
rm -f "$lockfile"
|
||
|
||
# PID reuse — PID alive but belongs to different process
|
||
elif [[ "$locked_name" != "$script_name" ]]; then
|
||
warn "Lock PID $existing_pid reused by different process ($locked_name ≠ $script_name) — clearing"
|
||
rm -f "$lockfile"
|
||
|
||
else
|
||
# Lock is genuinely active — check age
|
||
local lock_age
|
||
lock_age=$(( $(date +%s) - $(stat -c %Y "$lockfile" 2>/dev/null || echo 0) ))
|
||
if [[ "$lock_age" -gt "$LOCK_WARN_AGE" ]] && [[ "$mode" != "continuous" ]]; then
|
||
warn "$script_name has been running for ${lock_age}s — may be stuck (PID $existing_pid)"
|
||
fi
|
||
|
||
if [[ "$mode" == "continuous" ]]; then
|
||
log "$ICON_SKIP $script_name already running healthy (PID $existing_pid) — skipping"
|
||
exit 0
|
||
elif [[ "$mode" == "wait" ]]; then
|
||
info "Another instance of $script_name is running — waiting up to ${LOCK_WAIT_TIMEOUT}s"
|
||
local waited=0
|
||
while [[ -f "$lockfile" ]] && [[ "$waited" -lt "$LOCK_WAIT_TIMEOUT" ]]; do
|
||
sleep 1
|
||
((waited++))
|
||
lock_content=$(cat "$lockfile" 2>/dev/null)
|
||
existing_pid="${lock_content%%:*}"
|
||
locked_name="${lock_content##*:}"
|
||
if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then
|
||
warn "Lock became stale while waiting — clearing"
|
||
rm -f "$lockfile"
|
||
break
|
||
elif [[ "$locked_name" != "$script_name" ]]; then
|
||
warn "Lock PID reused while waiting — clearing"
|
||
rm -f "$lockfile"
|
||
break
|
||
fi
|
||
done
|
||
if [[ -f "$lockfile" ]]; then
|
||
error "$script_name still locked after ${LOCK_WAIT_TIMEOUT}s — exiting"
|
||
exit 1
|
||
fi
|
||
else
|
||
error "Another instance of $script_name is already running (PID $existing_pid) — exiting"
|
||
case "$script_name" in
|
||
fallback|transcode_management|daily_sync_maintenance|stability_watchdog)
|
||
notify "$script_name lock collision on $(hostname) — concurrent instance detected" "$script_name" "warning"
|
||
;;
|
||
esac
|
||
exit 1
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
# Acquire lock — store PID:scriptname to prevent PID reuse false positives
|
||
echo "$$:$script_name" > "$lockfile"
|
||
_register_lock "$lockfile"
|
||
log "$ICON_LOCK Lock acquired: $script_name (PID $$)"
|
||
}
|
||
|
||
# Acquire per-profile rsync lock + enforce global concurrent limit.
|
||
# Prevents same profile running twice and limits total concurrent rsync instances.
|
||
# Usage: acquire_rsync_lock "$PROFILE_NAME"
|
||
acquire_rsync_lock() {
|
||
local profile="$1"
|
||
local profile_lock
|
||
profile_lock="$(_lock_file "rsync_${profile}")"
|
||
|
||
mkdir -p "$LOCK_DIR"
|
||
|
||
# Per-profile lock — same profile cannot run twice
|
||
if [[ -f "$profile_lock" ]]; then
|
||
local lock_content existing_pid locked_name
|
||
lock_content=$(cat "$profile_lock" 2>/dev/null)
|
||
existing_pid="${lock_content%%:*}"
|
||
locked_name="${lock_content##*:}"
|
||
if [[ -n "$existing_pid" ]] && kill -0 "$existing_pid" 2>/dev/null && \
|
||
[[ "$locked_name" == "rsync_${profile}" ]]; then
|
||
error "rsync profile '$profile' is already running (PID $existing_pid) — exiting"
|
||
exit 1
|
||
else
|
||
warn "Stale rsync lock for profile '$profile' — clearing"
|
||
rm -f "$profile_lock"
|
||
fi
|
||
fi
|
||
|
||
# Global concurrent limit — validate count against actual live locks
|
||
local current_count=0
|
||
if [[ -f "$RSYNC_COUNT_FILE" ]]; then
|
||
current_count=$(cat "$RSYNC_COUNT_FILE" 2>/dev/null || echo 0)
|
||
local actual_count=0
|
||
for lf in "$LOCK_DIR"/rsync_*.lock; do
|
||
[[ -f "$lf" ]] || continue
|
||
local lpid lf_content
|
||
lf_content=$(cat "$lf" 2>/dev/null)
|
||
lpid="${lf_content%%:*}"
|
||
kill -0 "$lpid" 2>/dev/null && ((actual_count++))
|
||
done
|
||
if [[ "$actual_count" -ne "$current_count" ]]; then
|
||
log "rsync count corrected: $current_count → $actual_count"
|
||
current_count=$actual_count
|
||
echo "$current_count" > "$RSYNC_COUNT_FILE"
|
||
fi
|
||
fi
|
||
|
||
if [[ "$current_count" -ge "$RSYNC_MAX_CONCURRENT" ]]; then
|
||
error "Maximum concurrent rsync limit ($RSYNC_MAX_CONCURRENT) reached — exiting"
|
||
info "Active rsync locks: $(ls "$LOCK_DIR"/rsync_*.lock 2>/dev/null | \
|
||
xargs -I{} basename {} .lock | tr '\n' ' ')"
|
||
exit 1
|
||
fi
|
||
|
||
# Acquire profile lock and increment counter
|
||
echo "$$:rsync_${profile}" > "$profile_lock"
|
||
echo $(( current_count + 1 )) > "$RSYNC_COUNT_FILE"
|
||
_register_lock "$profile_lock"
|
||
log "$ICON_LOCK rsync lock acquired: profile '$profile' (PID $$, active: $(( current_count + 1 ))/$RSYNC_MAX_CONCURRENT)"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── FLAT STATE-FILE KEY/VALUE HELPERS ─────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Generic get/set for flat "key<sep>value" state files (one entry per line) — the pattern
|
||
# every watchdog's strike-tracking and state-file logic was independently reimplementing.
|
||
# grep -v + tempfile-swap on write, not sed -i in place — avoids sed treating a key containing
|
||
# regex metacharacters (container names, etc.) as part of the substitution pattern.
|
||
#
|
||
# Usage: wd_state_get "$key" "$file" [sep=:]
|
||
# wd_state_set "$key" "$value" "$file" [sep=:]
|
||
#
|
||
# Callers with a fixed state file and/or separator (e.g. stability_watchdog.sh's
|
||
# get_strikes/get_state_val, resource_watchdog.sh's rm_state_get/rm_state_get_eq) should
|
||
# keep their own thin same-named wrapper around these rather than changing call sites.
|
||
wd_state_get() {
|
||
local key="$1" file="$2" sep="${3:-:}"
|
||
grep -E "^${key}${sep}" "$file" 2>/dev/null | cut -d"$sep" -f2-
|
||
}
|
||
|
||
wd_state_set() {
|
||
local key="$1" value="$2" file="$3" sep="${4:-:}"
|
||
grep -vE "^${key}${sep}" "$file" 2>/dev/null > "${file}.tmp"
|
||
echo "${key}${sep}${value}" >> "${file}.tmp"
|
||
mv "${file}.tmp" "$file"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── ORCHESTRATOR CHILD EXECUTION ──────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Shared child-script runner for Orchestrators/*.sh. Replaces the run_job()/run_watchdog()
|
||
# copies that used to be hand-duplicated (with small inconsistencies) into each orchestrator —
|
||
# one implementation now, one place to fix a bug or extend behavior (e.g. per-child logging).
|
||
#
|
||
# Usage — caller declares its own tracking arrays before the loop:
|
||
# JOB_PASS=()
|
||
# JOB_FAIL=()
|
||
# for entry in "${SOME_SCRIPTS[@]}"; do
|
||
# run_orch_child "$entry"
|
||
# done
|
||
#
|
||
# $entry is "relative/path.sh [extra args...]" — the same format every master.conf
|
||
# script-list array already uses. Resolved against $ECOSYSTEM_ROOT, which the caller
|
||
# must set before calling this (the absolute repo root — "$(cd "$SCRIPT_DIR/.." && pwd)").
|
||
#
|
||
# --dry-run / --log are threaded down automatically from $DRY_RUN / $ENABLE_LOGGING —
|
||
# never $VERBOSE, which nothing in this codebase ever assigns.
|
||
|
||
run_orch_child() {
|
||
local entry="$1"
|
||
local script_args script_path script_name label extra_args run_args
|
||
|
||
read -r -a script_args <<< "$entry"
|
||
script_path="$ECOSYSTEM_ROOT/${script_args[0]}"
|
||
script_name=$(basename "${script_args[0]}")
|
||
extra_args=("${script_args[@]:1}")
|
||
label="$script_name"
|
||
[[ -n "${extra_args[*]}" ]] && label="$script_name ${extra_args[*]}"
|
||
|
||
if [[ ! -f "$script_path" ]]; then
|
||
error "$script_name — not found at $script_path"
|
||
JOB_FAIL+=("$label")
|
||
return 1
|
||
fi
|
||
[[ ! -x "$script_path" ]] && chmod +x "$script_path"
|
||
|
||
run_args=("${extra_args[@]}")
|
||
[[ "$DRY_RUN" == true ]] && run_args+=("--dry-run")
|
||
[[ "$ENABLE_LOGGING" == true ]] && run_args+=("--log")
|
||
|
||
local _start _ec
|
||
_start=$(date +%s)
|
||
log "Running: $label"
|
||
if bash "$script_path" "${run_args[@]}"; then
|
||
log "$script_name — done in $(format_duration $(( $(date +%s) - _start )))"
|
||
JOB_PASS+=("$label")
|
||
return 0
|
||
else
|
||
_ec=$?
|
||
error "$label — failed (exit $_ec, $(format_duration $(( $(date +%s) - _start ))))"
|
||
JOB_FAIL+=("$label")
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── PATH TRANSLATION ──────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Translates container-internal file paths to host paths for arr cleanup scripts.
|
||
# Arr APIs return file paths as seen inside the container — scripts need host paths to scan.
|
||
#
|
||
# Uses ARR_PATH_MAP associative array — must be declared before calling.
|
||
# Longest-match wins — prevents partial path collisions.
|
||
#
|
||
# Usage:
|
||
# declare -A ARR_PATH_MAP=(["/ext-music"]="/mnt/user/Music")
|
||
# translate_path "/ext-music/Artist/Album/track.flac"
|
||
# Returns: /mnt/user/Music/Artist/Album/track.flac
|
||
#
|
||
# If no match found — returns path unchanged (safe fallback).
|
||
|
||
translate_path() {
|
||
local api_path="$1"
|
||
local best_match=""
|
||
local best_len=0
|
||
|
||
for container_path in "${!ARR_PATH_MAP[@]}"; do
|
||
if [[ "$api_path" == "$container_path"* ]]; then
|
||
if [[ "${#container_path}" -gt "$best_len" ]]; then
|
||
best_match="$container_path"
|
||
best_len="${#container_path}"
|
||
fi
|
||
fi
|
||
done
|
||
|
||
if [[ -n "$best_match" ]]; then
|
||
echo "${ARR_PATH_MAP[$best_match]}${api_path#$best_match}"
|
||
else
|
||
echo "$api_path" # no match — return unchanged
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── CONTAINER STOP/RESTART FOR MAINTENANCE ────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# The "stop a container for a maintenance window, guarantee it comes back" shape shared by
|
||
# container_data_export.sh and emby_database_repair.sh. Callers still register their own EXIT
|
||
# trap (they may need extra cleanup, e.g. removing a partial archive) — the trap should call
|
||
# container_force_restart_if_needed() for the actual "still down? bring it back" part.
|
||
|
||
# Usage: container_stop_for_maintenance "$container" was_running_var_name \
|
||
# ["pre-stop reason message"] [log|warn] [post_stop_sleep=0]
|
||
# The reason message (if given) prints via the chosen level (default: log) right before the
|
||
# stop attempt, only when the container was actually running — callers use this for their
|
||
# own context ("stopping for clean export" vs "active sessions will be interrupted").
|
||
# Sets the named var true/false. Returns 1 (caller should exit) if the container doesn't
|
||
# exist or fails to stop; 0 otherwise (including the "wasn't running" case — caller logs
|
||
# its own context-specific message for that, since wording/visibility differs per caller).
|
||
container_stop_for_maintenance() {
|
||
local container="$1"
|
||
local -n _csm_was_running="$2"
|
||
local pre_stop_msg="${3:-}" pre_stop_level="${4:-log}" post_stop_sleep="${5:-0}"
|
||
_csm_was_running=false
|
||
|
||
local status
|
||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||
case "$status" in
|
||
true)
|
||
_csm_was_running=true
|
||
if [[ -n "$pre_stop_msg" ]]; then
|
||
if [[ "$pre_stop_level" == "warn" ]]; then warn "$pre_stop_msg"; else log "$pre_stop_msg"; fi
|
||
fi
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
if timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1; then
|
||
echo "$container stopped ✅"
|
||
[[ "$post_stop_sleep" -gt 0 ]] && sleep "$post_stop_sleep"
|
||
else
|
||
error "Failed to stop $container — aborting"
|
||
return 1
|
||
fi
|
||
else
|
||
warn "DRY RUN — would stop $container"
|
||
fi
|
||
;;
|
||
false) : ;; # not running — caller logs its own context-specific message
|
||
"")
|
||
error "$container not found — check container name"
|
||
return 1
|
||
;;
|
||
*)
|
||
warn "$container status: $status — proceeding with caution"
|
||
;;
|
||
esac
|
||
return 0
|
||
}
|
||
|
||
# Force-restart from inside a script's own EXIT trap if the container is still down after
|
||
# a crash mid-maintenance. No-op if it wasn't running before, in dry-run, or already up.
|
||
# Usage: container_force_restart_if_needed "$container" "$was_running"
|
||
container_force_restart_if_needed() {
|
||
local container="$1" was_running="$2"
|
||
[[ "$was_running" == true && "$DRY_RUN" == false ]] || return 0
|
||
local status
|
||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||
if [[ "$status" != "true" ]]; then
|
||
warn "Restarting $container (cleanup)..."
|
||
timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1 || \
|
||
error "Failed to restart $container — start it manually"
|
||
fi
|
||
}
|
||
|
||
# Normal-path restart after maintenance completes — verifies it stayed up, notifies on failure.
|
||
# Sets RESTART_OK true/false.
|
||
# Usage: container_restart_after_maintenance "$container" "$was_running" [settle_sleep=3] [notify_label]
|
||
container_restart_after_maintenance() {
|
||
local container="$1" was_running="$2" settle_sleep="${3:-3}" notify_label="${4:-Container Maintenance}"
|
||
RESTART_OK=false
|
||
|
||
if [[ "$was_running" != true ]]; then
|
||
echo "$container was not running — leaving stopped (state respected) ✅"
|
||
RESTART_OK=true
|
||
return 0
|
||
fi
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would restart $container"
|
||
RESTART_OK=true
|
||
return 0
|
||
fi
|
||
|
||
log "Restarting $container..."
|
||
if timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1; then
|
||
sleep "$settle_sleep"
|
||
local post_status
|
||
post_status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||
if [[ "$post_status" == "true" ]]; then
|
||
echo "$container restarted and running ✅"
|
||
RESTART_OK=true
|
||
else
|
||
error "$container started but crashed — check container logs"
|
||
notify "$container failed to stay running after maintenance on $(hostname)" \
|
||
"$notify_label" "warning"
|
||
fi
|
||
else
|
||
error "Failed to restart $container — start it manually"
|
||
notify "$container failed to restart after maintenance on $(hostname)" \
|
||
"$notify_label" "warning"
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── ARR CLEANUP SAFETY GATES ──────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Shared by lidarr/radarr/sonarr_cleanup.sh — these scripts DELETE files, so every function
|
||
# here is a direct byte-for-byte port of what was independently duplicated three times, not
|
||
# a rewrite. Callers should behave identically to before this consolidation.
|
||
|
||
# Safety Layer 1 — container running, not starting/unhealthy. Exits 1 (with notify) on failure.
|
||
# Usage: check_container_health "$LIDARR_CONTAINER" "$ARR_DOCKER_TIMEOUT" "Lidarr Cleanup"
|
||
check_container_health() {
|
||
local container="$1" docker_timeout="$2" notify_label="$3"
|
||
|
||
local running
|
||
running=$(timeout "$docker_timeout" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||
if [[ "$running" != "true" ]]; then
|
||
error "$container is not running — aborting"
|
||
notify "$notify_label aborted on $(hostname) — container not running" "$notify_label" "warning"
|
||
exit 1
|
||
fi
|
||
|
||
local health
|
||
health=$(timeout "$docker_timeout" docker inspect -f '{{.State.Health.Status}}' "$container" 2>/dev/null)
|
||
case "$health" in
|
||
healthy) info "$container is healthy" ;;
|
||
"") info "$container has no health check — proceeding" ;;
|
||
starting)
|
||
error "$container is still starting — aborting"
|
||
notify "$notify_label aborted on $(hostname) — container still starting" "$notify_label" "warning"
|
||
exit 1 ;;
|
||
unhealthy)
|
||
error "$container is unhealthy — aborting"
|
||
notify "$notify_label aborted on $(hostname) — container unhealthy" "$notify_label" "warning"
|
||
exit 1 ;;
|
||
*) warn "$container health: $health — proceeding with caution" ;;
|
||
esac
|
||
|
||
info "Safety layer 1 passed — container healthy"
|
||
}
|
||
|
||
# Safety Layer 6 — abort if tracked count dropped below min_pct of the last known count
|
||
# (protects against a misconfigured path / partial API response silently wiping the library).
|
||
# Writes the new baseline only if the check passes. Exits 1 (with notify) on failure.
|
||
# Usage: check_tracked_count_floor "$TRACKED_COUNT" "$LIDARR_TRACKED_COUNT_FILE" "$LIDARR_MIN_TRACKED_PCT" "Lidarr Cleanup"
|
||
check_tracked_count_floor() {
|
||
local tracked_count="$1" baseline_file="$2" min_pct="$3" notify_label="$4"
|
||
|
||
if [[ -f "$baseline_file" ]]; then
|
||
local last_count pct
|
||
last_count=$(cat "$baseline_file" 2>/dev/null || echo 0)
|
||
if [[ "$last_count" -gt 0 ]]; then
|
||
pct=$(awk "BEGIN {printf \"%d\", ($tracked_count / $last_count) * 100}")
|
||
if [[ "$pct" -lt "$min_pct" ]]; then
|
||
error "Tracked count dropped to ${pct}% of last run ($tracked_count vs $last_count)"
|
||
error "Suggests API issue — aborting to prevent mass deletion"
|
||
error "If expected (large library removal) delete: $baseline_file"
|
||
notify "$notify_label aborted on $(hostname) — tracked count dropped to ${pct}%" \
|
||
"$notify_label" "warning"
|
||
exit 1
|
||
fi
|
||
info "Tracked count: ${pct}% of last run ($tracked_count vs $last_count) ✅"
|
||
fi
|
||
else
|
||
info "No previous count on record — first run, saving baseline"
|
||
fi
|
||
|
||
echo "$tracked_count" > "$baseline_file"
|
||
}
|
||
|
||
# Filters --i-know-what-im-doing / --skip-age-check out of "$@" before parse_args sees them
|
||
# (both are cleanup-script-specific, not part of the shared arg parser).
|
||
# Sets I_KNOW, SKIP_AGE_CHECK, FILTERED_ARGS — call parse_args "${FILTERED_ARGS[@]}" after.
|
||
parse_destructive_flags() {
|
||
I_KNOW=false
|
||
SKIP_AGE_CHECK=false
|
||
FILTERED_ARGS=()
|
||
local arg
|
||
for arg in "$@"; do
|
||
case "$arg" in
|
||
--i-know-what-im-doing) I_KNOW=true ;;
|
||
--skip-age-check) SKIP_AGE_CHECK=true ;;
|
||
*) FILTERED_ARGS+=("$arg") ;;
|
||
esac
|
||
done
|
||
}
|
||
|
||
# Prints the nuclear-mode banner and sleeps 10s (Ctrl+C window) when both destructive flags
|
||
# are set and this isn't a dry run. Call after parse_args. No-op otherwise.
|
||
nuclear_mode_warning() {
|
||
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_AGE_CHECK" == true ]] && [[ "$DRY_RUN" != true ]]; then
|
||
echo ""
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo " Flags: --i-know-what-im-doing --skip-age-check"
|
||
echo " Age check: BYPASSED — deletes on first pass"
|
||
echo " Size threshold: BYPASSED — no GB limit"
|
||
echo " Data recovery: NOT POSSIBLE after deletion"
|
||
echo ""
|
||
echo " Review --dry-run output before proceeding."
|
||
echo " You have 10 seconds to cancel (Ctrl+C)..."
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
sleep 10
|
||
echo " Proceeding..."
|
||
echo ""
|
||
fi
|
||
}
|
||
|
||
# Safety Layer 7 — abort if total deletion size exceeds max_gb and --i-know-what-im-doing
|
||
# wasn't passed. Sets TOTAL_HUMAN. Exits 1 (with notify) on failure; warns and continues
|
||
# if I_KNOW is true (set by parse_destructive_flags).
|
||
# Usage: check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$LIDARR_MAX_DELETE_GB" "Lidarr Cleanup"
|
||
check_delete_size_threshold() {
|
||
local total_bytes="$1" max_gb="$2" notify_label="$3"
|
||
local max_bytes
|
||
max_bytes=$(awk "BEGIN {printf \"%d\", $max_gb * 1073741824}")
|
||
if [[ "$total_bytes" -gt "$max_bytes" ]]; then
|
||
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $total_bytes / 1073741824}")
|
||
if [[ "$I_KNOW" != true ]]; then
|
||
echo ""
|
||
error "Deletion would exceed ${max_gb}GB — $TOTAL_HUMAN would be deleted"
|
||
error "Review ORPHAN lines above carefully before proceeding"
|
||
error "Rerun with: --i-know-what-im-doing"
|
||
error "To also bypass age check: add --skip-age-check"
|
||
notify "$notify_label halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
|
||
"$notify_label" "warning"
|
||
exit 1
|
||
else
|
||
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
|
||
fi
|
||
fi
|
||
}
|
||
|
||
# True if filepath's extension (case-insensitive) matches one of the given extensions.
|
||
# Usage: has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"
|
||
has_extension() {
|
||
local filepath="$1"; shift
|
||
local ext="${filepath##*.}"
|
||
ext="${ext,,}"
|
||
local valid_ext
|
||
for valid_ext in "$@"; do
|
||
[[ "$ext" == "$valid_ext" ]] && return 0
|
||
done
|
||
return 1
|
||
}
|
||
|
||
# True if filepath's basename matches one of the given glob patterns.
|
||
# Usage: matches_pattern_list "$filepath" "${LIDARR_PROTECTED_PATTERNS[@]}"
|
||
#
|
||
# Parameter expansion instead of external basename (2026-07-17) — called once per
|
||
# non-tracked file in lidarr_cleanup.sh/sonarr_cleanup.sh/radarr_cleanup.sh's classification
|
||
# loop, which is most files in a media library (every protected sidecar: nfo/jpg/srt/etc).
|
||
# Measured: this was the actual dominant cost left in those scripts even after fixing the
|
||
# per-file stat fork and dirname elsewhere — 47.3s vs 1.65s for 10,000 calls (~28.6x),
|
||
# confirmed identical results across a spot check first.
|
||
matches_pattern_list() {
|
||
local filepath="$1"; shift
|
||
local filename="${filepath##*/}"
|
||
local pattern
|
||
for pattern in "$@"; do
|
||
# shellcheck disable=SC2254
|
||
case "$filename" in
|
||
$pattern) return 0 ;;
|
||
esac
|
||
done
|
||
return 1
|
||
}
|
||
|
||
# Builds the global ARR_PATH_MAP associative array (container path → host path) from
|
||
# ${MY_ID}_<ARR_TYPE>_PATH_MAP, for translate_path() to use. Usage: build_arr_path_map "LIDARR"
|
||
build_arr_path_map() {
|
||
local arr_type="$1"
|
||
declare -gA ARR_PATH_MAP=()
|
||
local _bapm_var="${MY_ID}_${arr_type}_PATH_MAP"
|
||
eval "for key in \"\${!${_bapm_var}[@]}\"; do
|
||
ARR_PATH_MAP[\"\$key\"]=\"\${${_bapm_var}[\$key]}\"
|
||
done"
|
||
}
|
||
|
||
# Generic arr API GET wrapper with HTTP status check.
|
||
# Usage: arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "movie" "Radarr"
|
||
arr_api() {
|
||
local base_url="$1" api_key="$2" api_version="$3" endpoint="$4" label="${5:-Arr}"
|
||
local response http_code body
|
||
|
||
response=$(curl -sf \
|
||
--max-time 30 \
|
||
-H "X-Api-Key: $api_key" \
|
||
-w "\n%{http_code}" \
|
||
"${base_url}/api/${api_version}/${endpoint}" 2>/dev/null)
|
||
|
||
http_code=$(echo "$response" | tail -1)
|
||
body=$(echo "$response" | head -n -1)
|
||
|
||
if [[ "$http_code" != "200" ]]; then
|
||
error "$label API HTTP $http_code for: $endpoint"
|
||
return 1
|
||
fi
|
||
echo "$body"
|
||
}
|
||
|
||
# Triggers an arr "command" endpoint with the given JSON payload, then polls every 10s until
|
||
# completed/failed/timeout, logging progress every 60s. Best-effort pre-flight — never treats
|
||
# an unreachable/timed-out scan as fatal, matching original per-script behavior.
|
||
#
|
||
# On a genuine "completed" observation, records the actual elapsed duration keyed by the
|
||
# command's own name via arr_record_rescan_duration() (needs arr_type to know which arr's
|
||
# duration DB to write to) — this is the one place in the codebase that reliably watches a
|
||
# command from trigger to completion, so it's the natural spot to build up real historical
|
||
# duration data for the wait-calibration logic in arr_get_tracked_data(). A timeout doesn't
|
||
# record anything — we only know a lower bound, not the true duration, and recording that
|
||
# would corrupt future wait calculations downward. arr_type is optional — omit it (or pass
|
||
# empty) to skip duration recording entirely, e.g. for one-off commands that aren't one of
|
||
# the three tracked arrs.
|
||
#
|
||
# Usage: trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" \
|
||
# '{"name": "DownloadedAlbumsScan"}' "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}" "lidarr"
|
||
trigger_and_await_command() {
|
||
local base_url="$1" api_key="$2" api_version="$3" payload="$4" poll_timeout="${5:-600}" arr_type="${6:-}"
|
||
|
||
local cmd_name
|
||
cmd_name=$(echo "$payload" | jq -r '.name // empty' 2>/dev/null)
|
||
|
||
local scan_response scan_cmd_id
|
||
scan_response=$(curl -sf --max-time 30 -X POST \
|
||
-H "X-Api-Key: $api_key" \
|
||
-H "Content-Type: application/json" \
|
||
-d "$payload" \
|
||
"${base_url}/api/${api_version}/command" 2>/dev/null)
|
||
|
||
scan_cmd_id=$(echo "$scan_response" | jq -r '.id // empty' 2>/dev/null)
|
||
|
||
if [[ -z "$scan_cmd_id" ]]; then
|
||
warn "Could not trigger import scan — proceeding without pre-flight"
|
||
return 0
|
||
fi
|
||
|
||
info "Import scan queued (command ID: $scan_cmd_id) — waiting for completion..."
|
||
local start_epoch=$(date +%s)
|
||
local polled=0 scan_status
|
||
while [[ "$polled" -lt "$poll_timeout" ]]; do
|
||
scan_status=$(curl -sf --max-time 10 \
|
||
-H "X-Api-Key: $api_key" \
|
||
"${base_url}/api/${api_version}/command/${scan_cmd_id}" 2>/dev/null | \
|
||
jq -r '.status // empty' 2>/dev/null)
|
||
case "$scan_status" in
|
||
completed)
|
||
info "Import scan complete ✅"
|
||
[[ -n "$cmd_name" && -n "$arr_type" ]] && \
|
||
arr_record_rescan_duration "$arr_type" "$cmd_name" "$(( $(date +%s) - start_epoch ))"
|
||
return 0
|
||
;;
|
||
failed)
|
||
warn "Import scan reported failed — proceeding anyway"
|
||
return 0
|
||
;;
|
||
esac
|
||
sleep 10
|
||
(( polled += 10 ))
|
||
[[ $(( polled % 60 )) -eq 0 ]] && log " Still scanning... (${polled}s elapsed)"
|
||
done
|
||
warn "Import scan timed out after ${poll_timeout}s — proceeding anyway"
|
||
return 0
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── ARR TRACKED-DATA CACHE (Lidarr, Sonarr, Radarr) ──────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Shared cache for each arr's tracked-library data (Lidarr artists, Sonarr series, Radarr
|
||
# movies). Every script across all three arrs that needs "what does this arr think it has
|
||
# tracked" goes through this — 15+ consumers as of 2026-07-17 (all three cleanup scripts, the
|
||
# missing-art/release-fixer/duplicate-cleanup scripts, all three playback-aware discovery
|
||
# scripts, the TVDB/TMDb-removed checks, arr_sync.sh's local side, and the emby_to_*_sync
|
||
# bootstrap tools). Hitting the live API fresh every time is wasteful, and during an active
|
||
# rescan the live number is actively misleading — tracked counts dip and recover as files are
|
||
# detached/re-verified one by one (confirmed 2026-07-16 on Lidarr: a whole-library
|
||
# RescanFolders made trackFileCount read 22% of normal mid-scan, which is exactly the
|
||
# false-alarm lidarr_cleanup.sh's count-drop guard is meant to catch, but a genuine rescan
|
||
# isn't the "something's actually wrong" case that guard exists for — the same risk applies to
|
||
# Sonarr's RescanSeries and Radarr's RescanMovie).
|
||
#
|
||
# Built Lidarr-only first (2026-07-16), generalized the same day to cover all three arrs —
|
||
# identical mechanism, keyed by arr_type ("lidarr"/"sonarr"/"radarr") so each arr's cache and
|
||
# duration history stay separate.
|
||
#
|
||
# Storage (2026-07-17): primary copy lives on tmpfs (ARR_CACHE_DIR) — reads/writes never touch
|
||
# the array disk, and losing it on reboot costs nothing since a full rebuild for all three arrs
|
||
# measures ~12s live. A persistent backup on $DATA_DIR is kept in sync by every write and gets
|
||
# transparently restored into tmpfs by arr_cache_age_seconds() the moment it notices tmpfs is
|
||
# missing — so a cache that was fresh before a reboot reads as fresh after too.
|
||
#
|
||
# Write-through: any script that already does a live library-list fetch for its own purposes
|
||
# writes the result here as a side effect via arr_cache_write() — no dedicated polling timer
|
||
# needed. arr_cache_write() itself refuses to write while a rescan-type command is active for
|
||
# that arr (2026-07-17) — protects every caller uniformly, not just arr_get_tracked_data().
|
||
# Arrs_Stack/arr_cache_prefill.sh keeps the cache warm two ways: once at array start (10min
|
||
# wait ceiling, closes the cold-boot gap) and again every 30min via CRITICAL_MAINTENANCE_SCRIPTS
|
||
# (1min wait ceiling — a live fetch takes seconds, not the boot-time wait).
|
||
#
|
||
# Consumers should call arr_get_tracked_data() — never read a cache file directly. It handles
|
||
# the fresh/stale-no-rescan/stale-rescan-active branching so no script reimplements it.
|
||
#
|
||
# This is the top-level LIBRARY cache only. A separate, shorter-lived cache exists for the
|
||
# much more expensive per-item track/episode data — see "PER-ITEM TRACKED-FILE CACHE" below.
|
||
# ==============================================================================================
|
||
|
||
# Rescan-type command names per arr — operations long/heavy enough that overlapping with one
|
||
# should trigger a wait rather than a live fetch. Verified live against each arr's API
|
||
# 2026-07-16 (not assumed from memory — RescanSeries/RescanMovie confirmed to scan the whole
|
||
# library when given no id, matching Lidarr's RescanFolders).
|
||
declare -gA ARR_RESCAN_COMMANDS=(
|
||
[lidarr]="RescanFolders DownloadedAlbumsScan RefreshArtist"
|
||
[sonarr]="RescanSeries DownloadedEpisodesScan RefreshSeries"
|
||
[radarr]="RescanMovie DownloadedMoviesScan RefreshMovie"
|
||
)
|
||
|
||
# jq expression computing each arr's "total tracked" number from its library-list response —
|
||
# schemas differ: Lidarr/Sonarr have a per-item file count to sum, Radarr is a binary hasFile
|
||
# per movie (one file at most), so the aggregate has to be a count of true values instead.
|
||
declare -gA ARR_TRACKED_COUNT_EXPR=(
|
||
[lidarr]='[.[].statistics.trackFileCount] | add'
|
||
[sonarr]='[.[].statistics.episodeFileCount] | add'
|
||
[radarr]='[.[] | select(.hasFile==true)] | length'
|
||
)
|
||
|
||
# Library-list endpoint name per arr, for arr_api().
|
||
declare -gA ARR_LIBRARY_ENDPOINT=(
|
||
[lidarr]="artist"
|
||
[sonarr]="series"
|
||
[radarr]="movie"
|
||
)
|
||
|
||
# API version per arr — Lidarr is still v1, Sonarr/Radarr are v3.
|
||
declare -gA ARR_API_VERSION=(
|
||
[lidarr]="v1"
|
||
[sonarr]="v3"
|
||
[radarr]="v3"
|
||
)
|
||
|
||
# tmpfs — primary read/write location. Cleared every reboot; that's fine, a full rebuild for
|
||
# all three arrs measures ~12s live, and arr_cache_age_seconds() transparently restores from
|
||
# the persistent backup below the moment it notices this is missing.
|
||
arr_cache_file() { echo "${ARR_CACHE_DIR}/${1}_tracked_cache.json"; }
|
||
|
||
# Persistent backup — survives reboot. Kept in sync by arr_cache_write() dual-writing here on
|
||
# every write, so it's never more than one write-cycle stale (2026-07-17). Exists purely so a
|
||
# reboot doesn't leave the tmpfs cache genuinely empty until the next live fetch completes —
|
||
# not a source of truth in its own right, just what tmpfs gets restored from when missing.
|
||
arr_cache_backup_file() { echo "${DATA_DIR}/${1}_tracked_cache.json"; }
|
||
|
||
arr_rescan_duration_db() { echo "${DATA_DIR}/${1}_rescan_duration.db"; }
|
||
|
||
# Writes the current library-list JSON + computed total tracked count to arr_type's cache.
|
||
# Radarr's movie list alone runs ~16MB — passing that through jq's --argjson as a literal
|
||
# command-line argument blows past the OS's ARG_MAX ("Argument list too long"), the exact
|
||
# same class of bug the queue-pagination fix (2026-07-16, arrs_failed_stalled_recovery.sh)
|
||
# hit before. Fixed the same way: write the payload to a temp file and use --slurpfile,
|
||
# which reads from disk instead of argv.
|
||
#
|
||
# Refuses to write while a rescan-type command is active for arr_type (2026-07-17). Every
|
||
# direct caller here fetches the library list for its own purposes and writes it through as
|
||
# a side effect — none of them go through arr_get_tracked_data()'s fresh/stale/active-rescan
|
||
# branching, so none of them knew to check scan state first. Confirmed live: a caller wrote
|
||
# Lidarr's cache mid-RescanFolders at 22% of the real total, and that partial number then
|
||
# looked exactly like real data loss to every consumer (check_tracked_count_floor et al.)
|
||
# until the scan finished. The guard lives here instead of in each caller so it applies
|
||
# uniformly — last-known-good stays in place until something writes through the real
|
||
# post-scan number (arr_full_rescan.sh does this itself on completion; for a rescan someone
|
||
# else triggered, see arr_rescan_monitor.sh in Tools/).
|
||
# Args: arr_type, items_json (the full library-list response body)
|
||
arr_cache_write() {
|
||
local arr_type="$1" items_json="$2"
|
||
local expr="${ARR_TRACKED_COUNT_EXPR[$arr_type]:-}"
|
||
[[ -z "$expr" ]] && return 1
|
||
|
||
local url_var="${arr_type^^}_URL" key_var="${arr_type^^}_API_KEY"
|
||
local url="${!url_var:-}" api_key="${!key_var:-}" api_version="${ARR_API_VERSION[$arr_type]:-}"
|
||
if [[ -n "$url" && -n "$api_key" && -n "$api_version" ]]; then
|
||
local active_cmd
|
||
active_cmd=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version")
|
||
[[ -n "$active_cmd" ]] && return 1
|
||
fi
|
||
|
||
local total
|
||
total=$(echo "$items_json" | jq "$expr" 2>/dev/null)
|
||
[[ -z "$total" || "$total" == "null" ]] && return 1
|
||
|
||
local cache_file backup_file items_tmp
|
||
cache_file=$(arr_cache_file "$arr_type")
|
||
backup_file=$(arr_cache_backup_file "$arr_type")
|
||
items_tmp=$(mktemp)
|
||
echo "$items_json" > "$items_tmp"
|
||
mkdir -p "$(dirname "$cache_file")" "$(dirname "$backup_file")" 2>/dev/null
|
||
jq -c -n --slurpfile items "$items_tmp" --argjson total "$total" --argjson ts "$(date +%s)" \
|
||
'{ts:$ts, totalTracked:$total, items:$items[0]}' > "${cache_file}.tmp" 2>/dev/null \
|
||
&& mv "${cache_file}.tmp" "$cache_file" \
|
||
&& cp "$cache_file" "$backup_file" 2>/dev/null
|
||
rm -f "$items_tmp"
|
||
}
|
||
|
||
# Echoes the cached library-list JSON array for arr_type if a cache file exists, regardless
|
||
# of age — staleness is arr_get_tracked_data()'s decision, not this raw reader's.
|
||
arr_cache_read_raw() {
|
||
local arr_type="$1"
|
||
local cache_file
|
||
cache_file=$(arr_cache_file "$arr_type")
|
||
[[ -f "$cache_file" ]] || return 1
|
||
jq -c '.items // empty' "$cache_file" 2>/dev/null
|
||
}
|
||
|
||
# Echoes arr_type's cache age in seconds, or a very large number if no cache and no backup
|
||
# exist (so age comparisons naturally treat that as "very stale"). Transparently restores the
|
||
# tmpfs cache from its persistent backup first if tmpfs is missing (e.g. right after a reboot)
|
||
# — a cache that was fresh before reboot reads as fresh here too, not as if it never existed.
|
||
arr_cache_age_seconds() {
|
||
local arr_type="$1"
|
||
local cache_file
|
||
cache_file=$(arr_cache_file "$arr_type")
|
||
if [[ ! -f "$cache_file" ]]; then
|
||
local backup_file
|
||
backup_file=$(arr_cache_backup_file "$arr_type")
|
||
if [[ -f "$backup_file" ]]; then
|
||
mkdir -p "$(dirname "$cache_file")" 2>/dev/null
|
||
cp "$backup_file" "$cache_file" 2>/dev/null
|
||
fi
|
||
fi
|
||
[[ -f "$cache_file" ]] || { echo 999999999; return; }
|
||
local ts
|
||
ts=$(jq -r '.ts // 0' "$cache_file" 2>/dev/null)
|
||
echo $(( $(date +%s) - ${ts:-0} ))
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── PER-ITEM TRACKED-FILE CACHE (Lidarr trackFile, Sonarr episodefile) ───────────────────────
|
||
# ==============================================================================================
|
||
# Different in kind from the tracked-LIBRARY cache above. That one caches the top-level
|
||
# artist/series/movie list (one call). This caches the much more expensive per-item walk —
|
||
# one live API call per artist/series — that lidarr_cleanup.sh/sonarr_cleanup.sh already have
|
||
# to do for their own cleanup decisions regardless. Write-through only, from a walk that's
|
||
# already happening: no independent live-fetch-and-refresh path here, since the field set
|
||
# each consumer needs differs (lidarr_cleanup.sh only needs .path; lidarr_missing_art.sh also
|
||
# needs .albumId) and there's no single generic "fetch everything" call worth centralizing —
|
||
# each script keeps its own live per-item fallback, this is purely a fast path in front of it.
|
||
#
|
||
# Freshness is short (default 4h, not the library cache's 1 day) because this is meant for a
|
||
# script running shortly after in the same maintenance window (e.g. lidarr_missing_art.sh
|
||
# right after lidarr_cleanup.sh), not held across a whole day. tmpfs only, no persistent
|
||
# backup — unlike the library cache, nothing unique lives only here (every consumer already
|
||
# has its own live fallback), and it's short-lived by design, so surviving a reboot doesn't
|
||
# matter the way it did for the always-wanted library cache. (2026-07-17)
|
||
#
|
||
# Future scripts needing this data: call arr_get_cached_items() first, fall back to your own
|
||
# live per-item fetch on a miss, and write through via arr_item_cache_write() if you're the
|
||
# one doing that fetch anyway — same pattern as lidarr_cleanup.sh/sonarr_cleanup.sh below.
|
||
# ==============================================================================================
|
||
|
||
arr_item_cache_file() { echo "${ARR_CACHE_DIR}/${1}_items_cache.json"; }
|
||
|
||
# Args: arr_type, items_json (array of raw per-item track/episode-file objects)
|
||
arr_item_cache_write() {
|
||
local arr_type="$1" items_json="$2"
|
||
|
||
local url_var="${arr_type^^}_URL" key_var="${arr_type^^}_API_KEY"
|
||
local url="${!url_var:-}" api_key="${!key_var:-}" api_version="${ARR_API_VERSION[$arr_type]:-}"
|
||
if [[ -n "$url" && -n "$api_key" && -n "$api_version" ]]; then
|
||
local active_cmd
|
||
active_cmd=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version")
|
||
[[ -n "$active_cmd" ]] && return 1
|
||
fi
|
||
|
||
local cache_file items_tmp
|
||
cache_file=$(arr_item_cache_file "$arr_type")
|
||
items_tmp=$(mktemp)
|
||
echo "$items_json" > "$items_tmp"
|
||
mkdir -p "$(dirname "$cache_file")" 2>/dev/null
|
||
jq -c -n --slurpfile items "$items_tmp" --argjson ts "$(date +%s)" \
|
||
'{ts:$ts, items:$items[0]}' > "${cache_file}.tmp" 2>/dev/null \
|
||
&& mv "${cache_file}.tmp" "$cache_file"
|
||
rm -f "$items_tmp"
|
||
}
|
||
|
||
# Echoes the cached per-item array if present and fresher than max_age_seconds, empty/1 on a
|
||
# miss or stale cache — caller falls back to its own live per-item fetch exactly as it already
|
||
# does today.
|
||
# Args: arr_type, max_age_seconds (default 14400 = 4h)
|
||
arr_get_cached_items() {
|
||
local arr_type="$1" max_age="${2:-14400}"
|
||
local cache_file
|
||
cache_file=$(arr_item_cache_file "$arr_type")
|
||
[[ -f "$cache_file" ]] || return 1
|
||
local ts age
|
||
ts=$(jq -r '.ts // 0' "$cache_file" 2>/dev/null)
|
||
age=$(( $(date +%s) - ${ts:-0} ))
|
||
[[ "$age" -ge "$max_age" ]] && return 1
|
||
jq -c '.items // empty' "$cache_file" 2>/dev/null
|
||
}
|
||
|
||
# Records how long a rescan-type command actually took for arr_type, keyed by command name,
|
||
# so future waits can be calibrated per command type instead of guessed or blended across
|
||
# very different operations — a whole-library RescanFolders/RescanSeries/RescanMovie takes
|
||
# vastly longer than a targeted DownloadedXScan, and averaging them would miscalibrate the
|
||
# wait for both.
|
||
# Args: arr_type, command_name, duration_seconds
|
||
arr_record_rescan_duration() {
|
||
local arr_type="$1" cmd_name="$2" duration="$3"
|
||
[[ -z "$cmd_name" || -z "$duration" ]] && return 1
|
||
local db
|
||
db=$(arr_rescan_duration_db "$arr_type")
|
||
mkdir -p "$(dirname "$db")" 2>/dev/null
|
||
local tmp
|
||
tmp=$(mktemp)
|
||
[[ -f "$db" ]] && grep -v "^${cmd_name}|" "$db" > "$tmp" 2>/dev/null
|
||
echo "${cmd_name}|${duration}" >> "$tmp"
|
||
mv "$tmp" "$db"
|
||
}
|
||
|
||
# Echoes the last recorded duration (seconds) for arr_type + command name, or a fallback
|
||
# default if none has ever been recorded.
|
||
# Args: arr_type, command_name, fallback_default_seconds
|
||
arr_get_rescan_duration() {
|
||
local arr_type="$1" cmd_name="$2" fallback="${3:-300}"
|
||
local db
|
||
db=$(arr_rescan_duration_db "$arr_type")
|
||
[[ -f "$db" ]] || { echo "$fallback"; return; }
|
||
local val
|
||
val=$(grep "^${cmd_name}|" "$db" 2>/dev/null | tail -1 | cut -d'|' -f2)
|
||
echo "${val:-$fallback}"
|
||
}
|
||
|
||
# Checks arr_type's command queue for any currently-active (started or queued) rescan-type
|
||
# command (per ARR_RESCAN_COMMANDS). Echoes the command name of the first one found (so the
|
||
# caller can look up its specific historical duration), empty if none active.
|
||
# Args: arr_type, url, api_key, api_version
|
||
arr_active_rescan_command() {
|
||
local arr_type="$1" url="$2" api_key="$3" api_version="$4"
|
||
local names="${ARR_RESCAN_COMMANDS[$arr_type]:-}"
|
||
[[ -z "$names" ]] && return 1
|
||
local jq_names
|
||
jq_names=$(printf '%s\n' $names | jq -R . | jq -sc .)
|
||
curl -sf --max-time 15 -H "X-Api-Key: $api_key" \
|
||
"${url}/api/${api_version}/command" 2>/dev/null | \
|
||
jq -r --argjson names "$jq_names" '[.[] | select(
|
||
(.status=="started" or .status=="queued") and
|
||
(.name as $n | $names | index($n) != null)
|
||
)] | .[0].name // empty' 2>/dev/null
|
||
}
|
||
|
||
# Waits for an already-active rescan-type command for arr_type to finish — unlike
|
||
# trigger_and_await_command(), this never triggers anything, it only watches. For a rescan
|
||
# someone/something else started (manual intervention, another script) where the caller just
|
||
# needs to know when it's safe to fetch+cache the real post-scan numbers. Returns 0 once no
|
||
# active rescan-type command remains (including immediately if none was active to begin
|
||
# with), 1 on timeout.
|
||
# Args: arr_type, url, api_key, api_version, poll_timeout (default 7200s), poll_interval (default 15s)
|
||
arr_wait_for_active_rescan() {
|
||
local arr_type="$1" url="$2" api_key="$3" api_version="$4"
|
||
local poll_timeout="${5:-7200}" poll_interval="${6:-15}"
|
||
local polled=0 active
|
||
active=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version")
|
||
[[ -z "$active" ]] && return 0
|
||
while [[ "$polled" -lt "$poll_timeout" ]]; do
|
||
sleep "$poll_interval"
|
||
(( polled += poll_interval ))
|
||
active=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version")
|
||
[[ -z "$active" ]] && return 0
|
||
[[ $(( polled % 300 )) -lt "$poll_interval" ]] && log " Still waiting on ${arr_type}'s ${active} (${polled}s elapsed)"
|
||
done
|
||
return 1
|
||
}
|
||
|
||
# Main entry point — the only function consuming scripts should call for tracked library
|
||
# data. Handles fresh/stale-no-rescan/stale-rescan-active branching and always writes through
|
||
# on any live fetch it performs. Echoes the library-list JSON array on success, returns 1 if
|
||
# no usable data (no cache and live fetch impossible) could be obtained.
|
||
# Args: arr_type, url, api_key, api_version, max_age_days (default 1), max_wait_strikes (default 3)
|
||
arr_get_tracked_data() {
|
||
local arr_type="$1" url="$2" api_key="$3" api_version="$4"
|
||
local max_age_days="${5:-${LIDARR_CACHE_MAX_AGE_DAYS:-1}}" max_strikes="${6:-3}"
|
||
local max_age_seconds=$(( max_age_days * 86400 ))
|
||
local age
|
||
age=$(arr_cache_age_seconds "$arr_type")
|
||
|
||
if [[ "$age" -lt "$max_age_seconds" ]]; then
|
||
arr_cache_read_raw "$arr_type" && return 0
|
||
fi
|
||
|
||
# Cache missing or stale — check for an active rescan before deciding how to proceed.
|
||
local active_cmd
|
||
active_cmd=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version")
|
||
|
||
if [[ -n "$active_cmd" ]]; then
|
||
local wait_duration strike arr_label
|
||
wait_duration=$(( $(arr_get_rescan_duration "$arr_type" "$active_cmd" 300) / 2 ))
|
||
[[ "$wait_duration" -lt 30 ]] && wait_duration=30
|
||
arr_label="$(tr '[:lower:]' '[:upper:]' <<< "${arr_type:0:1}")${arr_type:1}"
|
||
for (( strike=1; strike<=max_strikes; strike++ )); do
|
||
# Redirected to stderr — this function's stdout is a data channel (callers
|
||
# capture it via command substitution), never mix log/warn output into it.
|
||
warn " $arr_label busy ($active_cmd) — waiting ${wait_duration}s (strike ${strike}/${max_strikes})" >&2
|
||
sleep "$wait_duration"
|
||
active_cmd=$(arr_active_rescan_command "$arr_type" "$url" "$api_key" "$api_version")
|
||
[[ -z "$active_cmd" ]] && break
|
||
done
|
||
if [[ -n "$active_cmd" ]]; then
|
||
warn " $arr_label still busy ($active_cmd) after ${max_strikes} strikes — using cache if present, else skipping" >&2
|
||
arr_cache_read_raw "$arr_type" && return 0
|
||
return 1
|
||
fi
|
||
fi
|
||
|
||
# No active rescan (or it just finished mid-wait) — safe to fetch live and refresh cache.
|
||
local endpoint fresh arr_label
|
||
endpoint="${ARR_LIBRARY_ENDPOINT[$arr_type]:-}"
|
||
[[ -z "$endpoint" ]] && return 1
|
||
arr_label="$(tr '[:lower:]' '[:upper:]' <<< "${arr_type:0:1}")${arr_type:1}"
|
||
fresh=$(arr_api "$url" "$api_key" "$api_version" "$endpoint" "$arr_label") || {
|
||
arr_cache_read_raw "$arr_type" && return 0
|
||
return 1
|
||
}
|
||
arr_cache_write "$arr_type" "$fresh"
|
||
echo "$fresh"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── ARR VERSION CHECK ─────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Verifies arr major version matches tested version in master.conf.
|
||
# Exits if version doesn't match — prevents running against untested API structure.
|
||
# Warns and proceeds if version endpoint is unreachable.
|
||
#
|
||
# Usage:
|
||
# check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" "Radarr"
|
||
#
|
||
# Arguments:
|
||
# $1 = arr base URL
|
||
# $2 = API key
|
||
# $3 = API path prefix (v3 or v1)
|
||
# $4 = expected major version number from master.conf
|
||
# $5 = arr name for error messages
|
||
|
||
check_arr_version() {
|
||
local url="$1"
|
||
local api_key="$2"
|
||
local api_prefix="$3"
|
||
local expected_major="$4"
|
||
local arr_name="${5:-Arr}"
|
||
|
||
local status_response version major_version
|
||
|
||
status_response=$(curl -sf --max-time 10 \
|
||
-H "X-Api-Key: $api_key" \
|
||
"${url}/api/${api_prefix}/system/status" 2>/dev/null)
|
||
|
||
if [[ -z "$status_response" ]]; then
|
||
warn "$arr_name version check failed — could not reach system/status endpoint"
|
||
warn "Proceeding without version verification — monitor for API errors"
|
||
return 0
|
||
fi
|
||
|
||
version=$(echo "$status_response" | \
|
||
grep -o '"version": *"[^"]*"' | \
|
||
grep -o '[0-9][^"]*' | head -1)
|
||
|
||
if [[ -z "$version" ]]; then
|
||
warn "$arr_name version check failed — could not parse version from response"
|
||
warn "Proceeding without version verification — monitor for API errors"
|
||
return 0
|
||
fi
|
||
|
||
major_version="${version%%.*}"
|
||
|
||
if [[ "$major_version" == "$expected_major" ]]; then
|
||
success "$arr_name version: $version (major $major_version — tested ✅)"
|
||
return 0
|
||
else
|
||
error "$arr_name version mismatch — running v${major_version}, tested against v${expected_major}"
|
||
error "The API endpoint structure may have changed — exiting to protect your library"
|
||
error "Update ${arr_name^^}_VERSION_MAJOR in master.conf after verifying the script works with v${major_version}"
|
||
notify "$arr_name version mismatch on $(hostname) — running v${major_version}, script tested against v${expected_major}" "$arr_name Cleanup" "warning"
|
||
exit 1
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── TMDB DISCOVERY SCORING ────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Shared by playback_aware_radarr_discovery.sh and playback_aware_sonarr_discovery.sh — stage 2
|
||
# candidate scoring against TMDB rating/vote/seed-breadth data. Byte-for-byte identical between
|
||
# the two before this consolidation.
|
||
|
||
# Formats a TMDB vote_average×10 integer back to one decimal place (e.g. 78 → 7.8).
|
||
_fmt_rating() { local v="${1:-0}"; echo "${v::-1}.${v: -1}" 2>/dev/null || echo "$v"; }
|
||
|
||
# Stage 2: TMDB vote_average × 10 as integer (0-40)
|
||
_rating_score_s2() {
|
||
local v="$1"
|
||
if (( v >= 80 )); then echo 40
|
||
elif (( v >= 75 )); then echo 32
|
||
elif (( v >= 70 )); then echo 25
|
||
elif (( v >= 65 )); then echo 18
|
||
elif (( v >= 60 )); then echo 12
|
||
else echo 5
|
||
fi
|
||
}
|
||
|
||
# Stage 2: vote count (0-20)
|
||
_votes_score() {
|
||
local c="$1"
|
||
if (( c >= 10000 )); then echo 20
|
||
elif (( c >= 5000 )); then echo 15
|
||
elif (( c >= 1000 )); then echo 10
|
||
elif (( c >= 200 )); then echo 5
|
||
else echo 2
|
||
fi
|
||
}
|
||
|
||
# Stage 2: seed breadth (0-40)
|
||
_breadth_score() {
|
||
local seeds="$1"
|
||
if (( seeds >= 3 )); then echo 40
|
||
elif (( seeds == 2 )); then echo 25
|
||
else echo 10
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── EMBY LIBRARY SCAN ─────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Triggers Emby "Clean Missing Files" scheduled task after arr cleanup scripts delete files.
|
||
# Without this Emby keeps showing deleted files as ghost entries until its next scheduled scan.
|
||
# User clicks a ghost entry → file not found error. This prevents that entirely.
|
||
#
|
||
# Uses EMBY_URL and EMBY_API_KEY — aliased by detect_hosts() from HOST*_EMBY_* vars.
|
||
# Skips cleanly if Emby is not configured on this host (empty URL or API key).
|
||
#
|
||
# Emby task discovery:
|
||
# GET /ScheduledTasks → find task where Name contains "Clean Missing"
|
||
# POST /ScheduledTasks/Running/{taskId} → trigger it
|
||
#
|
||
# Non-fatal — logs warning if task cannot be triggered but does not exit.
|
||
# Clean Missing Files runs across all libraries at once — one call handles
|
||
# music, TV, and movies together regardless of which arr triggered the cleanup.
|
||
#
|
||
# Usage: notify_emby_scan (no args — always triggers Clean Missing Files)
|
||
|
||
notify_emby_scan() {
|
||
# Skip if Emby not configured on this host
|
||
if [[ -z "${EMBY_URL:-}" ]] || [[ -z "${EMBY_API_KEY:-}" ]]; then
|
||
log "Emby not configured on $MY_ID — skipping library scan notification"
|
||
return 0
|
||
fi
|
||
|
||
log "Notifying Emby to clean missing files..."
|
||
|
||
# Get all scheduled tasks
|
||
local tasks_response
|
||
tasks_response=$(curl -sf --max-time 15 \
|
||
-H "X-Emby-Token: $EMBY_API_KEY" \
|
||
"${EMBY_URL}/ScheduledTasks" 2>/dev/null)
|
||
|
||
if [[ -z "$tasks_response" ]]; then
|
||
warn "Could not reach Emby scheduled tasks API at $EMBY_URL — skipping scan"
|
||
return 0
|
||
fi
|
||
|
||
# Find the "Clean Missing Files" task ID
|
||
local task_id
|
||
task_id=$(echo "$tasks_response" | \
|
||
grep -o '"Id":"[^"]*","Name":"[^"]*Clean Missing[^"]*"' | \
|
||
grep -o '"Id":"[^"]*"' | \
|
||
sed 's/"Id":"//;s/"//' | head -1)
|
||
|
||
# Fallback — try "Scan Media Library" if Clean Missing not found
|
||
if [[ -z "$task_id" ]]; then
|
||
task_id=$(echo "$tasks_response" | \
|
||
grep -o '"Id":"[^"]*","Name":"[^"]*Scan Media Library[^"]*"' | \
|
||
grep -o '"Id":"[^"]*"' | \
|
||
sed 's/"Id":"//;s/"//' | head -1)
|
||
[[ -n "$task_id" ]] && log "Clean Missing Files not found — using Scan Media Library"
|
||
fi
|
||
|
||
if [[ -z "$task_id" ]]; then
|
||
warn "Could not find Emby Clean Missing Files or Scan Media Library task"
|
||
warn "Check Emby scheduled tasks — ghost entries will persist until next Emby scan"
|
||
return 0
|
||
fi
|
||
|
||
log "Found Emby task ID: $task_id"
|
||
|
||
# Trigger the task
|
||
local http_code
|
||
http_code=$(curl -sf --max-time 15 \
|
||
-X POST \
|
||
-H "X-Emby-Token: $EMBY_API_KEY" \
|
||
-w "%{http_code}" \
|
||
-o /dev/null \
|
||
"${EMBY_URL}/ScheduledTasks/Running/${task_id}" 2>/dev/null)
|
||
|
||
if [[ "$http_code" == "204" ]] || [[ "$http_code" == "200" ]]; then
|
||
warn "$ICON_EMBY Emby Clean Missing Files triggered — ghost entries will be removed"
|
||
else
|
||
warn "Emby task trigger returned HTTP $http_code — ghost entries may persist"
|
||
warn "Check Emby dashboard — manually run Clean Missing Files if needed"
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── API REACHABILITY CHECK ────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Pre-flight check — verifies an API endpoint is reachable before attempting operations.
|
||
# Non-fatal — returns 0 if reachable, 1 if not. Caller decides whether to exit.
|
||
#
|
||
# Usage: check_api "http://localhost:8989" "Sonarr" || exit 1
|
||
|
||
check_api() {
|
||
local url="$1"
|
||
local service="${2:-API}"
|
||
local timeout="${3:-10}"
|
||
|
||
if curl -sf --max-time "$timeout" "$url" >/dev/null 2>&1; then
|
||
log "$service API reachable: $url"
|
||
return 0
|
||
else
|
||
error "$service API not reachable: $url"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── UNRAID VERSION PARITY CHECK ───────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Verifies local and remote servers are running compatible unRAID versions before any
|
||
# remote operation. Version mismatches can mean changed APIs, commands, or behaviours
|
||
# that silently break remote container operations, rsync, or fallback logic.
|
||
#
|
||
# Reads platform OS version on both sides via adapter.
|
||
#
|
||
# Mismatch behaviour (UNRAID_VERSION_MISMATCH_ACTION in master.conf):
|
||
# "warn" — log warning and continue (default for minor/patch differences)
|
||
# "abort" — exit the calling script (default for major version differences)
|
||
# Major version mismatch always aborts regardless of setting
|
||
#
|
||
# Usage: check_os_version_parity || exit 1
|
||
|
||
check_os_version_parity() {
|
||
local local_version remote_version
|
||
|
||
# Read local version
|
||
local_version=$(platform_get_os_version 2>/dev/null)
|
||
if [[ -z "$local_version" ]]; then
|
||
warn "Cannot read local OS version — skipping version parity check"
|
||
return 0
|
||
fi
|
||
|
||
# Read remote version via SSH
|
||
local _probe_cmd
|
||
_probe_cmd=$(platform_os_version_probe_cmd)
|
||
remote_version=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
||
"$_probe_cmd" 2>/dev/null)
|
||
if [[ -z "$remote_version" ]]; then
|
||
warn "Cannot read remote OS version from $REMOTE_SERVER_NAME — skipping parity check"
|
||
return 0
|
||
fi
|
||
|
||
log "Version parity: local=$local_version remote=$remote_version"
|
||
|
||
if [[ "$local_version" == "$remote_version" ]]; then
|
||
log "OS versions match: $local_version ✅"
|
||
return 0
|
||
fi
|
||
|
||
# Parse major versions
|
||
local local_major remote_major
|
||
local_major="${local_version%%.*}"
|
||
remote_major="${remote_version%%.*}"
|
||
|
||
if [[ "$local_major" != "$remote_major" ]]; then
|
||
error "OS MAJOR version mismatch — local: $local_version remote: $remote_version"
|
||
error "Major version differences may break remote APIs, commands, and behaviours"
|
||
error "Update both servers to the same major version before running remote operations"
|
||
notify "OS major version mismatch on $(hostname) — local: $local_version remote: $remote_version — remote operations aborted" \
|
||
"Version Parity" "warning"
|
||
return 1
|
||
fi
|
||
|
||
# Minor/patch mismatch — action depends on config
|
||
local action="${UNRAID_VERSION_MISMATCH_ACTION:-warn}"
|
||
warn "OS version mismatch — local: $local_version remote: $remote_version"
|
||
|
||
if [[ "$action" == "abort" ]]; then
|
||
error "UNRAID_VERSION_MISMATCH_ACTION=abort — refusing to continue"
|
||
notify "OS version mismatch on $(hostname) — local: $local_version remote: $remote_version — aborted" \
|
||
"Version Parity" "warning"
|
||
return 1
|
||
fi
|
||
|
||
warn "UNRAID_VERSION_MISMATCH_ACTION=warn — continuing despite mismatch (monitor for issues)"
|
||
return 0
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── REMOTE DOCKER DAEMON CHECK ────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Verifies the remote Docker daemon is responsive before issuing any remote container commands.
|
||
# A hung remote daemon causes docker commands to hang or silently fail — checking first
|
||
# prevents cascading failures and gives clear diagnostic output.
|
||
#
|
||
# Strike system:
|
||
# Each consecutive failed check adds a strike to REMOTE_DOCKER_STRIKES
|
||
# Below REMOTE_DOCKER_STRIKE_LIMIT → warn and return 1 (caller skips operation)
|
||
# At limit → notify critical and return 1 (caller should exit or escalate)
|
||
# Strikes reset when daemon recovers
|
||
#
|
||
# Usage: check_remote_docker_daemon || return 1
|
||
|
||
REMOTE_DOCKER_STRIKES=0
|
||
|
||
check_remote_docker_daemon() {
|
||
local timeout="${1:-10}"
|
||
|
||
if ssh -i "$SSH_KEY" -o ConnectTimeout="$timeout" root@"$REMOTE_SERVER" \
|
||
"timeout $timeout docker info" >/dev/null 2>&1; then
|
||
# Daemon healthy — clear strikes if we had issues
|
||
if [[ "$REMOTE_DOCKER_STRIKES" -gt 0 ]]; then
|
||
info "Remote Docker daemon on $REMOTE_SERVER_NAME recovered — clearing strikes"
|
||
REMOTE_DOCKER_STRIKES=0
|
||
fi
|
||
log "Remote Docker daemon healthy on $REMOTE_SERVER_NAME"
|
||
return 0
|
||
fi
|
||
|
||
REMOTE_DOCKER_STRIKES=$(( REMOTE_DOCKER_STRIKES + 1 ))
|
||
local limit="${REMOTE_DOCKER_STRIKE_LIMIT:-3}"
|
||
warn "Remote Docker daemon not responding on $REMOTE_SERVER_NAME (strike $REMOTE_DOCKER_STRIKES/$limit)"
|
||
|
||
if [[ "$REMOTE_DOCKER_STRIKES" -ge "$limit" ]]; then
|
||
error "Remote Docker daemon unresponsive on $REMOTE_SERVER_NAME after $limit consecutive checks"
|
||
error "Skipping all remote container operations — check Docker on $REMOTE_SERVER_NAME"
|
||
notify "Remote Docker daemon unresponsive on $REMOTE_SERVER_NAME — remote operations skipped on $(hostname)" "Remote Docker Daemon" "warning"
|
||
fi
|
||
|
||
return 1
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── STATUS DISPLAY ────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Prints current runtime configuration — triggered by --status flag in any script.
|
||
# Useful for verifying detect_hosts() resolved correctly and profile loaded as expected.
|
||
|
||
show_status() {
|
||
local local_ver
|
||
local_ver=$(platform_get_os_version 2>/dev/null || echo "unknown")
|
||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||
echo "Local: $LOCAL_SERVER_NAME"
|
||
echo "Remote: $REMOTE_SERVER_NAME"
|
||
echo "IP: ${REMOTE_SERVER:-not resolved}"
|
||
echo "My ID: ${MY_ID:-not set}"
|
||
echo "Remote ID: ${REMOTE_ID:-not set}"
|
||
echo "OS ver: $local_ver"
|
||
echo "Profile: ${PROFILE_NAME:-n/a}"
|
||
|
||
echo "DryRun: ${DRY_RUN:-false}"
|
||
echo "Logging: ${ENABLE_LOGGING:-false}"
|
||
echo "SSH Key: ${SSH_KEY:-not set}"
|
||
echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]:-n/a}"
|
||
echo "Delayed: ${DELAYED_CONTAINERS[*]:-n/a}"
|
||
echo "Excludes: ${EXCLUDE_DIRS[*]:-n/a}"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
}
|
||
|
||
require_partnership() {
|
||
[[ "${PARTNERSHIP_ENABLED:-false}" == "true" ]] && return 0
|
||
log "PARTNERSHIP_ENABLED=false — skipping cross-server operation"
|
||
exit 0
|
||
} |